CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser: WARNING: ThreadSanitizer: data race (pid=929) Write of size 8 at 0x7b44001f3b88 by main thread:
#0 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::__destruct_at_end(kudu::MemTracker**) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:429:12 (libkudu_util.so+0x3a6d30)
#1 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libkudu_util.so+0x3a6c44)
#2 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libkudu_util.so+0x3a6a3b)
#3 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libkudu_util.so+0x39ecf1)
#4 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x39e8a3)
#5 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x3a5b65)
#6 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libtablet_test_util.so+0x35485)
#7 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libtablet_test_util.so+0x35429)
#8 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#9 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41c8f4)
#10 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cf6d5)
#11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cf8e9)
#12 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1002d7)
#13 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x100299)
#14 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100259)
#15 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0x1001ba)
#16 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc669)
#17 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc640)
#18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc600)
#19 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b1096)
#20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b0fb4)
#21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b0d6b)
#22 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x289451)
#23 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b43be)
#24 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#25 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#26 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#27 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#28 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#29 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#30 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#31 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#32 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#33 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#34 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#35 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#36 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#37 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#38 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#39 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#40 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#41 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#42 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#43 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#44 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#45 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#46 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Previous read of size 8 at 0x7b44001f3b88 by thread T148 (mutexes: write M165642749574833856):
#0 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::end() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:1536:30 (libkudu_util.so+0x39fc0a)
#1 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:284:22 (libkudu_util.so+0x39d2e6)
#2 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d2c0)
#3 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416322)
#4 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415ab3)
#5 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419cc1)
#6 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419c79)
#7 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419c09)
#8 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419bd1)
#9 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x418dfd)
#10 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#11 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#12 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#13 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#14 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#15 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#16 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#17 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#18 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#19 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#20 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e85c2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d66a6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41dba0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d57e)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d13e7)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1a2e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b42b7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Location is heap block of size 264 at 0x7b44001f3b00 allocated by thread T187:
#0 operator new(unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:64 (catalog_manager-test+0x342a87)
#1 std::__1::__libcpp_allocate(unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:253:10 (libkudu_util.so+0x29a306)
#2 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::allocate(unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1789:34 (libkudu_util.so+0x3a5445)
#3 std::__1::enable_if<!(is_array<long&>::value), std::__1::shared_ptr<long&> >::type std::__1::make_shared<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler&&...) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4290:45 (libkudu_util.so+0x3a5221)
#4 std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/make_shared.h:61:12 (libkudu_util.so+0x39eb55)
#5 kudu::MemTracker::CreateTracker(long, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:54:34 (libkudu_util.so+0x39cb7e)
#6 kudu::tablet::OpTracker::StartMemoryTracking(std::__1::shared_ptr<kudu::MemTracker> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:311:20 (libtablet.so+0x41dd7b)
#7 kudu::tablet::TabletReplica::Start(kudu::consensus::ConsensusBootstrapInfo const&, std::__1::shared_ptr<kudu::tablet::Tablet>, kudu::clock::Clock*, std::__1::shared_ptr<kudu::rpc::Messenger>, scoped_refptr<kudu::rpc::ResultTracker>, scoped_refptr<kudu::log::Log>, kudu::ThreadPool*, kudu::DnsResolver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:277:19 (libtablet.so+0x3d0736)
#8 kudu::tserver::TSTabletManager::OpenTablet(scoped_refptr<kudu::tablet::TabletReplica> const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&, std::__1::atomic<int>*, std::__1::atomic<int>*, kudu::Timer*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1430:18 (libtserver.so+0x2b9b90)
#9 kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:703:3 (libtserver.so+0x2cdb01)
#10 decltype(std::__1::forward<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&>(fp)()) std::__1::__invoke<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&>(kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x2cdaa9)
#11 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&>(kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x2cda39)
#12 std::__1::__function::__alloc_func<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11, std::__1::allocator<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x2cda01)
#13 std::__1::__function::__func<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11, std::__1::allocator<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x2cccdd)
#14 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#15 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#16 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#17 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#18 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#19 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#20 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#21 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#22 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#23 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#24 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
Mutex M165642749574833856 is already destroyed.
Thread T148 'apply [worker]-' (tid=2051, running) created by thread T68 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c7095)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643:15 (libkudu_util.so+0x4423c8)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libmaster.so+0x2d4b89)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45ea4b)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45ccef)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45efb2)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x414e50)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x4153e9)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x41767a)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417608)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x417588)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417543)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41675c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3ea336)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4b88)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa2c0)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x1819a3)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b294)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138644)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1385f3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138568)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138523)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13780c)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x1614a6)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x14946b)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x133acd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x1374a5)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137439)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1373c9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137391)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13668d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
Thread T187 (tid=2018, finished) created by thread T125 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c7095)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643:15 (libkudu_util.so+0x4423c8)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libmaster.so+0x2d4b89)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45ea4b)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45ccef)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45efb2)
#6 kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:703:3 (libtserver.so+0x2b6d82)
#7 kudu::tserver::TabletServiceAdminImpl::CreateTablet(kudu::tserver::CreateTabletRequestPB const*, kudu::tserver::CreateTabletResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1475:34 (libtserver.so+0x267209)
#8 kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/tserver/tserver_admin.service.cc:189:13 (libtserver_admin_proto.so+0x68bf4)
#9 decltype(std::__1::forward<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver_admin_proto.so+0x68b82)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver_admin_proto.so+0x68ab1)
#11 std::__1::__function::__alloc_func<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver_admin_proto.so+0x68a2c)
#12 std::__1::__function::__func<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver_admin_proto.so+0x67ce2)
#13 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x1f3ccc)
#14 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x1f3106)
#15 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x1f2aaf)
#16 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:229:15 (libkrpc.so+0x1f5dc3)
#17 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:92:5 (libkrpc.so+0x1f70e1)
#18 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x1f7099)
#19 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x1f7029)
#20 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x1f6ff1)
#21 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x1f62ed)
#22 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#23 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#24 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser: WARNING: ThreadSanitizer: data race (pid=929) Write of size 8 at 0x7b08000a3340 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x343359)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34f289)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34f229)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29aaa2)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29aa39)
#5 std::__1::allocator<kudu::MemTracker*>::deallocate(kudu::MemTracker**, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a6e0a)
#6 std::__1::allocator_traits<std::__1::allocator<kudu::MemTracker*> >::deallocate(std::__1::allocator<kudu::MemTracker*>&, kudu::MemTracker**, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1533:14 (libkudu_util.so+0x3a6c89)
#7 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:466:9 (libkudu_util.so+0x3a6a67)
#8 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libkudu_util.so+0x39ecf1)
#9 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x39e8a3)
#10 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x3a5b65)
#11 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libtablet_test_util.so+0x35485)
#12 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libtablet_test_util.so+0x35429)
#13 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#14 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41c8f4)
#15 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cf6d5)
#16 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cf8e9)
#17 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1002d7)
#18 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x100299)
#19 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100259)
#20 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0x1001ba)
#21 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc669)
#22 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc640)
#23 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc600)
#24 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b1096)
#25 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b0fb4)
#26 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b0d6b)
#27 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x289451)
#28 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b43be)
#29 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#30 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#31 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#32 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#33 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#34 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#35 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#36 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#37 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#38 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#39 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#40 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#41 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#42 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#43 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#44 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#45 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#46 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#47 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#48 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#49 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#50 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#51 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Previous read of size 8 at 0x7b08000a3340 by thread T148 (mutexes: write M165642749574833856):
#0 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:5 (libkudu_util.so+0x39d322)
#1 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d2c0)
#2 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416322)
#3 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415ab3)
#4 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419cc1)
#5 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419c79)
#6 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419c09)
#7 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419bd1)
#8 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x418dfd)
#9 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#10 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#11 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#12 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#13 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#14 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#15 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#16 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#17 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#18 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#19 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e85c2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d66a6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41dba0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d57e)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d13e7)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1a2e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b42b7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Mutex M165642749574833856 is already destroyed.
Thread T148 'apply [worker]-' (tid=2051, running) created by thread T68 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c7095)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643:15 (libkudu_util.so+0x4423c8)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libmaster.so+0x2d4b89)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45ea4b)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45ccef)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45efb2)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x414e50)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x4153e9)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x41767a)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417608)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x417588)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417543)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41675c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3ea336)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4b88)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa2c0)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x1819a3)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b294)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138644)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1385f3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138568)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138523)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13780c)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x1614a6)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x14946b)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x133acd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x1374a5)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137439)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1373c9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137391)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13668d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser: WARNING: ThreadSanitizer: data race (pid=929) Write of size 8 at 0x7b44001f32b0 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x343359)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34f289)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34f229)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29aaa2)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29aa39)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a622d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a5bd0)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x39e8ab)
#11 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x3a5b65)
#12 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libtablet_test_util.so+0x35485)
#13 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libtablet_test_util.so+0x35429)
#14 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#15 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41c8f4)
#16 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cf6d5)
#17 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cf8e9)
#18 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1002d7)
#19 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x100299)
#20 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100259)
#21 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0x1001ba)
#22 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc669)
#23 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc640)
#24 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc600)
#25 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b1096)
#26 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b0fb4)
#27 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b0d6b)
#28 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x289451)
#29 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b43be)
#30 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#31 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#32 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#33 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#34 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#35 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#36 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#37 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#38 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#39 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#40 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#41 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#42 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#43 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#44 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#45 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#46 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#47 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#48 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#49 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#50 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#51 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#52 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Previous atomic write of size 8 at 0x7b44001f32b0 by thread T148 (mutexes: write M165642749574833856):
#0 __tsan_atomic64_fetch_add /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:620 (catalog_manager-test+0x316635)
#1 long std::__1::__cxx_atomic_fetch_add<long>(std::__1::__cxx_atomic_base_impl<long>*, long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1042:12 (libkudu_util.so+0x2a5263)
#2 std::__1::__atomic_base<long, true>::fetch_add(long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1706:17 (libkudu_util.so+0x2a51a9)
#3 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:30 (libkudu_util.so+0x39fcc2)
#4 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x39d331)
#5 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d2c0)
#6 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416322)
#7 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415ab3)
#8 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419cc1)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419c79)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419c09)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419bd1)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x418dfd)
#13 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#14 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#15 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#16 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#17 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#18 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#19 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#20 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#21 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#22 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#23 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e85c2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d66a6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41dba0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d57e)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d13e7)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1a2e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b42b7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Mutex M165642749574833856 is already destroyed.
Thread T148 'apply [worker]-' (tid=2051, running) created by thread T68 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c7095)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643:15 (libkudu_util.so+0x4423c8)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libmaster.so+0x2d4b89)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45ea4b)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45ccef)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45efb2)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x414e50)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x4153e9)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x41767a)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417608)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x417588)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417543)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41675c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3ea336)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4b88)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa2c0)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x1819a3)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b294)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138644)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1385f3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138568)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138523)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13780c)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x1614a6)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x14946b)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x133acd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x1374a5)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137439)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1373c9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137391)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13668d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser: WARNING: ThreadSanitizer: data race (pid=929) Write of size 8 at 0x7b44001f32b8 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x343359)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34f289)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34f229)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29aaa2)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29aa39)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a622d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a5bd0)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x39e8ab)
#11 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x3a5b65)
#12 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libtablet_test_util.so+0x35485)
#13 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libtablet_test_util.so+0x35429)
#14 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#15 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41c8f4)
#16 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cf6d5)
#17 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cf8e9)
#18 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1002d7)
#19 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x100299)
#20 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100259)
#21 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0x1001ba)
#22 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc669)
#23 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc640)
#24 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc600)
#25 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b1096)
#26 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b0fb4)
#27 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b0d6b)
#28 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x289451)
#29 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b43be)
#30 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#31 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#32 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#33 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#34 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#35 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#36 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#37 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#38 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#39 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#40 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#41 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#42 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#43 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#44 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#45 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#46 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#47 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#48 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#49 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#50 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#51 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#52 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Previous atomic write of size 8 at 0x7b44001f32b8 by thread T148 (mutexes: write M165642749574833856):
#0 __tsan_atomic64_compare_exchange_val /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:850 (catalog_manager-test+0x31d708)
#1 bool std::__1::__cxx_atomic_compare_exchange_weak<long>(std::__1::__cxx_atomic_base_impl<long>*, long*, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1031:12 (liblog.so+0x8bd24)
#2 std::__1::__atomic_base<long, false>::compare_exchange_weak(long&, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1622:17 (liblog.so+0x8bb49)
#3 void kudu::AtomicStoreMax<long>(std::__1::atomic<long>&, long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/atomic-utils.h:33:13 (liblog.so+0x7eaa0)
#4 kudu::HighWaterMark::UpdateMax(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:80:5 (libkudu_util.so+0x3a4e64)
#5 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:5 (libkudu_util.so+0x39fccd)
#6 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x39d331)
#7 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d2c0)
#8 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416322)
#9 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415ab3)
#10 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419cc1)
#11 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419c79)
#12 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419c09)
#13 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419bd1)
#14 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x418dfd)
#15 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#16 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#17 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#18 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#19 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#21 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#22 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#23 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#24 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#25 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e85c2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d66a6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41dba0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d57e)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d13e7)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1a2e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b42b7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Mutex M165642749574833856 is already destroyed.
Thread T148 'apply [worker]-' (tid=2051, running) created by thread T68 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c7095)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643:15 (libkudu_util.so+0x4423c8)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libmaster.so+0x2d4b89)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45ea4b)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45ccef)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45efb2)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x414e50)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x4153e9)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x41767a)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417608)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x417588)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417543)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41675c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3ea336)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4b88)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa2c0)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x1819a3)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b294)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138644)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1385f3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138568)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138523)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13780c)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x1614a6)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x14946b)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x133acd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x1374a5)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137439)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1373c9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137391)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13668d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser: WARNING: ThreadSanitizer: data race (pid=929) Write of size 8 at 0x7b44001f3b70 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x343359)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34f289)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34f229)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29aaa2)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29aa39)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a622d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a5bd0)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41c8f4)
#11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cf6d5)
#12 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cf8e9)
#13 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1002d7)
#14 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x100299)
#15 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100259)
#16 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0x1001ba)
#17 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc669)
#18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc640)
#19 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc600)
#20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b1096)
#21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b0fb4)
#22 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b0d6b)
#23 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x289451)
#24 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b43be)
#25 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#26 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#27 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#28 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#29 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#30 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#31 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#32 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#33 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#34 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#35 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#36 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#37 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#38 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#39 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#40 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#41 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#42 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#43 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#44 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#45 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#46 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#47 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Previous atomic write of size 8 at 0x7b44001f3b70 by thread T148 (mutexes: write M165642749574833856):
#0 __tsan_atomic64_fetch_add /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:620 (catalog_manager-test+0x316635)
#1 long std::__1::__cxx_atomic_fetch_add<long>(std::__1::__cxx_atomic_base_impl<long>*, long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1042:12 (libkudu_util.so+0x2a5263)
#2 std::__1::__atomic_base<long, true>::fetch_add(long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1706:17 (libkudu_util.so+0x2a51a9)
#3 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:30 (libkudu_util.so+0x39fcc2)
#4 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x39d331)
#5 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d2c0)
#6 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416322)
#7 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415ab3)
#8 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419cc1)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419c79)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419c09)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419bd1)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x418dfd)
#13 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#14 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#15 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#16 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#17 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#18 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#19 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#20 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#21 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#22 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#23 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e85c2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d66a6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41dba0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d57e)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d13e7)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1a2e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b42b7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Mutex M165642749574833856 is already destroyed.
Thread T148 'apply [worker]-' (tid=2051, running) created by thread T68 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c7095)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643:15 (libkudu_util.so+0x4423c8)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libmaster.so+0x2d4b89)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45ea4b)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45ccef)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45efb2)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x414e50)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x4153e9)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x41767a)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417608)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x417588)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417543)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41675c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3ea336)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4b88)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa2c0)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x1819a3)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b294)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138644)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1385f3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138568)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138523)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13780c)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x1614a6)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x14946b)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x133acd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x1374a5)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137439)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1373c9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137391)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13668d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser: WARNING: ThreadSanitizer: data race (pid=929) Write of size 8 at 0x7b44001f3b78 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x343359)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34f289)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34f229)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29aaa2)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29aa39)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a622d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a5bd0)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41c8f4)
#11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cf6d5)
#12 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cf8e9)
#13 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1002d7)
#14 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x100299)
#15 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100259)
#16 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0x1001ba)
#17 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc669)
#18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc640)
#19 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc600)
#20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b1096)
#21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b0fb4)
#22 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b0d6b)
#23 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x289451)
#24 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b43be)
#25 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#26 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#27 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#28 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#29 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#30 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#31 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#32 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#33 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#34 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#35 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#36 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#37 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#38 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#39 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#40 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#41 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#42 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#43 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#44 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#45 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#46 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#47 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Previous atomic write of size 8 at 0x7b44001f3b78 by thread T148 (mutexes: write M165642749574833856):
#0 __tsan_atomic64_compare_exchange_val /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:850 (catalog_manager-test+0x31d708)
#1 bool std::__1::__cxx_atomic_compare_exchange_weak<long>(std::__1::__cxx_atomic_base_impl<long>*, long*, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1031:12 (liblog.so+0x8bd24)
#2 std::__1::__atomic_base<long, false>::compare_exchange_weak(long&, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1622:17 (liblog.so+0x8bb49)
#3 void kudu::AtomicStoreMax<long>(std::__1::atomic<long>&, long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/atomic-utils.h:33:13 (liblog.so+0x7eaa0)
#4 kudu::HighWaterMark::UpdateMax(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:80:5 (libkudu_util.so+0x3a4e64)
#5 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:5 (libkudu_util.so+0x39fccd)
#6 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x39d331)
#7 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d2c0)
#8 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416322)
#9 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415ab3)
#10 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419cc1)
#11 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419c79)
#12 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419c09)
#13 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419bd1)
#14 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x418dfd)
#15 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#16 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#17 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#18 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#19 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#21 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#22 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#23 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#24 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#25 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e85c2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d66a6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41dba0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d57e)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d13e7)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1a2e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b42b7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Mutex M165642749574833856 is already destroyed.
Thread T148 'apply [worker]-' (tid=2051, running) created by thread T68 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c7095)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643:15 (libkudu_util.so+0x4423c8)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libmaster.so+0x2d4b89)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45ea4b)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45ccef)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45efb2)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x414e50)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x4153e9)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x41767a)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417608)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x417588)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417543)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41675c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3ea336)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4b88)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa2c0)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x1819a3)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b294)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138644)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1385f3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138568)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138523)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13780c)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x1614a6)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x14946b)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x133acd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x1374a5)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137439)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1373c9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137391)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13668d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser: WARNING: ThreadSanitizer: data race (pid=929) Write of size 8 at 0x7b44001f3b80 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x343359)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34f289)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34f229)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29aaa2)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29aa39)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a622d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a5bd0)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41c8f4)
#11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cf6d5)
#12 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cf8e9)
#13 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1002d7)
#14 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x100299)
#15 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100259)
#16 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0x1001ba)
#17 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc669)
#18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc640)
#19 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc600)
#20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b1096)
#21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b0fb4)
#22 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b0d6b)
#23 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x289451)
#24 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b43be)
#25 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#26 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#27 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#28 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#29 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#30 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#31 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#32 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#33 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#34 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#35 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#36 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#37 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#38 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#39 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#40 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#41 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#42 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#43 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#44 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#45 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#46 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#47 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Previous read of size 8 at 0x7b44001f3b80 by thread T148 (mutexes: write M165642749574833856):
#0 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::begin() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:1520:30 (libkudu_util.so+0x39fbc9)
#1 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:284:22 (libkudu_util.so+0x39d2da)
#2 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d2c0)
#3 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416322)
#4 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415ab3)
#5 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419cc1)
#6 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419c79)
#7 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419c09)
#8 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419bd1)
#9 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x418dfd)
#10 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#11 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#12 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#13 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#14 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#15 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#16 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#17 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#18 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#19 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#20 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e85c2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d66a6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41dba0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d57e)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d13e7)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1a2e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b42b7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Mutex M165642749574833856 is already destroyed.
Thread T148 'apply [worker]-' (tid=2051, running) created by thread T68 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c7095)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643:15 (libkudu_util.so+0x4423c8)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libmaster.so+0x2d4b89)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45ea4b)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45ccef)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45efb2)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x414e50)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x4153e9)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x41767a)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417608)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x417588)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417543)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41675c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3ea336)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4b88)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa2c0)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x1819a3)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b294)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138644)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1385f3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138568)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138523)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13780c)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x1614a6)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x14946b)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x133acd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x1374a5)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137439)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1373c9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137391)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13668d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
[==========] Running 10 tests from 3 test suites.
[----------] Global test environment set-up.
[----------] 3 tests from TableInfoTest
[ RUN ] TableInfoTest.TestAssignmentRanges
WARNING: Logging before InitGoogleLogging() is written to STDERR
I20250901 08:00:48.730932 929 catalog_manager-test.cc:126] Key found in tablet tablet--a
I20250901 08:00:48.731566 929 catalog_manager-test.cc:126] Key a found in tablet tablet-a-b
I20250901 08:00:48.731868 929 catalog_manager-test.cc:126] Key b found in tablet tablet-b-c
I20250901 08:00:48.732132 929 catalog_manager-test.cc:126] Key c found in tablet tablet-c-
[ OK ] TableInfoTest.TestAssignmentRanges (8 ms)
[ RUN ] TableInfoTest.GetTableLocationsLegacyCustomHashSchemas
[ OK ] TableInfoTest.GetTableLocationsLegacyCustomHashSchemas (2 ms)
[ RUN ] TableInfoTest.MaxReturnedLocationsNotSpecified
[ OK ] TableInfoTest.MaxReturnedLocationsNotSpecified (44 ms)
[----------] 3 tests from TableInfoTest (55 ms total)
[----------] 1 test from TestTSDescriptor
[ RUN ] TestTSDescriptor.TestReplicaCreationsDecay
[ OK ] TestTSDescriptor.TestReplicaCreationsDecay (10 ms)
[----------] 1 test from TestTSDescriptor (10 ms total)
[----------] 6 tests from CatalogManagerRpcAndUserFunctionsTest
[ RUN ] CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable
I20250901 08:00:48.797411 929 internal_mini_cluster.cc:156] Creating distributed mini masters. Addrs: 127.0.232.126:35739
I20250901 08:00:48.798908 929 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20250901 08:00:48.800005 929 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250901 08:00:48.816205 936 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:48.817090 935 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250901 08:00:48.819046 929 server_base.cc:1047] running on GCE node
W20250901 08:00:48.817896 938 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250901 08:00:49.941109 929 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250901 08:00:49.941344 929 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250901 08:00:49.941488 929 hybrid_clock.cc:648] HybridClock initialized: now 1756713649941472 us; error 0 us; skew 500 ppm
I20250901 08:00:49.942252 929 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250901 08:00:49.947996 929 webserver.cc:480] Webserver started at http://127.0.232.126:43459/ using document root <none> and password file <none>
I20250901 08:00:49.948876 929 fs_manager.cc:362] Metadata directory not provided
I20250901 08:00:49.949047 929 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250901 08:00:49.949433 929 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
I20250901 08:00:49.955466 929 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1756713648558816-929-0/minicluster-data/master-0-root/instance:
uuid: "d2f9b605e6bf4316b9b08db00a97c234"
format_stamp: "Formatted at 2025-09-01 08:00:49 on dist-test-slave-v0s2"
I20250901 08:00:49.963732 929 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
I20250901 08:00:49.969480 944 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:49.970676 929 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
I20250901 08:00:49.971017 929 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1756713648558816-929-0/minicluster-data/master-0-root
uuid: "d2f9b605e6bf4316b9b08db00a97c234"
format_stamp: "Formatted at 2025-09-01 08:00:49 on dist-test-slave-v0s2"
I20250901 08:00:49.971379 929 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1756713648558816-929-0/minicluster-data/master-0-root
metadata directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1756713648558816-929-0/minicluster-data/master-0-root
1 data directories: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1756713648558816-929-0/minicluster-data/master-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250901 08:00:50.018977 929 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250901 08:00:50.020586 929 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20250901 08:00:50.021162 929 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250901 08:00:50.099974 929 rpc_server.cc:307] RPC server started. Bound to: 127.0.232.126:35739
I20250901 08:00:50.100062 995 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.232.126:35739 every 8 connection(s)
I20250901 08:00:50.108242 996 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:50.132434 996 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234: Bootstrap starting.
I20250901 08:00:50.139303 996 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:50.141294 996 log.cc:826] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234: Log is configured to *not* fsync() on all Append() calls
I20250901 08:00:50.146530 996 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234: No bootstrap required, opened a new log
I20250901 08:00:50.167467 996 raft_consensus.cc:357] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2f9b605e6bf4316b9b08db00a97c234" member_type: VOTER }
I20250901 08:00:50.168068 996 raft_consensus.cc:383] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:50.168267 996 raft_consensus.cc:738] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d2f9b605e6bf4316b9b08db00a97c234, State: Initialized, Role: FOLLOWER
I20250901 08:00:50.169113 996 consensus_queue.cc:260] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2f9b605e6bf4316b9b08db00a97c234" member_type: VOTER }
I20250901 08:00:50.169605 996 raft_consensus.cc:397] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:50.169865 996 raft_consensus.cc:491] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:50.170143 996 raft_consensus.cc:3058] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:50.174528 996 raft_consensus.cc:513] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2f9b605e6bf4316b9b08db00a97c234" member_type: VOTER }
I20250901 08:00:50.175295 996 leader_election.cc:304] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: d2f9b605e6bf4316b9b08db00a97c234; no voters:
I20250901 08:00:50.177062 996 leader_election.cc:290] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:50.177460 999 raft_consensus.cc:2802] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:50.179812 999 raft_consensus.cc:695] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [term 1 LEADER]: Becoming Leader. State: Replica: d2f9b605e6bf4316b9b08db00a97c234, State: Running, Role: LEADER
I20250901 08:00:50.180801 999 consensus_queue.cc:237] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2f9b605e6bf4316b9b08db00a97c234" member_type: VOTER }
I20250901 08:00:50.182184 996 sys_catalog.cc:564] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [sys.catalog]: configured and running, proceeding with master startup.
I20250901 08:00:50.188766 1001 sys_catalog.cc:455] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d2f9b605e6bf4316b9b08db00a97c234. Latest consensus state: current_term: 1 leader_uuid: "d2f9b605e6bf4316b9b08db00a97c234" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2f9b605e6bf4316b9b08db00a97c234" member_type: VOTER } }
I20250901 08:00:50.189431 1001 sys_catalog.cc:458] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [sys.catalog]: This master's current role is: LEADER
I20250901 08:00:50.191457 1000 sys_catalog.cc:455] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "d2f9b605e6bf4316b9b08db00a97c234" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d2f9b605e6bf4316b9b08db00a97c234" member_type: VOTER } }
I20250901 08:00:50.192155 1000 sys_catalog.cc:458] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [sys.catalog]: This master's current role is: LEADER
I20250901 08:00:50.194515 1007 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250901 08:00:50.208176 1007 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250901 08:00:50.211104 929 internal_mini_cluster.cc:184] Waiting to initialize catalog manager on master 0
I20250901 08:00:50.226976 1007 catalog_manager.cc:1349] Generated new cluster ID: 280c7f40398245f78fccb31059c1db22
I20250901 08:00:50.227258 1007 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250901 08:00:50.266060 1007 catalog_manager.cc:1372] Generated new certificate authority record
I20250901 08:00:50.268491 1007 catalog_manager.cc:1506] Loading token signing keys...
I20250901 08:00:50.284982 1007 catalog_manager.cc:5955] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234: Generated new TSK 0
I20250901 08:00:50.286355 1007 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250901 08:00:50.342227 929 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250901 08:00:50.351200 1018 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:50.352221 1019 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250901 08:00:50.358434 929 server_base.cc:1047] running on GCE node
W20250901 08:00:50.358999 1021 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250901 08:00:50.359949 929 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250901 08:00:50.360141 929 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250901 08:00:50.360301 929 hybrid_clock.cc:648] HybridClock initialized: now 1756713650360273 us; error 0 us; skew 500 ppm
I20250901 08:00:50.360801 929 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250901 08:00:50.365263 929 webserver.cc:480] Webserver started at http://127.0.232.65:37023/ using document root <none> and password file <none>
I20250901 08:00:50.365752 929 fs_manager.cc:362] Metadata directory not provided
I20250901 08:00:50.365929 929 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250901 08:00:50.366174 929 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
I20250901 08:00:50.367304 929 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1756713648558816-929-0/minicluster-data/ts-0-root/instance:
uuid: "58257f0e12384d9c9d7a63afc87126d9"
format_stamp: "Formatted at 2025-09-01 08:00:50 on dist-test-slave-v0s2"
I20250901 08:00:50.373212 929 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.006s sys 0.001s
I20250901 08:00:50.377842 1026 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:50.378630 929 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20250901 08:00:50.378885 929 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1756713648558816-929-0/minicluster-data/ts-0-root
uuid: "58257f0e12384d9c9d7a63afc87126d9"
format_stamp: "Formatted at 2025-09-01 08:00:50 on dist-test-slave-v0s2"
I20250901 08:00:50.379150 929 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1756713648558816-929-0/minicluster-data/ts-0-root
metadata directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1756713648558816-929-0/minicluster-data/ts-0-root
1 data directories: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1756713648558816-929-0/minicluster-data/ts-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250901 08:00:50.396903 929 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250901 08:00:50.398444 929 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250901 08:00:50.401330 929 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20250901 08:00:50.406430 929 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20250901 08:00:50.406648 929 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:50.406931 929 ts_tablet_manager.cc:610] Registered 0 tablets
I20250901 08:00:50.407078 929 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:50.512082 929 rpc_server.cc:307] RPC server started. Bound to: 127.0.232.65:41283
I20250901 08:00:50.512151 1088 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.232.65:41283 every 8 connection(s)
I20250901 08:00:50.544338 1089 heartbeater.cc:344] Connected to a master server at 127.0.232.126:35739
I20250901 08:00:50.544970 1089 heartbeater.cc:461] Registering TS with master...
I20250901 08:00:50.546226 1089 heartbeater.cc:507] Master 127.0.232.126:35739 requested a full tablet report, sending...
I20250901 08:00:50.549722 961 ts_manager.cc:194] Registered new tserver with Master: 58257f0e12384d9c9d7a63afc87126d9 (127.0.232.65:41283)
I20250901 08:00:50.549964 929 internal_mini_cluster.cc:371] 1 TS(s) registered with all masters after 0.031903311s
I20250901 08:00:50.552946 961 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.0.1:60448
I20250901 08:00:50.583997 961 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:60458:
name: "test_table"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
rows: "\004\001d\000\000\000\004\001\310\000\000\000\004\001,\001\000\000\004\001\220\001\000\000\004\001\364\001\000\000\004\001X\002\000\000\004\001\274\002\000\000\004\001 \003\000\000\004\001\204\003\000\000""\004\001d\000\000\000\004\001\310\000\000\000\004\001,\001\000\000\004\001\220\001\000\000\004\001\364\001\000\000\004\001X\002\000\000\004\001\274\002\000\000\004\001 \003\000\000\004\001\204\003\000\000"
indirect_data: """"
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
I20250901 08:00:50.664083 1054 tablet_service.cc:1468] Processing CreateTablet for tablet 4efdef006e94469abf17449e6242c0cc (DEFAULT_TABLE table=test_table [id=0defc9c68b5e4d93a5beccfb8cb9e866]), partition=RANGE (key) PARTITION VALUES < 100
I20250901 08:00:50.665223 1053 tablet_service.cc:1468] Processing CreateTablet for tablet ee102e89171f4aec960decb0f6e5ab9a (DEFAULT_TABLE table=test_table [id=0defc9c68b5e4d93a5beccfb8cb9e866]), partition=RANGE (key) PARTITION 100 <= VALUES < 200
I20250901 08:00:50.666003 1054 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4efdef006e94469abf17449e6242c0cc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:50.666069 1052 tablet_service.cc:1468] Processing CreateTablet for tablet 7e0c8b25ce9c406481826e9dbf6eba99 (DEFAULT_TABLE table=test_table [id=0defc9c68b5e4d93a5beccfb8cb9e866]), partition=RANGE (key) PARTITION 200 <= VALUES < 300
I20250901 08:00:50.666852 1053 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ee102e89171f4aec960decb0f6e5ab9a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:50.667452 1052 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7e0c8b25ce9c406481826e9dbf6eba99. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:50.668407 1048 tablet_service.cc:1468] Processing CreateTablet for tablet abcdf97de9d648b59d03b1cefd57c1e6 (DEFAULT_TABLE table=test_table [id=0defc9c68b5e4d93a5beccfb8cb9e866]), partition=RANGE (key) PARTITION 600 <= VALUES < 700
I20250901 08:00:50.669660 1048 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet abcdf97de9d648b59d03b1cefd57c1e6. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:50.671514 1049 tablet_service.cc:1468] Processing CreateTablet for tablet 7a14041f6c204a4d89383bacdbe28476 (DEFAULT_TABLE table=test_table [id=0defc9c68b5e4d93a5beccfb8cb9e866]), partition=RANGE (key) PARTITION 500 <= VALUES < 600
I20250901 08:00:50.672899 1049 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7a14041f6c204a4d89383bacdbe28476. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:50.675366 1050 tablet_service.cc:1468] Processing CreateTablet for tablet db4ff5c5241c4040b986f8e9fff52c0b (DEFAULT_TABLE table=test_table [id=0defc9c68b5e4d93a5beccfb8cb9e866]), partition=RANGE (key) PARTITION 400 <= VALUES < 500
I20250901 08:00:50.676579 1050 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet db4ff5c5241c4040b986f8e9fff52c0b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:50.679351 1051 tablet_service.cc:1468] Processing CreateTablet for tablet a30984dccb064270b7e27276fd10194f (DEFAULT_TABLE table=test_table [id=0defc9c68b5e4d93a5beccfb8cb9e866]), partition=RANGE (key) PARTITION 300 <= VALUES < 400
I20250901 08:00:50.680537 1051 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a30984dccb064270b7e27276fd10194f. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:50.693055 1047 tablet_service.cc:1468] Processing CreateTablet for tablet d406316990504241bc0c23d889ca7ed6 (DEFAULT_TABLE table=test_table [id=0defc9c68b5e4d93a5beccfb8cb9e866]), partition=RANGE (key) PARTITION 700 <= VALUES < 800
I20250901 08:00:50.696506 1045 tablet_service.cc:1468] Processing CreateTablet for tablet a85c64db55f74e4d9fc799c4ce5f075e (DEFAULT_TABLE table=test_table [id=0defc9c68b5e4d93a5beccfb8cb9e866]), partition=RANGE (key) PARTITION 900 <= VALUES
I20250901 08:00:50.696530 1046 tablet_service.cc:1468] Processing CreateTablet for tablet d2565aeb2e3b4b03a1022c289f9110bf (DEFAULT_TABLE table=test_table [id=0defc9c68b5e4d93a5beccfb8cb9e866]), partition=RANGE (key) PARTITION 800 <= VALUES < 900
I20250901 08:00:50.699560 1047 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d406316990504241bc0c23d889ca7ed6. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:50.701628 1046 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d2565aeb2e3b4b03a1022c289f9110bf. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:50.703012 1045 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a85c64db55f74e4d9fc799c4ce5f075e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:50.722015 1101 tablet_bootstrap.cc:492] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9: Bootstrap starting.
I20250901 08:00:50.742516 1101 tablet_bootstrap.cc:654] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:50.747185 1101 tablet_bootstrap.cc:492] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9: No bootstrap required, opened a new log
I20250901 08:00:50.747542 1101 ts_tablet_manager.cc:1397] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9: Time spent bootstrapping tablet: real 0.026s user 0.001s sys 0.010s
I20250901 08:00:50.749490 1101 raft_consensus.cc:357] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.749939 1101 raft_consensus.cc:383] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:50.750159 1101 raft_consensus.cc:738] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Initialized, Role: FOLLOWER
I20250901 08:00:50.750716 1101 consensus_queue.cc:260] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.751199 1101 raft_consensus.cc:397] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:50.751396 1101 raft_consensus.cc:491] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:50.751647 1101 raft_consensus.cc:3058] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:50.755991 1101 raft_consensus.cc:513] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.756508 1101 leader_election.cc:304] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 58257f0e12384d9c9d7a63afc87126d9; no voters:
I20250901 08:00:50.758853 1103 raft_consensus.cc:2802] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:50.759101 1101 leader_election.cc:290] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:50.762841 1089 heartbeater.cc:499] Master 127.0.232.126:35739 was elected leader, sending a full tablet report...
I20250901 08:00:50.763388 1101 ts_tablet_manager.cc:1428] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9: Time spent starting tablet: real 0.016s user 0.007s sys 0.008s
I20250901 08:00:50.764364 1101 tablet_bootstrap.cc:492] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9: Bootstrap starting.
I20250901 08:00:50.766335 1103 raft_consensus.cc:695] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Becoming Leader. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Running, Role: LEADER
I20250901 08:00:50.767083 1103 consensus_queue.cc:237] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
W20250901 08:00:50.768944 1090 tablet.cc:2378] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20250901 08:00:50.770226 1101 tablet_bootstrap.cc:654] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:50.779675 1101 tablet_bootstrap.cc:492] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9: No bootstrap required, opened a new log
I20250901 08:00:50.780033 1101 ts_tablet_manager.cc:1397] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9: Time spent bootstrapping tablet: real 0.016s user 0.002s sys 0.009s
I20250901 08:00:50.782183 1101 raft_consensus.cc:357] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.782724 1101 raft_consensus.cc:383] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:50.782991 1101 raft_consensus.cc:738] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Initialized, Role: FOLLOWER
I20250901 08:00:50.783633 1101 consensus_queue.cc:260] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.784349 1101 raft_consensus.cc:397] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:50.784642 1101 raft_consensus.cc:491] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:50.784978 1101 raft_consensus.cc:3058] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:50.785681 961 catalog_manager.cc:5582] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 58257f0e12384d9c9d7a63afc87126d9 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "58257f0e12384d9c9d7a63afc87126d9" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:50.789956 1101 raft_consensus.cc:513] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.790674 1101 leader_election.cc:304] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 58257f0e12384d9c9d7a63afc87126d9; no voters:
I20250901 08:00:50.791293 1101 leader_election.cc:290] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:50.792222 1103 raft_consensus.cc:2802] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:50.793112 1103 raft_consensus.cc:695] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Becoming Leader. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Running, Role: LEADER
I20250901 08:00:50.793781 1103 consensus_queue.cc:237] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.794121 1101 ts_tablet_manager.cc:1428] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9: Time spent starting tablet: real 0.014s user 0.012s sys 0.002s
I20250901 08:00:50.795141 1101 tablet_bootstrap.cc:492] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9: Bootstrap starting.
I20250901 08:00:50.801838 1101 tablet_bootstrap.cc:654] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:50.807755 1101 tablet_bootstrap.cc:492] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9: No bootstrap required, opened a new log
I20250901 08:00:50.808087 1101 ts_tablet_manager.cc:1397] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9: Time spent bootstrapping tablet: real 0.013s user 0.010s sys 0.000s
I20250901 08:00:50.809931 1101 raft_consensus.cc:357] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.810388 1101 raft_consensus.cc:383] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:50.809983 961 catalog_manager.cc:5582] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 58257f0e12384d9c9d7a63afc87126d9 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "58257f0e12384d9c9d7a63afc87126d9" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:50.810616 1101 raft_consensus.cc:738] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Initialized, Role: FOLLOWER
I20250901 08:00:50.811167 1101 consensus_queue.cc:260] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.811622 1101 raft_consensus.cc:397] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:50.811864 1101 raft_consensus.cc:491] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:50.812108 1101 raft_consensus.cc:3058] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:50.818301 1101 raft_consensus.cc:513] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.818837 1101 leader_election.cc:304] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 58257f0e12384d9c9d7a63afc87126d9; no voters:
I20250901 08:00:50.819303 1101 leader_election.cc:290] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:50.819613 1103 raft_consensus.cc:2802] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:50.820047 1103 raft_consensus.cc:695] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Becoming Leader. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Running, Role: LEADER
I20250901 08:00:50.821017 1101 ts_tablet_manager.cc:1428] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9: Time spent starting tablet: real 0.013s user 0.007s sys 0.004s
I20250901 08:00:50.820701 1103 consensus_queue.cc:237] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.821668 1101 tablet_bootstrap.cc:492] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9: Bootstrap starting.
I20250901 08:00:50.828091 1101 tablet_bootstrap.cc:654] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:50.828550 961 catalog_manager.cc:5582] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 58257f0e12384d9c9d7a63afc87126d9 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "58257f0e12384d9c9d7a63afc87126d9" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:50.840823 1101 tablet_bootstrap.cc:492] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9: No bootstrap required, opened a new log
I20250901 08:00:50.841207 1101 ts_tablet_manager.cc:1397] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9: Time spent bootstrapping tablet: real 0.020s user 0.010s sys 0.007s
I20250901 08:00:50.843691 1101 raft_consensus.cc:357] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.844189 1101 raft_consensus.cc:383] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:50.844471 1101 raft_consensus.cc:738] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Initialized, Role: FOLLOWER
I20250901 08:00:50.845081 1101 consensus_queue.cc:260] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.845701 1101 raft_consensus.cc:397] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:50.845991 1101 raft_consensus.cc:491] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:50.846329 1101 raft_consensus.cc:3058] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:50.852447 1101 raft_consensus.cc:513] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.853142 1101 leader_election.cc:304] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 58257f0e12384d9c9d7a63afc87126d9; no voters:
I20250901 08:00:50.853730 1101 leader_election.cc:290] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:50.853875 1103 raft_consensus.cc:2802] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:50.854377 1103 raft_consensus.cc:695] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Becoming Leader. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Running, Role: LEADER
I20250901 08:00:50.855053 1103 consensus_queue.cc:237] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.855881 1101 ts_tablet_manager.cc:1428] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9: Time spent starting tablet: real 0.014s user 0.013s sys 0.001s
I20250901 08:00:50.856659 1101 tablet_bootstrap.cc:492] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9: Bootstrap starting.
I20250901 08:00:50.861888 961 catalog_manager.cc:5582] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 58257f0e12384d9c9d7a63afc87126d9 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "58257f0e12384d9c9d7a63afc87126d9" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:50.862665 1101 tablet_bootstrap.cc:654] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:50.868289 1101 tablet_bootstrap.cc:492] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9: No bootstrap required, opened a new log
I20250901 08:00:50.868641 1101 ts_tablet_manager.cc:1397] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9: Time spent bootstrapping tablet: real 0.012s user 0.010s sys 0.000s
I20250901 08:00:50.870913 1101 raft_consensus.cc:357] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.871277 1101 raft_consensus.cc:383] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:50.871513 1101 raft_consensus.cc:738] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Initialized, Role: FOLLOWER
I20250901 08:00:50.871965 1101 consensus_queue.cc:260] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.872349 1101 raft_consensus.cc:397] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:50.872561 1101 raft_consensus.cc:491] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:50.872788 1101 raft_consensus.cc:3058] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:50.876674 1101 raft_consensus.cc:513] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.877105 1101 leader_election.cc:304] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 58257f0e12384d9c9d7a63afc87126d9; no voters:
I20250901 08:00:50.877534 1101 leader_election.cc:290] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:50.877650 1103 raft_consensus.cc:2802] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:50.878055 1103 raft_consensus.cc:695] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Becoming Leader. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Running, Role: LEADER
I20250901 08:00:50.878793 1103 consensus_queue.cc:237] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.879396 1101 ts_tablet_manager.cc:1428] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9: Time spent starting tablet: real 0.010s user 0.011s sys 0.000s
I20250901 08:00:50.880205 1101 tablet_bootstrap.cc:492] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9: Bootstrap starting.
I20250901 08:00:50.885421 961 catalog_manager.cc:5582] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 58257f0e12384d9c9d7a63afc87126d9 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "58257f0e12384d9c9d7a63afc87126d9" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:50.886473 1101 tablet_bootstrap.cc:654] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:50.891386 1101 tablet_bootstrap.cc:492] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9: No bootstrap required, opened a new log
I20250901 08:00:50.891659 1101 ts_tablet_manager.cc:1397] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9: Time spent bootstrapping tablet: real 0.012s user 0.006s sys 0.004s
I20250901 08:00:50.893301 1101 raft_consensus.cc:357] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.893635 1101 raft_consensus.cc:383] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:50.893801 1101 raft_consensus.cc:738] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Initialized, Role: FOLLOWER
I20250901 08:00:50.894265 1101 consensus_queue.cc:260] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.894759 1101 raft_consensus.cc:397] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:50.895007 1101 raft_consensus.cc:491] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:50.895254 1101 raft_consensus.cc:3058] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:50.900043 1101 raft_consensus.cc:513] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.900530 1101 leader_election.cc:304] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 58257f0e12384d9c9d7a63afc87126d9; no voters:
I20250901 08:00:50.900933 1101 leader_election.cc:290] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:50.901047 1103 raft_consensus.cc:2802] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:50.901978 1103 raft_consensus.cc:695] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Becoming Leader. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Running, Role: LEADER
I20250901 08:00:50.902408 1101 ts_tablet_manager.cc:1428] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9: Time spent starting tablet: real 0.011s user 0.012s sys 0.000s
I20250901 08:00:50.902736 1103 consensus_queue.cc:237] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.903107 1101 tablet_bootstrap.cc:492] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9: Bootstrap starting.
I20250901 08:00:50.908808 1101 tablet_bootstrap.cc:654] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:50.910121 961 catalog_manager.cc:5582] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 58257f0e12384d9c9d7a63afc87126d9 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "58257f0e12384d9c9d7a63afc87126d9" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:50.914062 1101 tablet_bootstrap.cc:492] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9: No bootstrap required, opened a new log
I20250901 08:00:50.914446 1101 ts_tablet_manager.cc:1397] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9: Time spent bootstrapping tablet: real 0.011s user 0.006s sys 0.004s
I20250901 08:00:50.916441 1101 raft_consensus.cc:357] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.916923 1101 raft_consensus.cc:383] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:50.917150 1101 raft_consensus.cc:738] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Initialized, Role: FOLLOWER
I20250901 08:00:50.917636 1101 consensus_queue.cc:260] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.918126 1101 raft_consensus.cc:397] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:50.918403 1101 raft_consensus.cc:491] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:50.918704 1101 raft_consensus.cc:3058] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:50.922363 1101 raft_consensus.cc:513] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.922827 1101 leader_election.cc:304] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 58257f0e12384d9c9d7a63afc87126d9; no voters:
I20250901 08:00:50.923215 1101 leader_election.cc:290] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:50.923336 1103 raft_consensus.cc:2802] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:50.923730 1103 raft_consensus.cc:695] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Becoming Leader. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Running, Role: LEADER
I20250901 08:00:50.924364 1103 consensus_queue.cc:237] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.924773 1101 ts_tablet_manager.cc:1428] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9: Time spent starting tablet: real 0.010s user 0.008s sys 0.000s
I20250901 08:00:50.925963 1101 tablet_bootstrap.cc:492] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9: Bootstrap starting.
I20250901 08:00:50.931005 1101 tablet_bootstrap.cc:654] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:50.931298 961 catalog_manager.cc:5582] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 58257f0e12384d9c9d7a63afc87126d9 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "58257f0e12384d9c9d7a63afc87126d9" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:50.936110 1101 tablet_bootstrap.cc:492] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9: No bootstrap required, opened a new log
I20250901 08:00:50.936430 1101 ts_tablet_manager.cc:1397] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9: Time spent bootstrapping tablet: real 0.011s user 0.006s sys 0.003s
I20250901 08:00:50.938289 1101 raft_consensus.cc:357] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.938633 1101 raft_consensus.cc:383] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:50.938810 1101 raft_consensus.cc:738] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Initialized, Role: FOLLOWER
I20250901 08:00:50.939256 1101 consensus_queue.cc:260] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.939611 1101 raft_consensus.cc:397] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:50.939790 1101 raft_consensus.cc:491] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:50.940013 1101 raft_consensus.cc:3058] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:50.945042 1101 raft_consensus.cc:513] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.945669 1101 leader_election.cc:304] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 58257f0e12384d9c9d7a63afc87126d9; no voters:
I20250901 08:00:50.946199 1101 leader_election.cc:290] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:50.946445 1103 raft_consensus.cc:2802] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:50.947537 1103 raft_consensus.cc:695] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Becoming Leader. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Running, Role: LEADER
I20250901 08:00:50.948431 1101 ts_tablet_manager.cc:1428] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9: Time spent starting tablet: real 0.012s user 0.012s sys 0.000s
I20250901 08:00:50.948545 1103 consensus_queue.cc:237] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.949204 1101 tablet_bootstrap.cc:492] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9: Bootstrap starting.
I20250901 08:00:50.955044 1101 tablet_bootstrap.cc:654] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:50.955500 961 catalog_manager.cc:5582] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 58257f0e12384d9c9d7a63afc87126d9 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "58257f0e12384d9c9d7a63afc87126d9" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:50.960311 1101 tablet_bootstrap.cc:492] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9: No bootstrap required, opened a new log
I20250901 08:00:50.960568 1101 ts_tablet_manager.cc:1397] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9: Time spent bootstrapping tablet: real 0.012s user 0.007s sys 0.001s
I20250901 08:00:50.961869 1101 raft_consensus.cc:357] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.962155 1101 raft_consensus.cc:383] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:50.962352 1101 raft_consensus.cc:738] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Initialized, Role: FOLLOWER
I20250901 08:00:50.962813 1101 consensus_queue.cc:260] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.963202 1101 raft_consensus.cc:397] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:50.963397 1101 raft_consensus.cc:491] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:50.963649 1101 raft_consensus.cc:3058] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:50.966885 1101 raft_consensus.cc:513] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.967276 1101 leader_election.cc:304] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 58257f0e12384d9c9d7a63afc87126d9; no voters:
I20250901 08:00:50.967679 1101 leader_election.cc:290] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:50.967778 1103 raft_consensus.cc:2802] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:50.968223 1103 raft_consensus.cc:695] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Becoming Leader. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Running, Role: LEADER
I20250901 08:00:50.968796 1103 consensus_queue.cc:237] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.969192 1101 ts_tablet_manager.cc:1428] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9: Time spent starting tablet: real 0.008s user 0.008s sys 0.000s
I20250901 08:00:50.969802 1101 tablet_bootstrap.cc:492] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9: Bootstrap starting.
I20250901 08:00:50.976177 1101 tablet_bootstrap.cc:654] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:50.982901 961 catalog_manager.cc:5582] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 58257f0e12384d9c9d7a63afc87126d9 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "58257f0e12384d9c9d7a63afc87126d9" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:50.983889 1101 tablet_bootstrap.cc:492] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9: No bootstrap required, opened a new log
I20250901 08:00:50.984187 1101 ts_tablet_manager.cc:1397] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9: Time spent bootstrapping tablet: real 0.015s user 0.010s sys 0.003s
I20250901 08:00:50.985659 1101 raft_consensus.cc:357] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.985958 1101 raft_consensus.cc:383] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:50.986126 1101 raft_consensus.cc:738] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Initialized, Role: FOLLOWER
I20250901 08:00:50.986557 1101 consensus_queue.cc:260] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.986958 1101 raft_consensus.cc:397] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:50.987140 1101 raft_consensus.cc:491] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:50.987344 1101 raft_consensus.cc:3058] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:50.991070 1101 raft_consensus.cc:513] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:50.991454 1101 leader_election.cc:304] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 58257f0e12384d9c9d7a63afc87126d9; no voters:
I20250901 08:00:50.991834 1101 leader_election.cc:290] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:50.991945 1103 raft_consensus.cc:2802] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:50.992507 1103 raft_consensus.cc:695] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Becoming Leader. State: Replica: 58257f0e12384d9c9d7a63afc87126d9, State: Running, Role: LEADER
I20250901 08:00:50.993307 1101 ts_tablet_manager.cc:1428] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9: Time spent starting tablet: real 0.009s user 0.003s sys 0.004s
I20250901 08:00:50.993098 1103 consensus_queue.cc:237] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } }
I20250901 08:00:51.000046 961 catalog_manager.cc:5582] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 58257f0e12384d9c9d7a63afc87126d9 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "58257f0e12384d9c9d7a63afc87126d9" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "58257f0e12384d9c9d7a63afc87126d9" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 41283 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:51.031419 929 catalog_manager.cc:2730] Servicing DeleteTable request from internal request:
table { table_name: "test_table" }
I20250901 08:00:51.044088 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234: Sending DeleteTablet for 1 replicas of tablet 4efdef006e94469abf17449e6242c0cc
I20250901 08:00:51.044756 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234: Sending DeleteTablet for 1 replicas of tablet ee102e89171f4aec960decb0f6e5ab9a
I20250901 08:00:51.045719 1054 tablet_service.cc:1515] Processing DeleteTablet for tablet 4efdef006e94469abf17449e6242c0cc with delete_type TABLET_DATA_DELETED (Table deleted at 2025-09-01 08:00:51 UTC) from {username='slave'} at 127.0.0.1:43416
I20250901 08:00:51.045300 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234: Sending DeleteTablet for 1 replicas of tablet 7e0c8b25ce9c406481826e9dbf6eba99
I20250901 08:00:51.046057 1045 tablet_service.cc:1515] Processing DeleteTablet for tablet ee102e89171f4aec960decb0f6e5ab9a with delete_type TABLET_DATA_DELETED (Table deleted at 2025-09-01 08:00:51 UTC) from {username='slave'} at 127.0.0.1:43416
I20250901 08:00:51.046876 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234: Sending DeleteTablet for 1 replicas of tablet a30984dccb064270b7e27276fd10194f
I20250901 08:00:51.047690 1045 tablet_service.cc:1515] Processing DeleteTablet for tablet 7e0c8b25ce9c406481826e9dbf6eba99 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-09-01 08:00:51 UTC) from {username='slave'} at 127.0.0.1:43416
I20250901 08:00:51.047775 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234: Sending DeleteTablet for 1 replicas of tablet db4ff5c5241c4040b986f8e9fff52c0b
I20250901 08:00:51.048710 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234: Sending DeleteTablet for 1 replicas of tablet 7a14041f6c204a4d89383bacdbe28476
I20250901 08:00:51.048957 1045 tablet_service.cc:1515] Processing DeleteTablet for tablet a30984dccb064270b7e27276fd10194f with delete_type TABLET_DATA_DELETED (Table deleted at 2025-09-01 08:00:51 UTC) from {username='slave'} at 127.0.0.1:43416
I20250901 08:00:51.049552 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234: Sending DeleteTablet for 1 replicas of tablet abcdf97de9d648b59d03b1cefd57c1e6
I20250901 08:00:51.050401 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234: Sending DeleteTablet for 1 replicas of tablet d406316990504241bc0c23d889ca7ed6
I20250901 08:00:51.050745 1047 tablet_service.cc:1515] Processing DeleteTablet for tablet 7a14041f6c204a4d89383bacdbe28476 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-09-01 08:00:51 UTC) from {username='slave'} at 127.0.0.1:43416
I20250901 08:00:51.051172 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234: Sending DeleteTablet for 1 replicas of tablet d2565aeb2e3b4b03a1022c289f9110bf
I20250901 08:00:51.051445 1045 tablet_service.cc:1515] Processing DeleteTablet for tablet db4ff5c5241c4040b986f8e9fff52c0b with delete_type TABLET_DATA_DELETED (Table deleted at 2025-09-01 08:00:51 UTC) from {username='slave'} at 127.0.0.1:43416
I20250901 08:00:51.052007 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234: Sending DeleteTablet for 1 replicas of tablet a85c64db55f74e4d9fc799c4ce5f075e
I20250901 08:00:51.052848 1045 tablet_service.cc:1515] Processing DeleteTablet for tablet abcdf97de9d648b59d03b1cefd57c1e6 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-09-01 08:00:51 UTC) from {username='slave'} at 127.0.0.1:43416
I20250901 08:00:51.054847 1054 tablet_service.cc:1515] Processing DeleteTablet for tablet d2565aeb2e3b4b03a1022c289f9110bf with delete_type TABLET_DATA_DELETED (Table deleted at 2025-09-01 08:00:51 UTC) from {username='slave'} at 127.0.0.1:43416
I20250901 08:00:51.055227 1126 tablet_replica.cc:331] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9: stopping tablet replica
I20250901 08:00:51.055416 929 tablet_server.cc:178] TabletServer@127.0.232.65:0 shutting down...
I20250901 08:00:51.056159 1126 raft_consensus.cc:2241] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:51.056769 1126 raft_consensus.cc:2270] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:51.058602 1045 tablet_service.cc:1515] Processing DeleteTablet for tablet d406316990504241bc0c23d889ca7ed6 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-09-01 08:00:51 UTC) from {username='slave'} at 127.0.0.1:43416
W20250901 08:00:51.062665 945 proxy.cc:239] Call had error, refreshing address and retrying: Remote error: Service unavailable: service kudu.tserver.TabletServerAdminService not registered on TabletServer
W20250901 08:00:51.065973 945 catalog_manager.cc:4671] TS 58257f0e12384d9c9d7a63afc87126d9 (127.0.232.65:41283): DeleteTablet:TABLET_DATA_DELETED RPC failed for tablet a85c64db55f74e4d9fc799c4ce5f075e: Remote error: Service unavailable: service kudu.tserver.TabletServerAdminService not registered on TabletServer
I20250901 08:00:51.067866 1126 ts_tablet_manager.cc:1905] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250901 08:00:51.080487 1126 ts_tablet_manager.cc:1918] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250901 08:00:51.080825 1126 log.cc:1199] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9: Deleting WAL directory at /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1756713648558816-929-0/minicluster-data/ts-0-root/wals/4efdef006e94469abf17449e6242c0cc
I20250901 08:00:51.081511 1126 ts_tablet_manager.cc:1939] T 4efdef006e94469abf17449e6242c0cc P 58257f0e12384d9c9d7a63afc87126d9: Deleting consensus metadata
I20250901 08:00:51.084146 1126 tablet_replica.cc:331] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9: stopping tablet replica
I20250901 08:00:51.084321 945 catalog_manager.cc:4928] TS 58257f0e12384d9c9d7a63afc87126d9 (127.0.232.65:41283): tablet 4efdef006e94469abf17449e6242c0cc (table test_table [id=0defc9c68b5e4d93a5beccfb8cb9e866]) successfully deleted
I20250901 08:00:51.085048 1126 raft_consensus.cc:2241] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:51.085456 1126 raft_consensus.cc:2270] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:51.089588 1126 ts_tablet_manager.cc:1905] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250901 08:00:51.099889 929 ts_tablet_manager.cc:1500] Shutting down tablet manager...
I20250901 08:00:51.100152 1126 ts_tablet_manager.cc:1918] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250901 08:00:51.100502 1126 log.cc:1199] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9: Deleting WAL directory at /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1756713648558816-929-0/minicluster-data/ts-0-root/wals/ee102e89171f4aec960decb0f6e5ab9a
I20250901 08:00:51.101092 1126 ts_tablet_manager.cc:1939] T ee102e89171f4aec960decb0f6e5ab9a P 58257f0e12384d9c9d7a63afc87126d9: Deleting consensus metadata
W20250901 08:00:51.102790 945 catalog_manager.cc:4920] TS 58257f0e12384d9c9d7a63afc87126d9 (127.0.232.65:41283): delete failed for tablet ee102e89171f4aec960decb0f6e5ab9a with error code TABLET_NOT_RUNNING: Service unavailable: Tablet Manager is not running: MANAGER_QUIESCING
I20250901 08:00:51.103618 929 tablet_replica.cc:331] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9: stopping tablet replica
I20250901 08:00:51.104168 929 raft_consensus.cc:2241] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:51.104529 929 raft_consensus.cc:2270] T a85c64db55f74e4d9fc799c4ce5f075e P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:51.107239 929 tablet_replica.cc:331] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9: stopping tablet replica
I20250901 08:00:51.107694 929 raft_consensus.cc:2241] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:51.108021 929 raft_consensus.cc:2270] T d406316990504241bc0c23d889ca7ed6 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:51.109817 929 tablet_replica.cc:331] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9: stopping tablet replica
I20250901 08:00:51.110222 929 raft_consensus.cc:2241] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:51.110663 929 raft_consensus.cc:2270] T 7a14041f6c204a4d89383bacdbe28476 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:51.113135 929 tablet_replica.cc:331] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9: stopping tablet replica
I20250901 08:00:51.113576 929 raft_consensus.cc:2241] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:51.114002 929 raft_consensus.cc:2270] T d2565aeb2e3b4b03a1022c289f9110bf P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:51.115851 929 tablet_replica.cc:331] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9: stopping tablet replica
I20250901 08:00:51.116250 929 raft_consensus.cc:2241] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:51.116551 929 raft_consensus.cc:2270] T db4ff5c5241c4040b986f8e9fff52c0b P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:51.118439 929 tablet_replica.cc:331] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9: stopping tablet replica
I20250901 08:00:51.118850 929 raft_consensus.cc:2241] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:51.119167 929 raft_consensus.cc:2270] T 7e0c8b25ce9c406481826e9dbf6eba99 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:51.120757 929 tablet_replica.cc:331] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9: stopping tablet replica
I20250901 08:00:51.121127 929 raft_consensus.cc:2241] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:51.121431 929 raft_consensus.cc:2270] T abcdf97de9d648b59d03b1cefd57c1e6 P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:51.123220 929 tablet_replica.cc:331] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9: stopping tablet replica
I20250901 08:00:51.123606 929 raft_consensus.cc:2241] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:51.123920 929 raft_consensus.cc:2270] T a30984dccb064270b7e27276fd10194f P 58257f0e12384d9c9d7a63afc87126d9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:51.276134 929 tablet_server.cc:195] TabletServer@127.0.232.65:0 shutdown complete.
I20250901 08:00:51.288323 929 master.cc:561] Master@127.0.232.126:35739 shutting down...
I20250901 08:00:51.700232 929 raft_consensus.cc:2241] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:51.700747 929 raft_consensus.cc:2270] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:51.701028 929 tablet_replica.cc:331] T 00000000000000000000000000000000 P d2f9b605e6bf4316b9b08db00a97c234: stopping tablet replica
I20250901 08:00:51.719270 929 master.cc:583] Master@127.0.232.126:35739 shutdown complete.
[ OK ] CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable (2948 ms)
[ RUN ] CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser
I20250901 08:00:51.741479 929 internal_mini_cluster.cc:156] Creating distributed mini masters. Addrs: 127.0.232.126:45715
I20250901 08:00:51.742482 929 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250901 08:00:51.747153 1128 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:51.748620 1129 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:51.750885 1131 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250901 08:00:51.751256 929 server_base.cc:1047] running on GCE node
I20250901 08:00:51.752023 929 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250901 08:00:51.752202 929 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250901 08:00:51.752341 929 hybrid_clock.cc:648] HybridClock initialized: now 1756713651752324 us; error 0 us; skew 500 ppm
I20250901 08:00:51.752846 929 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250901 08:00:51.755539 929 webserver.cc:480] Webserver started at http://127.0.232.126:35753/ using document root <none> and password file <none>
I20250901 08:00:51.755976 929 fs_manager.cc:362] Metadata directory not provided
I20250901 08:00:51.756142 929 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250901 08:00:51.756377 929 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
I20250901 08:00:51.757314 929 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1756713648558816-929-0/minicluster-data/master-0-root/instance:
uuid: "2734f355997d434791b649f97ba0c2f2"
format_stamp: "Formatted at 2025-09-01 08:00:51 on dist-test-slave-v0s2"
I20250901 08:00:51.761296 929 fs_manager.cc:696] Time spent creating directory manager: real 0.003s user 0.005s sys 0.000s
I20250901 08:00:51.764621 1136 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:51.765357 929 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20250901 08:00:51.765662 929 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1756713648558816-929-0/minicluster-data/master-0-root
uuid: "2734f355997d434791b649f97ba0c2f2"
format_stamp: "Formatted at 2025-09-01 08:00:51 on dist-test-slave-v0s2"
I20250901 08:00:51.765974 929 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1756713648558816-929-0/minicluster-data/master-0-root
metadata directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1756713648558816-929-0/minicluster-data/master-0-root
1 data directories: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1756713648558816-929-0/minicluster-data/master-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250901 08:00:51.796329 929 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250901 08:00:51.797446 929 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250901 08:00:51.846612 929 rpc_server.cc:307] RPC server started. Bound to: 127.0.232.126:45715
I20250901 08:00:51.846711 1187 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.232.126:45715 every 8 connection(s)
I20250901 08:00:51.850534 1188 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:51.859629 1188 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2: Bootstrap starting.
I20250901 08:00:51.863333 1188 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:51.866876 1188 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2: No bootstrap required, opened a new log
I20250901 08:00:51.868423 1188 raft_consensus.cc:357] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "2734f355997d434791b649f97ba0c2f2" member_type: VOTER }
I20250901 08:00:51.868733 1188 raft_consensus.cc:383] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:51.868927 1188 raft_consensus.cc:738] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2734f355997d434791b649f97ba0c2f2, State: Initialized, Role: FOLLOWER
I20250901 08:00:51.869360 1188 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "2734f355997d434791b649f97ba0c2f2" member_type: VOTER }
I20250901 08:00:51.869719 1188 raft_consensus.cc:397] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:51.869911 1188 raft_consensus.cc:491] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:51.870129 1188 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:51.873492 1188 raft_consensus.cc:513] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "2734f355997d434791b649f97ba0c2f2" member_type: VOTER }
I20250901 08:00:51.873967 1188 leader_election.cc:304] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 2734f355997d434791b649f97ba0c2f2; no voters:
I20250901 08:00:51.875103 1188 leader_election.cc:290] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:51.875384 1191 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:51.876643 1191 raft_consensus.cc:695] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [term 1 LEADER]: Becoming Leader. State: Replica: 2734f355997d434791b649f97ba0c2f2, State: Running, Role: LEADER
I20250901 08:00:51.877236 1191 consensus_queue.cc:237] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "2734f355997d434791b649f97ba0c2f2" member_type: VOTER }
I20250901 08:00:51.877672 1188 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [sys.catalog]: configured and running, proceeding with master startup.
I20250901 08:00:51.879667 1192 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "2734f355997d434791b649f97ba0c2f2" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "2734f355997d434791b649f97ba0c2f2" member_type: VOTER } }
I20250901 08:00:51.879735 1193 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2734f355997d434791b649f97ba0c2f2. Latest consensus state: current_term: 1 leader_uuid: "2734f355997d434791b649f97ba0c2f2" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "2734f355997d434791b649f97ba0c2f2" member_type: VOTER } }
I20250901 08:00:51.880242 1192 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [sys.catalog]: This master's current role is: LEADER
I20250901 08:00:51.880357 1193 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [sys.catalog]: This master's current role is: LEADER
I20250901 08:00:51.892153 1196 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250901 08:00:51.899899 1196 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250901 08:00:51.900766 929 internal_mini_cluster.cc:184] Waiting to initialize catalog manager on master 0
I20250901 08:00:51.908407 1196 catalog_manager.cc:1349] Generated new cluster ID: 6f0d4b204b604d9c91fa1f9638dde6ec
I20250901 08:00:51.908650 1196 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250901 08:00:51.926460 1196 catalog_manager.cc:1372] Generated new certificate authority record
I20250901 08:00:51.927386 1196 catalog_manager.cc:1506] Loading token signing keys...
I20250901 08:00:51.941692 1196 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2: Generated new TSK 0
I20250901 08:00:51.942166 1196 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250901 08:00:51.966670 929 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250901 08:00:51.972229 1209 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:51.972669 1210 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:51.975121 1212 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250901 08:00:51.976029 929 server_base.cc:1047] running on GCE node
I20250901 08:00:51.976787 929 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250901 08:00:51.976946 929 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250901 08:00:51.977088 929 hybrid_clock.cc:648] HybridClock initialized: now 1756713651977073 us; error 0 us; skew 500 ppm
I20250901 08:00:51.977530 929 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250901 08:00:51.980000 929 webserver.cc:480] Webserver started at http://127.0.232.65:40639/ using document root <none> and password file <none>
I20250901 08:00:51.980410 929 fs_manager.cc:362] Metadata directory not provided
I20250901 08:00:51.980566 929 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250901 08:00:51.980809 929 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
I20250901 08:00:51.981773 929 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1756713648558816-929-0/minicluster-data/ts-0-root/instance:
uuid: "dd9be895c9cd4a45bf57d15af612c38a"
format_stamp: "Formatted at 2025-09-01 08:00:51 on dist-test-slave-v0s2"
I20250901 08:00:51.986054 929 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.004s sys 0.002s
I20250901 08:00:51.989001 1217 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:51.989758 929 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20250901 08:00:51.990093 929 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1756713648558816-929-0/minicluster-data/ts-0-root
uuid: "dd9be895c9cd4a45bf57d15af612c38a"
format_stamp: "Formatted at 2025-09-01 08:00:51 on dist-test-slave-v0s2"
I20250901 08:00:51.990468 929 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1756713648558816-929-0/minicluster-data/ts-0-root
metadata directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1756713648558816-929-0/minicluster-data/ts-0-root
1 data directories: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1756713648558816-929-0/minicluster-data/ts-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250901 08:00:52.005950 929 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250901 08:00:52.007094 929 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250901 08:00:52.008785 929 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20250901 08:00:52.011389 929 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20250901 08:00:52.011611 929 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:52.011857 929 ts_tablet_manager.cc:610] Registered 0 tablets
I20250901 08:00:52.012038 929 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:52.054363 929 rpc_server.cc:307] RPC server started. Bound to: 127.0.232.65:37463
I20250901 08:00:52.054459 1279 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.232.65:37463 every 8 connection(s)
I20250901 08:00:52.071048 1280 heartbeater.cc:344] Connected to a master server at 127.0.232.126:45715
I20250901 08:00:52.071385 1280 heartbeater.cc:461] Registering TS with master...
I20250901 08:00:52.072080 1280 heartbeater.cc:507] Master 127.0.232.126:45715 requested a full tablet report, sending...
I20250901 08:00:52.073891 1153 ts_manager.cc:194] Registered new tserver with Master: dd9be895c9cd4a45bf57d15af612c38a (127.0.232.65:37463)
I20250901 08:00:52.074740 929 internal_mini_cluster.cc:371] 1 TS(s) registered with all masters after 0.013876623s
I20250901 08:00:52.076076 1153 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.0.1:53658
I20250901 08:00:52.096352 1153 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53660:
name: "test_table"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
rows: "\004\001d\000\000\000\004\001\310\000\000\000\004\001,\001\000\000\004\001\220\001\000\000\004\001\364\001\000\000\004\001X\002\000\000\004\001\274\002\000\000\004\001 \003\000\000\004\001\204\003\000\000""\004\001d\000\000\000\004\001\310\000\000\000\004\001,\001\000\000\004\001\220\001\000\000\004\001\364\001\000\000\004\001X\002\000\000\004\001\274\002\000\000\004\001 \003\000\000\004\001\204\003\000\000"
indirect_data: """"
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
I20250901 08:00:52.146018 1245 tablet_service.cc:1468] Processing CreateTablet for tablet c9d2524c92a34e7a8772b398ea393331 (DEFAULT_TABLE table=test_table [id=87e7b8ce792c474db5b8b4362e3dde6c]), partition=RANGE (key) PARTITION VALUES < 100
I20250901 08:00:52.146312 1244 tablet_service.cc:1468] Processing CreateTablet for tablet 2ea8f3cbf9624b71a8a23bc095e1cdf3 (DEFAULT_TABLE table=test_table [id=87e7b8ce792c474db5b8b4362e3dde6c]), partition=RANGE (key) PARTITION 100 <= VALUES < 200
I20250901 08:00:52.147249 1245 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c9d2524c92a34e7a8772b398ea393331. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:52.147922 1242 tablet_service.cc:1468] Processing CreateTablet for tablet 8621ac3710b149e0bebe2b83c4ded465 (DEFAULT_TABLE table=test_table [id=87e7b8ce792c474db5b8b4362e3dde6c]), partition=RANGE (key) PARTITION 300 <= VALUES < 400
I20250901 08:00:52.148389 1243 tablet_service.cc:1468] Processing CreateTablet for tablet 21c0226670c64ecd891b1aab4fe6dcfc (DEFAULT_TABLE table=test_table [id=87e7b8ce792c474db5b8b4362e3dde6c]), partition=RANGE (key) PARTITION 200 <= VALUES < 300
I20250901 08:00:52.149096 1242 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8621ac3710b149e0bebe2b83c4ded465. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:52.149657 1243 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 21c0226670c64ecd891b1aab4fe6dcfc. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:52.150600 1241 tablet_service.cc:1468] Processing CreateTablet for tablet ca216a330d1b4e3ca91931137bf88618 (DEFAULT_TABLE table=test_table [id=87e7b8ce792c474db5b8b4362e3dde6c]), partition=RANGE (key) PARTITION 400 <= VALUES < 500
I20250901 08:00:52.151750 1241 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ca216a330d1b4e3ca91931137bf88618. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:52.152683 1237 tablet_service.cc:1468] Processing CreateTablet for tablet 389febcccb654700a6b2bc0992f5f678 (DEFAULT_TABLE table=test_table [id=87e7b8ce792c474db5b8b4362e3dde6c]), partition=RANGE (key) PARTITION 800 <= VALUES < 900
I20250901 08:00:52.153810 1237 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 389febcccb654700a6b2bc0992f5f678. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:52.155133 1240 tablet_service.cc:1468] Processing CreateTablet for tablet 1cabdad2ea8841c3899c8125ee95ea4f (DEFAULT_TABLE table=test_table [id=87e7b8ce792c474db5b8b4362e3dde6c]), partition=RANGE (key) PARTITION 500 <= VALUES < 600
I20250901 08:00:52.156257 1240 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1cabdad2ea8841c3899c8125ee95ea4f. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:52.159089 1239 tablet_service.cc:1468] Processing CreateTablet for tablet ecd0e96dcc4846689964b2d1008c22ee (DEFAULT_TABLE table=test_table [id=87e7b8ce792c474db5b8b4362e3dde6c]), partition=RANGE (key) PARTITION 600 <= VALUES < 700
I20250901 08:00:52.160213 1239 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ecd0e96dcc4846689964b2d1008c22ee. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:52.164072 1236 tablet_service.cc:1468] Processing CreateTablet for tablet 899915a0800b4f52b9d90ff19a3501a3 (DEFAULT_TABLE table=test_table [id=87e7b8ce792c474db5b8b4362e3dde6c]), partition=RANGE (key) PARTITION 900 <= VALUES
I20250901 08:00:52.165625 1236 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 899915a0800b4f52b9d90ff19a3501a3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:52.167440 1238 tablet_service.cc:1468] Processing CreateTablet for tablet d31ca0c301874c609433cc2ac4752757 (DEFAULT_TABLE table=test_table [id=87e7b8ce792c474db5b8b4362e3dde6c]), partition=RANGE (key) PARTITION 700 <= VALUES < 800
I20250901 08:00:52.168920 1238 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d31ca0c301874c609433cc2ac4752757. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:52.173264 1244 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2ea8f3cbf9624b71a8a23bc095e1cdf3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:52.219367 1292 tablet_bootstrap.cc:492] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a: Bootstrap starting.
I20250901 08:00:52.224946 1292 tablet_bootstrap.cc:654] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:52.229167 1292 tablet_bootstrap.cc:492] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a: No bootstrap required, opened a new log
I20250901 08:00:52.229540 1292 ts_tablet_manager.cc:1397] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a: Time spent bootstrapping tablet: real 0.010s user 0.003s sys 0.005s
I20250901 08:00:52.231905 1292 raft_consensus.cc:357] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.232420 1292 raft_consensus.cc:383] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:52.232688 1292 raft_consensus.cc:738] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Initialized, Role: FOLLOWER
I20250901 08:00:52.233207 1292 consensus_queue.cc:260] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.233801 1292 raft_consensus.cc:397] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:52.234087 1292 raft_consensus.cc:491] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:52.234440 1292 raft_consensus.cc:3058] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:52.241714 1292 raft_consensus.cc:513] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.242704 1292 leader_election.cc:304] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: dd9be895c9cd4a45bf57d15af612c38a; no voters:
I20250901 08:00:52.244791 1292 leader_election.cc:290] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:52.245220 1294 raft_consensus.cc:2802] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:52.247294 1280 heartbeater.cc:499] Master 127.0.232.126:45715 was elected leader, sending a full tablet report...
I20250901 08:00:52.248042 1292 ts_tablet_manager.cc:1428] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a: Time spent starting tablet: real 0.018s user 0.013s sys 0.005s
I20250901 08:00:52.249085 1292 tablet_bootstrap.cc:492] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a: Bootstrap starting.
I20250901 08:00:52.250437 1294 raft_consensus.cc:695] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Becoming Leader. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Running, Role: LEADER
I20250901 08:00:52.251029 1294 consensus_queue.cc:237] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.254777 1292 tablet_bootstrap.cc:654] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:52.261278 1292 tablet_bootstrap.cc:492] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a: No bootstrap required, opened a new log
I20250901 08:00:52.261644 1292 ts_tablet_manager.cc:1397] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a: Time spent bootstrapping tablet: real 0.013s user 0.011s sys 0.000s
I20250901 08:00:52.263993 1292 raft_consensus.cc:357] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.264503 1292 raft_consensus.cc:383] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:52.264492 1153 catalog_manager.cc:5582] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a reported cstate change: term changed from 0 to 1, leader changed from <none> to dd9be895c9cd4a45bf57d15af612c38a (127.0.232.65). New cstate: current_term: 1 leader_uuid: "dd9be895c9cd4a45bf57d15af612c38a" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:52.264873 1292 raft_consensus.cc:738] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Initialized, Role: FOLLOWER
I20250901 08:00:52.265460 1292 consensus_queue.cc:260] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.266050 1292 raft_consensus.cc:397] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:52.266388 1292 raft_consensus.cc:491] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:52.266708 1292 raft_consensus.cc:3058] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:52.271970 1292 raft_consensus.cc:513] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.272574 1292 leader_election.cc:304] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: dd9be895c9cd4a45bf57d15af612c38a; no voters:
I20250901 08:00:52.273097 1292 leader_election.cc:290] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:52.273375 1296 raft_consensus.cc:2802] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:52.274344 1296 raft_consensus.cc:695] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Becoming Leader. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Running, Role: LEADER
I20250901 08:00:52.274655 1292 ts_tablet_manager.cc:1428] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a: Time spent starting tablet: real 0.013s user 0.004s sys 0.011s
I20250901 08:00:52.275388 1292 tablet_bootstrap.cc:492] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a: Bootstrap starting.
I20250901 08:00:52.275167 1296 consensus_queue.cc:237] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.279976 1292 tablet_bootstrap.cc:654] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:52.281075 1153 catalog_manager.cc:5582] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a reported cstate change: term changed from 0 to 1, leader changed from <none> to dd9be895c9cd4a45bf57d15af612c38a (127.0.232.65). New cstate: current_term: 1 leader_uuid: "dd9be895c9cd4a45bf57d15af612c38a" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:52.285633 1292 tablet_bootstrap.cc:492] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a: No bootstrap required, opened a new log
I20250901 08:00:52.285948 1292 ts_tablet_manager.cc:1397] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a: Time spent bootstrapping tablet: real 0.011s user 0.008s sys 0.000s
I20250901 08:00:52.287599 1292 raft_consensus.cc:357] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.288023 1292 raft_consensus.cc:383] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:52.288216 1292 raft_consensus.cc:738] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Initialized, Role: FOLLOWER
I20250901 08:00:52.288709 1292 consensus_queue.cc:260] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.289187 1292 raft_consensus.cc:397] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:52.289438 1292 raft_consensus.cc:491] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:52.289669 1292 raft_consensus.cc:3058] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:52.293347 1292 raft_consensus.cc:513] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.293970 1292 leader_election.cc:304] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: dd9be895c9cd4a45bf57d15af612c38a; no voters:
I20250901 08:00:52.294488 1292 leader_election.cc:290] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:52.294615 1296 raft_consensus.cc:2802] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:52.295027 1296 raft_consensus.cc:695] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Becoming Leader. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Running, Role: LEADER
I20250901 08:00:52.295650 1296 consensus_queue.cc:237] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.296182 1292 ts_tablet_manager.cc:1428] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a: Time spent starting tablet: real 0.010s user 0.010s sys 0.000s
I20250901 08:00:52.296886 1292 tablet_bootstrap.cc:492] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a: Bootstrap starting.
I20250901 08:00:52.300902 1153 catalog_manager.cc:5582] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a reported cstate change: term changed from 0 to 1, leader changed from <none> to dd9be895c9cd4a45bf57d15af612c38a (127.0.232.65). New cstate: current_term: 1 leader_uuid: "dd9be895c9cd4a45bf57d15af612c38a" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:52.301292 1292 tablet_bootstrap.cc:654] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:52.304929 1292 tablet_bootstrap.cc:492] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a: No bootstrap required, opened a new log
I20250901 08:00:52.305262 1292 ts_tablet_manager.cc:1397] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a: Time spent bootstrapping tablet: real 0.009s user 0.007s sys 0.000s
I20250901 08:00:52.307011 1292 raft_consensus.cc:357] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.307399 1292 raft_consensus.cc:383] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:52.307641 1292 raft_consensus.cc:738] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Initialized, Role: FOLLOWER
I20250901 08:00:52.308220 1292 consensus_queue.cc:260] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.308825 1292 raft_consensus.cc:397] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:52.309089 1292 raft_consensus.cc:491] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:52.309355 1292 raft_consensus.cc:3058] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:52.315517 1292 raft_consensus.cc:513] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.315970 1292 leader_election.cc:304] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: dd9be895c9cd4a45bf57d15af612c38a; no voters:
I20250901 08:00:52.316396 1292 leader_election.cc:290] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:52.316527 1296 raft_consensus.cc:2802] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:52.316907 1296 raft_consensus.cc:695] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Becoming Leader. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Running, Role: LEADER
I20250901 08:00:52.317531 1296 consensus_queue.cc:237] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.318041 1292 ts_tablet_manager.cc:1428] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a: Time spent starting tablet: real 0.013s user 0.006s sys 0.003s
I20250901 08:00:52.318843 1292 tablet_bootstrap.cc:492] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a: Bootstrap starting.
I20250901 08:00:52.323302 1292 tablet_bootstrap.cc:654] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:52.323529 1153 catalog_manager.cc:5582] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a reported cstate change: term changed from 0 to 1, leader changed from <none> to dd9be895c9cd4a45bf57d15af612c38a (127.0.232.65). New cstate: current_term: 1 leader_uuid: "dd9be895c9cd4a45bf57d15af612c38a" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:52.327236 1292 tablet_bootstrap.cc:492] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a: No bootstrap required, opened a new log
I20250901 08:00:52.327561 1292 ts_tablet_manager.cc:1397] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a: Time spent bootstrapping tablet: real 0.009s user 0.006s sys 0.001s
I20250901 08:00:52.329306 1292 raft_consensus.cc:357] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.329754 1292 raft_consensus.cc:383] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:52.330004 1292 raft_consensus.cc:738] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Initialized, Role: FOLLOWER
I20250901 08:00:52.330560 1292 consensus_queue.cc:260] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.331140 1292 raft_consensus.cc:397] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:52.331444 1292 raft_consensus.cc:491] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:52.331782 1292 raft_consensus.cc:3058] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:52.336048 1292 raft_consensus.cc:513] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.336460 1292 leader_election.cc:304] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: dd9be895c9cd4a45bf57d15af612c38a; no voters:
I20250901 08:00:52.336853 1292 leader_election.cc:290] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:52.336994 1296 raft_consensus.cc:2802] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:52.337404 1296 raft_consensus.cc:695] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Becoming Leader. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Running, Role: LEADER
I20250901 08:00:52.338009 1296 consensus_queue.cc:237] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.338474 1292 ts_tablet_manager.cc:1428] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a: Time spent starting tablet: real 0.011s user 0.008s sys 0.004s
I20250901 08:00:52.339196 1292 tablet_bootstrap.cc:492] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a: Bootstrap starting.
I20250901 08:00:52.343389 1292 tablet_bootstrap.cc:654] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:52.343215 1153 catalog_manager.cc:5582] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a reported cstate change: term changed from 0 to 1, leader changed from <none> to dd9be895c9cd4a45bf57d15af612c38a (127.0.232.65). New cstate: current_term: 1 leader_uuid: "dd9be895c9cd4a45bf57d15af612c38a" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:52.346827 1292 tablet_bootstrap.cc:492] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a: No bootstrap required, opened a new log
I20250901 08:00:52.347114 1292 ts_tablet_manager.cc:1397] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a: Time spent bootstrapping tablet: real 0.008s user 0.006s sys 0.001s
I20250901 08:00:52.348943 1292 raft_consensus.cc:357] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.349381 1292 raft_consensus.cc:383] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:52.349577 1292 raft_consensus.cc:738] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Initialized, Role: FOLLOWER
I20250901 08:00:52.350018 1292 consensus_queue.cc:260] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.350489 1292 raft_consensus.cc:397] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:52.350667 1292 raft_consensus.cc:491] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:52.350832 1292 raft_consensus.cc:3058] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:52.354218 1292 raft_consensus.cc:513] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.354671 1292 leader_election.cc:304] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: dd9be895c9cd4a45bf57d15af612c38a; no voters:
I20250901 08:00:52.355015 1292 leader_election.cc:290] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:52.355170 1296 raft_consensus.cc:2802] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:52.355762 1296 raft_consensus.cc:695] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Becoming Leader. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Running, Role: LEADER
I20250901 08:00:52.356530 1292 ts_tablet_manager.cc:1428] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a: Time spent starting tablet: real 0.009s user 0.008s sys 0.000s
I20250901 08:00:52.356407 1296 consensus_queue.cc:237] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.357209 1292 tablet_bootstrap.cc:492] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a: Bootstrap starting.
I20250901 08:00:52.360971 1292 tablet_bootstrap.cc:654] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:52.361976 1153 catalog_manager.cc:5582] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a reported cstate change: term changed from 0 to 1, leader changed from <none> to dd9be895c9cd4a45bf57d15af612c38a (127.0.232.65). New cstate: current_term: 1 leader_uuid: "dd9be895c9cd4a45bf57d15af612c38a" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:52.364486 1292 tablet_bootstrap.cc:492] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a: No bootstrap required, opened a new log
I20250901 08:00:52.364862 1292 ts_tablet_manager.cc:1397] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.003s
I20250901 08:00:52.366729 1292 raft_consensus.cc:357] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.367095 1292 raft_consensus.cc:383] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:52.367328 1292 raft_consensus.cc:738] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Initialized, Role: FOLLOWER
I20250901 08:00:52.367818 1292 consensus_queue.cc:260] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.368208 1292 raft_consensus.cc:397] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:52.368444 1292 raft_consensus.cc:491] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:52.368741 1292 raft_consensus.cc:3058] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:52.373378 1292 raft_consensus.cc:513] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.373911 1292 leader_election.cc:304] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: dd9be895c9cd4a45bf57d15af612c38a; no voters:
I20250901 08:00:52.374464 1292 leader_election.cc:290] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:52.374596 1296 raft_consensus.cc:2802] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:52.375010 1296 raft_consensus.cc:695] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Becoming Leader. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Running, Role: LEADER
I20250901 08:00:52.375631 1296 consensus_queue.cc:237] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.376031 1292 ts_tablet_manager.cc:1428] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a: Time spent starting tablet: real 0.011s user 0.011s sys 0.000s
I20250901 08:00:52.376684 1292 tablet_bootstrap.cc:492] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a: Bootstrap starting.
I20250901 08:00:52.380613 1292 tablet_bootstrap.cc:654] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:52.381353 1153 catalog_manager.cc:5582] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a reported cstate change: term changed from 0 to 1, leader changed from <none> to dd9be895c9cd4a45bf57d15af612c38a (127.0.232.65). New cstate: current_term: 1 leader_uuid: "dd9be895c9cd4a45bf57d15af612c38a" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:52.384045 1292 tablet_bootstrap.cc:492] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a: No bootstrap required, opened a new log
I20250901 08:00:52.384334 1292 ts_tablet_manager.cc:1397] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.003s
I20250901 08:00:52.385948 1292 raft_consensus.cc:357] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.386318 1292 raft_consensus.cc:383] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:52.386471 1292 raft_consensus.cc:738] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Initialized, Role: FOLLOWER
I20250901 08:00:52.386888 1292 consensus_queue.cc:260] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.387351 1292 raft_consensus.cc:397] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:52.387585 1292 raft_consensus.cc:491] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:52.387850 1292 raft_consensus.cc:3058] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:52.392834 1292 raft_consensus.cc:513] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.393348 1292 leader_election.cc:304] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: dd9be895c9cd4a45bf57d15af612c38a; no voters:
I20250901 08:00:52.393796 1292 leader_election.cc:290] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:52.393899 1296 raft_consensus.cc:2802] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:52.394361 1296 raft_consensus.cc:695] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Becoming Leader. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Running, Role: LEADER
I20250901 08:00:52.395258 1292 ts_tablet_manager.cc:1428] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a: Time spent starting tablet: real 0.011s user 0.011s sys 0.000s
I20250901 08:00:52.395025 1296 consensus_queue.cc:237] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.395911 1292 tablet_bootstrap.cc:492] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a: Bootstrap starting.
I20250901 08:00:52.399868 1292 tablet_bootstrap.cc:654] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:52.400416 1153 catalog_manager.cc:5582] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a reported cstate change: term changed from 0 to 1, leader changed from <none> to dd9be895c9cd4a45bf57d15af612c38a (127.0.232.65). New cstate: current_term: 1 leader_uuid: "dd9be895c9cd4a45bf57d15af612c38a" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:52.403002 1292 tablet_bootstrap.cc:492] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a: No bootstrap required, opened a new log
I20250901 08:00:52.403282 1292 ts_tablet_manager.cc:1397] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a: Time spent bootstrapping tablet: real 0.008s user 0.006s sys 0.000s
I20250901 08:00:52.404835 1292 raft_consensus.cc:357] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.405157 1292 raft_consensus.cc:383] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:52.405361 1292 raft_consensus.cc:738] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Initialized, Role: FOLLOWER
I20250901 08:00:52.405877 1292 consensus_queue.cc:260] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.406409 1292 raft_consensus.cc:397] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:52.406610 1292 raft_consensus.cc:491] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:52.406881 1292 raft_consensus.cc:3058] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:52.412292 1292 raft_consensus.cc:513] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.412707 1292 leader_election.cc:304] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: dd9be895c9cd4a45bf57d15af612c38a; no voters:
I20250901 08:00:52.413025 1292 leader_election.cc:290] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:52.413134 1296 raft_consensus.cc:2802] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:52.413558 1296 raft_consensus.cc:695] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Becoming Leader. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Running, Role: LEADER
I20250901 08:00:52.414186 1292 ts_tablet_manager.cc:1428] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a: Time spent starting tablet: real 0.011s user 0.008s sys 0.000s
I20250901 08:00:52.414098 1296 consensus_queue.cc:237] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.414799 1292 tablet_bootstrap.cc:492] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a: Bootstrap starting.
I20250901 08:00:52.419021 1292 tablet_bootstrap.cc:654] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:52.419894 1153 catalog_manager.cc:5582] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a reported cstate change: term changed from 0 to 1, leader changed from <none> to dd9be895c9cd4a45bf57d15af612c38a (127.0.232.65). New cstate: current_term: 1 leader_uuid: "dd9be895c9cd4a45bf57d15af612c38a" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:52.422770 1292 tablet_bootstrap.cc:492] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a: No bootstrap required, opened a new log
I20250901 08:00:52.423050 1292 ts_tablet_manager.cc:1397] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a: Time spent bootstrapping tablet: real 0.008s user 0.007s sys 0.000s
I20250901 08:00:52.425024 1292 raft_consensus.cc:357] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.425357 1292 raft_consensus.cc:383] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:52.425570 1292 raft_consensus.cc:738] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Initialized, Role: FOLLOWER
I20250901 08:00:52.426025 1292 consensus_queue.cc:260] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.426398 1292 raft_consensus.cc:397] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:52.426653 1292 raft_consensus.cc:491] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:52.426898 1292 raft_consensus.cc:3058] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:52.430449 1292 raft_consensus.cc:513] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.430994 1292 leader_election.cc:304] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: dd9be895c9cd4a45bf57d15af612c38a; no voters:
I20250901 08:00:52.431355 1292 leader_election.cc:290] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:52.431520 1296 raft_consensus.cc:2802] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:52.432096 1296 raft_consensus.cc:695] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Becoming Leader. State: Replica: dd9be895c9cd4a45bf57d15af612c38a, State: Running, Role: LEADER
I20250901 08:00:52.432793 1292 ts_tablet_manager.cc:1428] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a: Time spent starting tablet: real 0.010s user 0.001s sys 0.007s
I20250901 08:00:52.432674 1296 consensus_queue.cc:237] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } }
I20250901 08:00:52.438752 1153 catalog_manager.cc:5582] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a reported cstate change: term changed from 0 to 1, leader changed from <none> to dd9be895c9cd4a45bf57d15af612c38a (127.0.232.65). New cstate: current_term: 1 leader_uuid: "dd9be895c9cd4a45bf57d15af612c38a" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dd9be895c9cd4a45bf57d15af612c38a" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 37463 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:52.453847 929 catalog_manager.cc:2739] Servicing DeleteTable request from test_user:
table { table_name: "test_table" }
I20250901 08:00:52.465361 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2: Sending DeleteTablet for 1 replicas of tablet c9d2524c92a34e7a8772b398ea393331
I20250901 08:00:52.465834 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2: Sending DeleteTablet for 1 replicas of tablet 2ea8f3cbf9624b71a8a23bc095e1cdf3
I20250901 08:00:52.466226 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2: Sending DeleteTablet for 1 replicas of tablet 21c0226670c64ecd891b1aab4fe6dcfc
I20250901 08:00:52.466578 1241 tablet_service.cc:1515] Processing DeleteTablet for tablet c9d2524c92a34e7a8772b398ea393331 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-09-01 08:00:52 UTC) from {username='slave'} at 127.0.0.1:37900
I20250901 08:00:52.466777 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2: Sending DeleteTablet for 1 replicas of tablet 8621ac3710b149e0bebe2b83c4ded465
I20250901 08:00:52.466967 1245 tablet_service.cc:1515] Processing DeleteTablet for tablet 2ea8f3cbf9624b71a8a23bc095e1cdf3 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-09-01 08:00:52 UTC) from {username='slave'} at 127.0.0.1:37900
I20250901 08:00:52.467522 1238 tablet_service.cc:1515] Processing DeleteTablet for tablet 21c0226670c64ecd891b1aab4fe6dcfc with delete_type TABLET_DATA_DELETED (Table deleted at 2025-09-01 08:00:52 UTC) from {username='slave'} at 127.0.0.1:37900
I20250901 08:00:52.468111 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2: Sending DeleteTablet for 1 replicas of tablet ca216a330d1b4e3ca91931137bf88618
I20250901 08:00:52.468251 1317 tablet_replica.cc:331] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a: stopping tablet replica
I20250901 08:00:52.468441 1238 tablet_service.cc:1515] Processing DeleteTablet for tablet 8621ac3710b149e0bebe2b83c4ded465 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-09-01 08:00:52 UTC) from {username='slave'} at 127.0.0.1:37900
I20250901 08:00:52.468708 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2: Sending DeleteTablet for 1 replicas of tablet 1cabdad2ea8841c3899c8125ee95ea4f
I20250901 08:00:52.469066 1317 raft_consensus.cc:2241] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:52.469285 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2: Sending DeleteTablet for 1 replicas of tablet ecd0e96dcc4846689964b2d1008c22ee
I20250901 08:00:52.469463 1238 tablet_service.cc:1515] Processing DeleteTablet for tablet ca216a330d1b4e3ca91931137bf88618 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-09-01 08:00:52 UTC) from {username='slave'} at 127.0.0.1:37900
I20250901 08:00:52.469538 1317 raft_consensus.cc:2270] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:52.470136 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2: Sending DeleteTablet for 1 replicas of tablet d31ca0c301874c609433cc2ac4752757
I20250901 08:00:52.470046 1241 tablet_service.cc:1515] Processing DeleteTablet for tablet 1cabdad2ea8841c3899c8125ee95ea4f with delete_type TABLET_DATA_DELETED (Table deleted at 2025-09-01 08:00:52 UTC) from {username='slave'} at 127.0.0.1:37900
I20250901 08:00:52.470682 1238 tablet_service.cc:1515] Processing DeleteTablet for tablet ecd0e96dcc4846689964b2d1008c22ee with delete_type TABLET_DATA_DELETED (Table deleted at 2025-09-01 08:00:52 UTC) from {username='slave'} at 127.0.0.1:37900
I20250901 08:00:52.470790 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2: Sending DeleteTablet for 1 replicas of tablet 389febcccb654700a6b2bc0992f5f678
I20250901 08:00:52.471333 929 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2: Sending DeleteTablet for 1 replicas of tablet 899915a0800b4f52b9d90ff19a3501a3
I20250901 08:00:52.471787 1238 tablet_service.cc:1515] Processing DeleteTablet for tablet d31ca0c301874c609433cc2ac4752757 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-09-01 08:00:52 UTC) from {username='slave'} at 127.0.0.1:37900
I20250901 08:00:52.472831 1317 ts_tablet_manager.cc:1905] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250901 08:00:52.472702 1238 tablet_service.cc:1515] Processing DeleteTablet for tablet 389febcccb654700a6b2bc0992f5f678 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-09-01 08:00:52 UTC) from {username='slave'} at 127.0.0.1:37900
I20250901 08:00:52.473026 1241 tablet_service.cc:1515] Processing DeleteTablet for tablet 899915a0800b4f52b9d90ff19a3501a3 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-09-01 08:00:52 UTC) from {username='slave'} at 127.0.0.1:37900
I20250901 08:00:52.473675 929 tablet_server.cc:178] TabletServer@127.0.232.65:0 shutting down...
I20250901 08:00:52.487643 1317 ts_tablet_manager.cc:1918] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250901 08:00:52.487888 1317 log.cc:1199] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a: Deleting WAL directory at /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1756713648558816-929-0/minicluster-data/ts-0-root/wals/c9d2524c92a34e7a8772b398ea393331
I20250901 08:00:52.489998 1317 ts_tablet_manager.cc:1939] T c9d2524c92a34e7a8772b398ea393331 P dd9be895c9cd4a45bf57d15af612c38a: Deleting consensus metadata
I20250901 08:00:52.492568 929 ts_tablet_manager.cc:1500] Shutting down tablet manager...
I20250901 08:00:52.492807 1139 catalog_manager.cc:4928] TS dd9be895c9cd4a45bf57d15af612c38a (127.0.232.65:37463): tablet c9d2524c92a34e7a8772b398ea393331 (table test_table [id=87e7b8ce792c474db5b8b4362e3dde6c]) successfully deleted
W20250901 08:00:52.493567 1139 catalog_manager.cc:4920] TS dd9be895c9cd4a45bf57d15af612c38a (127.0.232.65:37463): delete failed for tablet 2ea8f3cbf9624b71a8a23bc095e1cdf3 with error code TABLET_NOT_RUNNING: Service unavailable: Tablet Manager is not running: MANAGER_QUIESCING
I20250901 08:00:52.496232 929 tablet_replica.cc:331] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a: stopping tablet replica
I20250901 08:00:52.496939 929 raft_consensus.cc:2241] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:52.497463 929 raft_consensus.cc:2270] T ca216a330d1b4e3ca91931137bf88618 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Raft consensus is shut down!
W20250901 08:00:52.499182 1139 catalog_manager.cc:4671] TS dd9be895c9cd4a45bf57d15af612c38a (127.0.232.65:37463): DeleteTablet:TABLET_DATA_DELETED RPC failed for tablet 21c0226670c64ecd891b1aab4fe6dcfc: Remote error: Service unavailable: service kudu.tserver.TabletServerAdminService not registered on TabletServer
I20250901 08:00:52.500123 929 tablet_replica.cc:331] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a: stopping tablet replica
I20250901 08:00:52.500927 929 raft_consensus.cc:2241] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:52.501359 929 raft_consensus.cc:2270] T d31ca0c301874c609433cc2ac4752757 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:52.503150 929 tablet_replica.cc:331] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a: stopping tablet replica
I20250901 08:00:52.503579 929 raft_consensus.cc:2241] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:52.503892 929 raft_consensus.cc:2270] T 2ea8f3cbf9624b71a8a23bc095e1cdf3 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:52.505607 929 tablet_replica.cc:331] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a: stopping tablet replica
I20250901 08:00:52.506003 929 raft_consensus.cc:2241] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:52.506361 929 raft_consensus.cc:2270] T 1cabdad2ea8841c3899c8125ee95ea4f P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:52.508041 929 tablet_replica.cc:331] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a: stopping tablet replica
I20250901 08:00:52.508450 929 raft_consensus.cc:2241] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:52.508749 929 raft_consensus.cc:2270] T ecd0e96dcc4846689964b2d1008c22ee P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:52.510406 929 tablet_replica.cc:331] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a: stopping tablet replica
I20250901 08:00:52.510778 929 raft_consensus.cc:2241] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:52.511063 929 raft_consensus.cc:2270] T 21c0226670c64ecd891b1aab4fe6dcfc P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:52.512815 929 tablet_replica.cc:331] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a: stopping tablet replica
I20250901 08:00:52.513190 929 raft_consensus.cc:2241] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:52.513480 929 raft_consensus.cc:2270] T 389febcccb654700a6b2bc0992f5f678 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:52.515565 929 tablet_replica.cc:331] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a: stopping tablet replica
I20250901 08:00:52.516069 929 raft_consensus.cc:2241] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:52.516505 929 raft_consensus.cc:2270] T 899915a0800b4f52b9d90ff19a3501a3 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:52.518245 929 tablet_replica.cc:331] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a: stopping tablet replica
I20250901 08:00:52.518648 929 raft_consensus.cc:2241] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:52.518937 929 raft_consensus.cc:2270] T 8621ac3710b149e0bebe2b83c4ded465 P dd9be895c9cd4a45bf57d15af612c38a [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:52.532666 929 tablet_server.cc:195] TabletServer@127.0.232.65:0 shutdown complete.
I20250901 08:00:52.543489 929 master.cc:561] Master@127.0.232.126:45715 shutting down...
I20250901 08:00:52.641588 929 raft_consensus.cc:2241] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:52.641988 929 raft_consensus.cc:2270] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:52.642191 929 tablet_replica.cc:331] T 00000000000000000000000000000000 P 2734f355997d434791b649f97ba0c2f2: stopping tablet replica
I20250901 08:00:52.659102 929 master.cc:583] Master@127.0.232.126:45715 shutdown complete.
[ OK ] CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser (935 ms)
[ RUN ] CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc
I20250901 08:00:52.677464 929 internal_mini_cluster.cc:156] Creating distributed mini masters. Addrs: 127.0.232.126:42783
I20250901 08:00:52.678339 929 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250901 08:00:52.682322 1319 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:52.683349 1320 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:52.684988 1322 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250901 08:00:52.685190 929 server_base.cc:1047] running on GCE node
I20250901 08:00:52.686153 929 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250901 08:00:52.686331 929 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250901 08:00:52.686476 929 hybrid_clock.cc:648] HybridClock initialized: now 1756713652686461 us; error 0 us; skew 500 ppm
I20250901 08:00:52.686893 929 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250901 08:00:52.688740 929 webserver.cc:480] Webserver started at http://127.0.232.126:43857/ using document root <none> and password file <none>
I20250901 08:00:52.689116 929 fs_manager.cc:362] Metadata directory not provided
I20250901 08:00:52.689265 929 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250901 08:00:52.689474 929 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
I20250901 08:00:52.690367 929 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1756713648558816-929-0/minicluster-data/master-0-root/instance:
uuid: "80b6e6ac1f7d448e92ed90ff4a697c14"
format_stamp: "Formatted at 2025-09-01 08:00:52 on dist-test-slave-v0s2"
I20250901 08:00:52.693821 929 fs_manager.cc:696] Time spent creating directory manager: real 0.003s user 0.000s sys 0.003s
I20250901 08:00:52.696777 1327 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:52.697366 929 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20250901 08:00:52.697592 929 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1756713648558816-929-0/minicluster-data/master-0-root
uuid: "80b6e6ac1f7d448e92ed90ff4a697c14"
format_stamp: "Formatted at 2025-09-01 08:00:52 on dist-test-slave-v0s2"
I20250901 08:00:52.697814 929 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1756713648558816-929-0/minicluster-data/master-0-root
metadata directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1756713648558816-929-0/minicluster-data/master-0-root
1 data directories: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1756713648558816-929-0/minicluster-data/master-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250901 08:00:52.727011 929 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250901 08:00:52.727871 929 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250901 08:00:52.759488 929 rpc_server.cc:307] RPC server started. Bound to: 127.0.232.126:42783
I20250901 08:00:52.759562 1378 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.232.126:42783 every 8 connection(s)
I20250901 08:00:52.762995 1379 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:52.772043 1379 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14: Bootstrap starting.
I20250901 08:00:52.775826 1379 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:52.779121 1379 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14: No bootstrap required, opened a new log
I20250901 08:00:52.780668 1379 raft_consensus.cc:357] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "80b6e6ac1f7d448e92ed90ff4a697c14" member_type: VOTER }
I20250901 08:00:52.780992 1379 raft_consensus.cc:383] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:52.781159 1379 raft_consensus.cc:738] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 80b6e6ac1f7d448e92ed90ff4a697c14, State: Initialized, Role: FOLLOWER
I20250901 08:00:52.781607 1379 consensus_queue.cc:260] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "80b6e6ac1f7d448e92ed90ff4a697c14" member_type: VOTER }
I20250901 08:00:52.782003 1379 raft_consensus.cc:397] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:52.782179 1379 raft_consensus.cc:491] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:52.782470 1379 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:52.786067 1379 raft_consensus.cc:513] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "80b6e6ac1f7d448e92ed90ff4a697c14" member_type: VOTER }
I20250901 08:00:52.786484 1379 leader_election.cc:304] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 80b6e6ac1f7d448e92ed90ff4a697c14; no voters:
I20250901 08:00:52.787344 1379 leader_election.cc:290] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:52.787619 1382 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:52.788899 1382 raft_consensus.cc:695] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [term 1 LEADER]: Becoming Leader. State: Replica: 80b6e6ac1f7d448e92ed90ff4a697c14, State: Running, Role: LEADER
I20250901 08:00:52.789516 1382 consensus_queue.cc:237] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "80b6e6ac1f7d448e92ed90ff4a697c14" member_type: VOTER }
I20250901 08:00:52.790009 1379 sys_catalog.cc:564] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [sys.catalog]: configured and running, proceeding with master startup.
I20250901 08:00:52.792074 1383 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "80b6e6ac1f7d448e92ed90ff4a697c14" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "80b6e6ac1f7d448e92ed90ff4a697c14" member_type: VOTER } }
I20250901 08:00:52.792167 1384 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 80b6e6ac1f7d448e92ed90ff4a697c14. Latest consensus state: current_term: 1 leader_uuid: "80b6e6ac1f7d448e92ed90ff4a697c14" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "80b6e6ac1f7d448e92ed90ff4a697c14" member_type: VOTER } }
I20250901 08:00:52.792795 1384 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [sys.catalog]: This master's current role is: LEADER
I20250901 08:00:52.792812 1383 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [sys.catalog]: This master's current role is: LEADER
I20250901 08:00:52.795543 1387 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250901 08:00:52.800037 1387 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250901 08:00:52.803938 929 internal_mini_cluster.cc:184] Waiting to initialize catalog manager on master 0
I20250901 08:00:52.807586 1387 catalog_manager.cc:1349] Generated new cluster ID: 839a6a51666c42449e2578eeaaaef05b
I20250901 08:00:52.807819 1387 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250901 08:00:52.828655 1387 catalog_manager.cc:1372] Generated new certificate authority record
I20250901 08:00:52.829674 1387 catalog_manager.cc:1506] Loading token signing keys...
I20250901 08:00:52.845921 1387 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14: Generated new TSK 0
I20250901 08:00:52.846446 1387 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250901 08:00:52.870785 929 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250901 08:00:52.876317 1400 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:52.877694 1401 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:52.879144 1403 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250901 08:00:52.879642 929 server_base.cc:1047] running on GCE node
I20250901 08:00:52.880332 929 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250901 08:00:52.880492 929 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250901 08:00:52.880604 929 hybrid_clock.cc:648] HybridClock initialized: now 1756713652880593 us; error 0 us; skew 500 ppm
I20250901 08:00:52.881012 929 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250901 08:00:52.883144 929 webserver.cc:480] Webserver started at http://127.0.232.65:37815/ using document root <none> and password file <none>
I20250901 08:00:52.883565 929 fs_manager.cc:362] Metadata directory not provided
I20250901 08:00:52.883714 929 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250901 08:00:52.883900 929 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
I20250901 08:00:52.884871 929 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1756713648558816-929-0/minicluster-data/ts-0-root/instance:
uuid: "f4c8669d6001428a8c6d53c7f0c9c10f"
format_stamp: "Formatted at 2025-09-01 08:00:52 on dist-test-slave-v0s2"
I20250901 08:00:52.888823 929 fs_manager.cc:696] Time spent creating directory manager: real 0.003s user 0.005s sys 0.000s
I20250901 08:00:52.891615 1408 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:52.892266 929 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20250901 08:00:52.892504 929 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1756713648558816-929-0/minicluster-data/ts-0-root
uuid: "f4c8669d6001428a8c6d53c7f0c9c10f"
format_stamp: "Formatted at 2025-09-01 08:00:52 on dist-test-slave-v0s2"
I20250901 08:00:52.892728 929 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1756713648558816-929-0/minicluster-data/ts-0-root
metadata directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1756713648558816-929-0/minicluster-data/ts-0-root
1 data directories: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1756713648558816-929-0/minicluster-data/ts-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250901 08:00:52.901830 929 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250901 08:00:52.902827 929 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250901 08:00:52.904183 929 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20250901 08:00:52.906216 929 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20250901 08:00:52.906432 929 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:52.906659 929 ts_tablet_manager.cc:610] Registered 0 tablets
I20250901 08:00:52.906801 929 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:52.945178 929 rpc_server.cc:307] RPC server started. Bound to: 127.0.232.65:37007
I20250901 08:00:52.945356 1470 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.232.65:37007 every 8 connection(s)
I20250901 08:00:52.958022 1471 heartbeater.cc:344] Connected to a master server at 127.0.232.126:42783
I20250901 08:00:52.958392 1471 heartbeater.cc:461] Registering TS with master...
I20250901 08:00:52.959112 1471 heartbeater.cc:507] Master 127.0.232.126:42783 requested a full tablet report, sending...
I20250901 08:00:52.960975 1344 ts_manager.cc:194] Registered new tserver with Master: f4c8669d6001428a8c6d53c7f0c9c10f (127.0.232.65:37007)
I20250901 08:00:52.961232 929 internal_mini_cluster.cc:371] 1 TS(s) registered with all masters after 0.01295404s
I20250901 08:00:52.962694 1344 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.0.1:47366
I20250901 08:00:52.980310 929 catalog_manager.cc:2232] Servicing CreateTable request from internal request:
name: "test_table"
schema {
columns {
name: "key"
type: INT32
is_key: true
}
columns {
name: "int_val"
type: INT32
}
}
num_replicas: 1
owner: "default"
I20250901 08:00:52.989625 929 tablet_server.cc:178] TabletServer@127.0.232.65:0 shutting down...
I20250901 08:00:53.016290 929 ts_tablet_manager.cc:1500] Shutting down tablet manager...
I20250901 08:00:53.033908 929 tablet_server.cc:195] TabletServer@127.0.232.65:0 shutdown complete.
I20250901 08:00:53.040982 929 master.cc:561] Master@127.0.232.126:42783 shutting down...
I20250901 08:00:53.081403 929 raft_consensus.cc:2241] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:53.081861 929 raft_consensus.cc:2270] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:53.082330 929 tablet_replica.cc:331] T 00000000000000000000000000000000 P 80b6e6ac1f7d448e92ed90ff4a697c14: stopping tablet replica
I20250901 08:00:53.101543 929 master.cc:583] Master@127.0.232.126:42783 shutdown complete.
[ OK ] CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc (442 ms)
[ RUN ] CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser
I20250901 08:00:53.120826 929 internal_mini_cluster.cc:156] Creating distributed mini masters. Addrs: 127.0.232.126:46017
I20250901 08:00:53.121847 929 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250901 08:00:53.126195 1484 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:53.127506 1485 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250901 08:00:53.128473 929 server_base.cc:1047] running on GCE node
W20250901 08:00:53.129190 1487 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250901 08:00:53.129956 929 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250901 08:00:53.130156 929 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250901 08:00:53.130385 929 hybrid_clock.cc:648] HybridClock initialized: now 1756713653130366 us; error 0 us; skew 500 ppm
I20250901 08:00:53.130903 929 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250901 08:00:53.133106 929 webserver.cc:480] Webserver started at http://127.0.232.126:37997/ using document root <none> and password file <none>
I20250901 08:00:53.133589 929 fs_manager.cc:362] Metadata directory not provided
I20250901 08:00:53.133767 929 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250901 08:00:53.134038 929 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
I20250901 08:00:53.135151 929 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1756713648558816-929-0/minicluster-data/master-0-root/instance:
uuid: "2ea244fc9015430aa30f5e011ed96e81"
format_stamp: "Formatted at 2025-09-01 08:00:53 on dist-test-slave-v0s2"
I20250901 08:00:53.139236 929 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.004s sys 0.000s
I20250901 08:00:53.142381 1492 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:53.143162 929 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
I20250901 08:00:53.143414 929 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1756713648558816-929-0/minicluster-data/master-0-root
uuid: "2ea244fc9015430aa30f5e011ed96e81"
format_stamp: "Formatted at 2025-09-01 08:00:53 on dist-test-slave-v0s2"
I20250901 08:00:53.143667 929 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1756713648558816-929-0/minicluster-data/master-0-root
metadata directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1756713648558816-929-0/minicluster-data/master-0-root
1 data directories: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1756713648558816-929-0/minicluster-data/master-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250901 08:00:53.152832 929 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250901 08:00:53.153609 929 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250901 08:00:53.187846 929 rpc_server.cc:307] RPC server started. Bound to: 127.0.232.126:46017
I20250901 08:00:53.187988 1543 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.232.126:46017 every 8 connection(s)
I20250901 08:00:53.191502 1544 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:53.202313 1544 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81: Bootstrap starting.
I20250901 08:00:53.206634 1544 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:53.210176 1544 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81: No bootstrap required, opened a new log
I20250901 08:00:53.211995 1544 raft_consensus.cc:357] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "2ea244fc9015430aa30f5e011ed96e81" member_type: VOTER }
I20250901 08:00:53.212409 1544 raft_consensus.cc:383] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:53.212599 1544 raft_consensus.cc:738] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ea244fc9015430aa30f5e011ed96e81, State: Initialized, Role: FOLLOWER
I20250901 08:00:53.213152 1544 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "2ea244fc9015430aa30f5e011ed96e81" member_type: VOTER }
I20250901 08:00:53.213577 1544 raft_consensus.cc:397] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:53.213776 1544 raft_consensus.cc:491] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:53.213986 1544 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:53.218513 1544 raft_consensus.cc:513] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "2ea244fc9015430aa30f5e011ed96e81" member_type: VOTER }
I20250901 08:00:53.218983 1544 leader_election.cc:304] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 2ea244fc9015430aa30f5e011ed96e81; no voters:
I20250901 08:00:53.220105 1544 leader_election.cc:290] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:53.220436 1547 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:53.221781 1547 raft_consensus.cc:695] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [term 1 LEADER]: Becoming Leader. State: Replica: 2ea244fc9015430aa30f5e011ed96e81, State: Running, Role: LEADER
I20250901 08:00:53.222493 1547 consensus_queue.cc:237] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "2ea244fc9015430aa30f5e011ed96e81" member_type: VOTER }
I20250901 08:00:53.223234 1544 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [sys.catalog]: configured and running, proceeding with master startup.
I20250901 08:00:53.225481 1548 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "2ea244fc9015430aa30f5e011ed96e81" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "2ea244fc9015430aa30f5e011ed96e81" member_type: VOTER } }
I20250901 08:00:53.225574 1549 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2ea244fc9015430aa30f5e011ed96e81. Latest consensus state: current_term: 1 leader_uuid: "2ea244fc9015430aa30f5e011ed96e81" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "2ea244fc9015430aa30f5e011ed96e81" member_type: VOTER } }
I20250901 08:00:53.226261 1549 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [sys.catalog]: This master's current role is: LEADER
I20250901 08:00:53.226306 1548 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [sys.catalog]: This master's current role is: LEADER
I20250901 08:00:53.234704 1553 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250901 08:00:53.240280 1553 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250901 08:00:53.241201 929 internal_mini_cluster.cc:184] Waiting to initialize catalog manager on master 0
I20250901 08:00:53.248238 1553 catalog_manager.cc:1349] Generated new cluster ID: cd3f01d14ce547eeae392671859db0b9
I20250901 08:00:53.248529 1553 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250901 08:00:53.278594 1553 catalog_manager.cc:1372] Generated new certificate authority record
I20250901 08:00:53.279783 1553 catalog_manager.cc:1506] Loading token signing keys...
I20250901 08:00:53.293902 1553 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81: Generated new TSK 0
I20250901 08:00:53.294602 1553 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250901 08:00:53.307385 929 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250901 08:00:53.312701 1565 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:53.314070 1566 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250901 08:00:53.315912 929 server_base.cc:1047] running on GCE node
W20250901 08:00:53.316095 1568 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250901 08:00:53.317063 929 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250901 08:00:53.317243 929 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250901 08:00:53.317363 929 hybrid_clock.cc:648] HybridClock initialized: now 1756713653317352 us; error 0 us; skew 500 ppm
I20250901 08:00:53.317835 929 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250901 08:00:53.320132 929 webserver.cc:480] Webserver started at http://127.0.232.65:44061/ using document root <none> and password file <none>
I20250901 08:00:53.320600 929 fs_manager.cc:362] Metadata directory not provided
I20250901 08:00:53.320755 929 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250901 08:00:53.320959 929 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
I20250901 08:00:53.321949 929 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1756713648558816-929-0/minicluster-data/ts-0-root/instance:
uuid: "6e29497107d84914a848007a3e9b267f"
format_stamp: "Formatted at 2025-09-01 08:00:53 on dist-test-slave-v0s2"
I20250901 08:00:53.326855 929 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.003s sys 0.001s
I20250901 08:00:53.329994 1573 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:53.330782 929 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
I20250901 08:00:53.331056 929 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1756713648558816-929-0/minicluster-data/ts-0-root
uuid: "6e29497107d84914a848007a3e9b267f"
format_stamp: "Formatted at 2025-09-01 08:00:53 on dist-test-slave-v0s2"
I20250901 08:00:53.331326 929 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1756713648558816-929-0/minicluster-data/ts-0-root
metadata directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1756713648558816-929-0/minicluster-data/ts-0-root
1 data directories: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1756713648558816-929-0/minicluster-data/ts-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250901 08:00:53.346297 929 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250901 08:00:53.347206 929 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250901 08:00:53.348480 929 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20250901 08:00:53.350495 929 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20250901 08:00:53.350656 929 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:53.350885 929 ts_tablet_manager.cc:610] Registered 0 tablets
I20250901 08:00:53.351014 929 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:53.388528 929 rpc_server.cc:307] RPC server started. Bound to: 127.0.232.65:42811
I20250901 08:00:53.388669 1635 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.232.65:42811 every 8 connection(s)
I20250901 08:00:53.402343 1636 heartbeater.cc:344] Connected to a master server at 127.0.232.126:46017
I20250901 08:00:53.402730 1636 heartbeater.cc:461] Registering TS with master...
I20250901 08:00:53.403391 1636 heartbeater.cc:507] Master 127.0.232.126:46017 requested a full tablet report, sending...
I20250901 08:00:53.405172 1509 ts_manager.cc:194] Registered new tserver with Master: 6e29497107d84914a848007a3e9b267f (127.0.232.65:42811)
I20250901 08:00:53.406095 929 internal_mini_cluster.cc:371] 1 TS(s) registered with all masters after 0.014300629s
I20250901 08:00:53.407009 1509 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.0.1:43616
I20250901 08:00:53.424569 929 catalog_manager.cc:2241] Servicing CreateTable request from test_user:
name: "test_table"
schema {
columns {
name: "key"
type: INT32
is_key: true
}
columns {
name: "int_val"
type: INT32
}
}
num_replicas: 1
owner: "default"
I20250901 08:00:53.433755 929 tablet_server.cc:178] TabletServer@127.0.232.65:0 shutting down...
I20250901 08:00:53.456828 929 ts_tablet_manager.cc:1500] Shutting down tablet manager...
I20250901 08:00:53.474965 929 tablet_server.cc:195] TabletServer@127.0.232.65:0 shutdown complete.
I20250901 08:00:53.481900 929 master.cc:561] Master@127.0.232.126:46017 shutting down...
W20250901 08:00:53.526062 1494 catalog_manager.cc:4671] TS 6e29497107d84914a848007a3e9b267f (127.0.232.65:42811): CreateTablet RPC failed for tablet 6011fd4e912141b6a8bb078fdbc43dba: Network error: Client connection negotiation failed: client connection to 127.0.232.65:42811: connect: Connection refused (error 111)
I20250901 08:00:53.533607 929 raft_consensus.cc:2241] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:53.534216 929 raft_consensus.cc:2270] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:53.534750 929 tablet_replica.cc:331] T 00000000000000000000000000000000 P 2ea244fc9015430aa30f5e011ed96e81: stopping tablet replica
I20250901 08:00:53.549121 929 master.cc:583] Master@127.0.232.126:46017 shutdown complete.
[ OK ] CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser (447 ms)
[ RUN ] CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc
I20250901 08:00:53.567647 929 internal_mini_cluster.cc:156] Creating distributed mini masters. Addrs: 127.0.232.126:44127
I20250901 08:00:53.568713 929 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250901 08:00:53.573446 1649 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:53.573951 1650 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:53.574990 1652 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250901 08:00:53.576169 929 server_base.cc:1047] running on GCE node
I20250901 08:00:53.576808 929 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250901 08:00:53.576993 929 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250901 08:00:53.577142 929 hybrid_clock.cc:648] HybridClock initialized: now 1756713653577124 us; error 0 us; skew 500 ppm
I20250901 08:00:53.577744 929 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250901 08:00:53.580152 929 webserver.cc:480] Webserver started at http://127.0.232.126:40003/ using document root <none> and password file <none>
I20250901 08:00:53.580619 929 fs_manager.cc:362] Metadata directory not provided
I20250901 08:00:53.580796 929 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250901 08:00:53.581046 929 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
I20250901 08:00:53.582007 929 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1756713648558816-929-0/minicluster-data/master-0-root/instance:
uuid: "7c4cba8999f940f2b954361943d196fb"
format_stamp: "Formatted at 2025-09-01 08:00:53 on dist-test-slave-v0s2"
I20250901 08:00:53.586254 929 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.003s sys 0.000s
I20250901 08:00:53.589336 1657 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:53.590099 929 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.000s sys 0.003s
I20250901 08:00:53.590395 929 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1756713648558816-929-0/minicluster-data/master-0-root
uuid: "7c4cba8999f940f2b954361943d196fb"
format_stamp: "Formatted at 2025-09-01 08:00:53 on dist-test-slave-v0s2"
I20250901 08:00:53.590662 929 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1756713648558816-929-0/minicluster-data/master-0-root
metadata directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1756713648558816-929-0/minicluster-data/master-0-root
1 data directories: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1756713648558816-929-0/minicluster-data/master-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250901 08:00:53.602094 929 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250901 08:00:53.603133 929 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250901 08:00:53.642119 929 rpc_server.cc:307] RPC server started. Bound to: 127.0.232.126:44127
I20250901 08:00:53.642205 1708 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.232.126:44127 every 8 connection(s)
I20250901 08:00:53.646484 1709 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:53.655627 1709 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb: Bootstrap starting.
I20250901 08:00:53.659102 1709 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:53.662884 1709 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb: No bootstrap required, opened a new log
I20250901 08:00:53.664609 1709 raft_consensus.cc:357] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "7c4cba8999f940f2b954361943d196fb" member_type: VOTER }
I20250901 08:00:53.664952 1709 raft_consensus.cc:383] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:53.665189 1709 raft_consensus.cc:738] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7c4cba8999f940f2b954361943d196fb, State: Initialized, Role: FOLLOWER
I20250901 08:00:53.665820 1709 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "7c4cba8999f940f2b954361943d196fb" member_type: VOTER }
I20250901 08:00:53.666204 1709 raft_consensus.cc:397] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:53.666455 1709 raft_consensus.cc:491] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:53.666702 1709 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:53.670338 1709 raft_consensus.cc:513] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "7c4cba8999f940f2b954361943d196fb" member_type: VOTER }
I20250901 08:00:53.670821 1709 leader_election.cc:304] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 7c4cba8999f940f2b954361943d196fb; no voters:
I20250901 08:00:53.671914 1709 leader_election.cc:290] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:53.672303 1712 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:53.673731 1712 raft_consensus.cc:695] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [term 1 LEADER]: Becoming Leader. State: Replica: 7c4cba8999f940f2b954361943d196fb, State: Running, Role: LEADER
I20250901 08:00:53.674510 1712 consensus_queue.cc:237] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "7c4cba8999f940f2b954361943d196fb" member_type: VOTER }
I20250901 08:00:53.674760 1709 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [sys.catalog]: configured and running, proceeding with master startup.
I20250901 08:00:53.677306 1714 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7c4cba8999f940f2b954361943d196fb. Latest consensus state: current_term: 1 leader_uuid: "7c4cba8999f940f2b954361943d196fb" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "7c4cba8999f940f2b954361943d196fb" member_type: VOTER } }
I20250901 08:00:53.677273 1713 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "7c4cba8999f940f2b954361943d196fb" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "7c4cba8999f940f2b954361943d196fb" member_type: VOTER } }
I20250901 08:00:53.677904 1713 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [sys.catalog]: This master's current role is: LEADER
I20250901 08:00:53.677892 1714 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [sys.catalog]: This master's current role is: LEADER
I20250901 08:00:53.685173 1717 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250901 08:00:53.689761 1717 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250901 08:00:53.692044 929 internal_mini_cluster.cc:184] Waiting to initialize catalog manager on master 0
I20250901 08:00:53.698158 1717 catalog_manager.cc:1349] Generated new cluster ID: ee358180070b4c229dca1a55f608b7a1
I20250901 08:00:53.698446 1717 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250901 08:00:53.713397 1717 catalog_manager.cc:1372] Generated new certificate authority record
I20250901 08:00:53.714708 1717 catalog_manager.cc:1506] Loading token signing keys...
I20250901 08:00:53.728919 1717 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb: Generated new TSK 0
I20250901 08:00:53.729447 1717 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250901 08:00:53.757869 929 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250901 08:00:53.763388 1730 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:53.764499 1731 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:53.766350 1733 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250901 08:00:53.767699 929 server_base.cc:1047] running on GCE node
I20250901 08:00:53.768507 929 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250901 08:00:53.768690 929 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250901 08:00:53.768841 929 hybrid_clock.cc:648] HybridClock initialized: now 1756713653768826 us; error 0 us; skew 500 ppm
I20250901 08:00:53.769354 929 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250901 08:00:53.771425 929 webserver.cc:480] Webserver started at http://127.0.232.65:44561/ using document root <none> and password file <none>
I20250901 08:00:53.771854 929 fs_manager.cc:362] Metadata directory not provided
I20250901 08:00:53.772025 929 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250901 08:00:53.772255 929 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
I20250901 08:00:53.773211 929 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1756713648558816-929-0/minicluster-data/ts-0-root/instance:
uuid: "e157df0541da4d51b137423c6059db7f"
format_stamp: "Formatted at 2025-09-01 08:00:53 on dist-test-slave-v0s2"
I20250901 08:00:53.777472 929 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.002s sys 0.004s
I20250901 08:00:53.780388 1738 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:53.781049 929 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20250901 08:00:53.781282 929 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1756713648558816-929-0/minicluster-data/ts-0-root
uuid: "e157df0541da4d51b137423c6059db7f"
format_stamp: "Formatted at 2025-09-01 08:00:53 on dist-test-slave-v0s2"
I20250901 08:00:53.781514 929 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1756713648558816-929-0/minicluster-data/ts-0-root
metadata directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1756713648558816-929-0/minicluster-data/ts-0-root
1 data directories: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1756713648558816-929-0/minicluster-data/ts-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250901 08:00:53.792244 929 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250901 08:00:53.793044 929 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250901 08:00:53.794421 929 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20250901 08:00:53.796526 929 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20250901 08:00:53.796694 929 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:53.796895 929 ts_tablet_manager.cc:610] Registered 0 tablets
I20250901 08:00:53.797032 929 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:53.835804 929 rpc_server.cc:307] RPC server started. Bound to: 127.0.232.65:39625
I20250901 08:00:53.835952 1800 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.232.65:39625 every 8 connection(s)
I20250901 08:00:53.850481 1801 heartbeater.cc:344] Connected to a master server at 127.0.232.126:44127
I20250901 08:00:53.850901 1801 heartbeater.cc:461] Registering TS with master...
I20250901 08:00:53.851572 1801 heartbeater.cc:507] Master 127.0.232.126:44127 requested a full tablet report, sending...
I20250901 08:00:53.853320 1674 ts_manager.cc:194] Registered new tserver with Master: e157df0541da4d51b137423c6059db7f (127.0.232.65:39625)
I20250901 08:00:53.854243 929 internal_mini_cluster.cc:371] 1 TS(s) registered with all masters after 0.01511366s
I20250901 08:00:53.854964 1674 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.0.1:48074
I20250901 08:00:53.874096 1674 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:48078:
name: "test_table"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
rows: "\004\001d\000\000\000\004\001\310\000\000\000\004\001,\001\000\000\004\001\220\001\000\000\004\001\364\001\000\000\004\001X\002\000\000\004\001\274\002\000\000\004\001 \003\000\000\004\001\204\003\000\000""\004\001d\000\000\000\004\001\310\000\000\000\004\001,\001\000\000\004\001\220\001\000\000\004\001\364\001\000\000\004\001X\002\000\000\004\001\274\002\000\000\004\001 \003\000\000\004\001\204\003\000\000"
indirect_data: """"
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
I20250901 08:00:53.915575 1766 tablet_service.cc:1468] Processing CreateTablet for tablet 4a0fdf28f1574910a568dd2ac7fb3ef9 (DEFAULT_TABLE table=test_table [id=4283360be0a44ad5ae6f37a7f4c356b3]), partition=RANGE (key) PARTITION VALUES < 100
I20250901 08:00:53.915977 1764 tablet_service.cc:1468] Processing CreateTablet for tablet d10c71c0a9554bb5a9eeffacf223f63f (DEFAULT_TABLE table=test_table [id=4283360be0a44ad5ae6f37a7f4c356b3]), partition=RANGE (key) PARTITION 200 <= VALUES < 300
I20250901 08:00:53.916770 1766 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4a0fdf28f1574910a568dd2ac7fb3ef9. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:53.916873 1763 tablet_service.cc:1468] Processing CreateTablet for tablet 07afa93082024fb698c19ff882507382 (DEFAULT_TABLE table=test_table [id=4283360be0a44ad5ae6f37a7f4c356b3]), partition=RANGE (key) PARTITION 300 <= VALUES < 400
I20250901 08:00:53.917333 1764 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d10c71c0a9554bb5a9eeffacf223f63f. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:53.917965 1763 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 07afa93082024fb698c19ff882507382. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:53.918510 1757 tablet_service.cc:1468] Processing CreateTablet for tablet ddc2ff1faf09480daa1cfbf5e61ae810 (DEFAULT_TABLE table=test_table [id=4283360be0a44ad5ae6f37a7f4c356b3]), partition=RANGE (key) PARTITION 900 <= VALUES
I20250901 08:00:53.918936 1758 tablet_service.cc:1468] Processing CreateTablet for tablet a75b3bb2c22c44d99e419ee594c3e8af (DEFAULT_TABLE table=test_table [id=4283360be0a44ad5ae6f37a7f4c356b3]), partition=RANGE (key) PARTITION 800 <= VALUES < 900
I20250901 08:00:53.919106 1762 tablet_service.cc:1468] Processing CreateTablet for tablet ca287cba715e442a951e43be1e1a2016 (DEFAULT_TABLE table=test_table [id=4283360be0a44ad5ae6f37a7f4c356b3]), partition=RANGE (key) PARTITION 400 <= VALUES < 500
I20250901 08:00:53.919595 1757 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ddc2ff1faf09480daa1cfbf5e61ae810. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:53.920202 1758 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a75b3bb2c22c44d99e419ee594c3e8af. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:53.920753 1761 tablet_service.cc:1468] Processing CreateTablet for tablet 0e27b40a10d9465d9a326582fef7ea9c (DEFAULT_TABLE table=test_table [id=4283360be0a44ad5ae6f37a7f4c356b3]), partition=RANGE (key) PARTITION 500 <= VALUES < 600
I20250901 08:00:53.921144 1759 tablet_service.cc:1468] Processing CreateTablet for tablet 72ffe2169403432183465bfc8b6211a7 (DEFAULT_TABLE table=test_table [id=4283360be0a44ad5ae6f37a7f4c356b3]), partition=RANGE (key) PARTITION 700 <= VALUES < 800
I20250901 08:00:53.921787 1761 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0e27b40a10d9465d9a326582fef7ea9c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:53.922839 1760 tablet_service.cc:1468] Processing CreateTablet for tablet c6702adb68ff43ed9c9303ad36cf0709 (DEFAULT_TABLE table=test_table [id=4283360be0a44ad5ae6f37a7f4c356b3]), partition=RANGE (key) PARTITION 600 <= VALUES < 700
I20250901 08:00:53.923964 1760 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c6702adb68ff43ed9c9303ad36cf0709. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:53.915724 1765 tablet_service.cc:1468] Processing CreateTablet for tablet ffdd04e5a86c459990f902ba33d16607 (DEFAULT_TABLE table=test_table [id=4283360be0a44ad5ae6f37a7f4c356b3]), partition=RANGE (key) PARTITION 100 <= VALUES < 200
I20250901 08:00:53.929483 1762 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ca287cba715e442a951e43be1e1a2016. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:53.930656 1759 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 72ffe2169403432183465bfc8b6211a7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:53.932871 1765 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ffdd04e5a86c459990f902ba33d16607. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:53.956347 1813 tablet_bootstrap.cc:492] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f: Bootstrap starting.
I20250901 08:00:53.960572 1813 tablet_bootstrap.cc:654] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:53.964334 1813 tablet_bootstrap.cc:492] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f: No bootstrap required, opened a new log
I20250901 08:00:53.964666 1813 ts_tablet_manager.cc:1397] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f: Time spent bootstrapping tablet: real 0.009s user 0.000s sys 0.007s
I20250901 08:00:53.966216 1813 raft_consensus.cc:357] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:53.966611 1813 raft_consensus.cc:383] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:53.966771 1813 raft_consensus.cc:738] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e157df0541da4d51b137423c6059db7f, State: Initialized, Role: FOLLOWER
I20250901 08:00:53.967142 1813 consensus_queue.cc:260] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:53.967514 1813 raft_consensus.cc:397] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:53.967674 1813 raft_consensus.cc:491] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:53.967846 1813 raft_consensus.cc:3058] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:53.971335 1813 raft_consensus.cc:513] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:53.971769 1813 leader_election.cc:304] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e157df0541da4d51b137423c6059db7f; no voters:
I20250901 08:00:53.972709 1813 leader_election.cc:290] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:53.972966 1815 raft_consensus.cc:2802] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:53.974319 1801 heartbeater.cc:499] Master 127.0.232.126:44127 was elected leader, sending a full tablet report...
I20250901 08:00:53.974797 1813 ts_tablet_manager.cc:1428] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f: Time spent starting tablet: real 0.010s user 0.009s sys 0.002s
I20250901 08:00:53.974267 1815 raft_consensus.cc:695] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Becoming Leader. State: Replica: e157df0541da4d51b137423c6059db7f, State: Running, Role: LEADER
I20250901 08:00:53.975461 1813 tablet_bootstrap.cc:492] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f: Bootstrap starting.
I20250901 08:00:53.975675 1815 consensus_queue.cc:237] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:53.980085 1813 tablet_bootstrap.cc:654] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:53.984253 1813 tablet_bootstrap.cc:492] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f: No bootstrap required, opened a new log
I20250901 08:00:53.984664 1813 ts_tablet_manager.cc:1397] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f: Time spent bootstrapping tablet: real 0.009s user 0.007s sys 0.000s
I20250901 08:00:53.986970 1813 raft_consensus.cc:357] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:53.987468 1813 raft_consensus.cc:383] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:53.987733 1813 raft_consensus.cc:738] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e157df0541da4d51b137423c6059db7f, State: Initialized, Role: FOLLOWER
I20250901 08:00:53.987696 1674 catalog_manager.cc:5582] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f reported cstate change: term changed from 0 to 1, leader changed from <none> to e157df0541da4d51b137423c6059db7f (127.0.232.65). New cstate: current_term: 1 leader_uuid: "e157df0541da4d51b137423c6059db7f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:53.988360 1813 consensus_queue.cc:260] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:53.988973 1813 raft_consensus.cc:397] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:53.989281 1813 raft_consensus.cc:491] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:53.989667 1813 raft_consensus.cc:3058] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:53.994532 1813 raft_consensus.cc:513] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:53.995183 1813 leader_election.cc:304] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e157df0541da4d51b137423c6059db7f; no voters:
I20250901 08:00:53.995782 1813 leader_election.cc:290] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:53.995925 1817 raft_consensus.cc:2802] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:53.996349 1817 raft_consensus.cc:695] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Becoming Leader. State: Replica: e157df0541da4d51b137423c6059db7f, State: Running, Role: LEADER
I20250901 08:00:53.996948 1817 consensus_queue.cc:237] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:53.997532 1813 ts_tablet_manager.cc:1428] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f: Time spent starting tablet: real 0.013s user 0.010s sys 0.003s
I20250901 08:00:53.998440 1813 tablet_bootstrap.cc:492] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f: Bootstrap starting.
I20250901 08:00:54.003086 1813 tablet_bootstrap.cc:654] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:54.005761 1674 catalog_manager.cc:5582] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f reported cstate change: term changed from 0 to 1, leader changed from <none> to e157df0541da4d51b137423c6059db7f (127.0.232.65). New cstate: current_term: 1 leader_uuid: "e157df0541da4d51b137423c6059db7f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:54.007341 1813 tablet_bootstrap.cc:492] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f: No bootstrap required, opened a new log
I20250901 08:00:54.007735 1813 ts_tablet_manager.cc:1397] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f: Time spent bootstrapping tablet: real 0.010s user 0.008s sys 0.000s
I20250901 08:00:54.009402 1813 raft_consensus.cc:357] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.009771 1813 raft_consensus.cc:383] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:54.009974 1813 raft_consensus.cc:738] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e157df0541da4d51b137423c6059db7f, State: Initialized, Role: FOLLOWER
I20250901 08:00:54.010474 1813 consensus_queue.cc:260] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.011077 1813 raft_consensus.cc:397] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:54.011358 1813 raft_consensus.cc:491] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:54.011691 1813 raft_consensus.cc:3058] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:54.016247 1813 raft_consensus.cc:513] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.016634 1813 leader_election.cc:304] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e157df0541da4d51b137423c6059db7f; no voters:
I20250901 08:00:54.016983 1813 leader_election.cc:290] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:54.017089 1817 raft_consensus.cc:2802] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:54.017472 1817 raft_consensus.cc:695] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Becoming Leader. State: Replica: e157df0541da4d51b137423c6059db7f, State: Running, Role: LEADER
I20250901 08:00:54.018187 1813 ts_tablet_manager.cc:1428] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f: Time spent starting tablet: real 0.010s user 0.008s sys 0.000s
I20250901 08:00:54.018025 1817 consensus_queue.cc:237] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.018816 1813 tablet_bootstrap.cc:492] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f: Bootstrap starting.
I20250901 08:00:54.023470 1674 catalog_manager.cc:5582] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f reported cstate change: term changed from 0 to 1, leader changed from <none> to e157df0541da4d51b137423c6059db7f (127.0.232.65). New cstate: current_term: 1 leader_uuid: "e157df0541da4d51b137423c6059db7f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:54.024302 1813 tablet_bootstrap.cc:654] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:54.028009 1813 tablet_bootstrap.cc:492] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f: No bootstrap required, opened a new log
I20250901 08:00:54.028352 1813 ts_tablet_manager.cc:1397] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f: Time spent bootstrapping tablet: real 0.010s user 0.005s sys 0.003s
I20250901 08:00:54.030206 1813 raft_consensus.cc:357] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.030586 1813 raft_consensus.cc:383] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:54.030781 1813 raft_consensus.cc:738] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e157df0541da4d51b137423c6059db7f, State: Initialized, Role: FOLLOWER
I20250901 08:00:54.031219 1813 consensus_queue.cc:260] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.031613 1813 raft_consensus.cc:397] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:54.031788 1813 raft_consensus.cc:491] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:54.032011 1813 raft_consensus.cc:3058] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:54.036090 1813 raft_consensus.cc:513] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.036532 1813 leader_election.cc:304] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e157df0541da4d51b137423c6059db7f; no voters:
I20250901 08:00:54.036931 1813 leader_election.cc:290] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:54.037081 1817 raft_consensus.cc:2802] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:54.037492 1817 raft_consensus.cc:695] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Becoming Leader. State: Replica: e157df0541da4d51b137423c6059db7f, State: Running, Role: LEADER
I20250901 08:00:54.038195 1813 ts_tablet_manager.cc:1428] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f: Time spent starting tablet: real 0.010s user 0.003s sys 0.004s
I20250901 08:00:54.038038 1817 consensus_queue.cc:237] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.038802 1813 tablet_bootstrap.cc:492] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f: Bootstrap starting.
I20250901 08:00:54.042994 1813 tablet_bootstrap.cc:654] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:54.047250 1674 catalog_manager.cc:5582] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f reported cstate change: term changed from 0 to 1, leader changed from <none> to e157df0541da4d51b137423c6059db7f (127.0.232.65). New cstate: current_term: 1 leader_uuid: "e157df0541da4d51b137423c6059db7f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:54.047845 1813 tablet_bootstrap.cc:492] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f: No bootstrap required, opened a new log
I20250901 08:00:54.048156 1813 ts_tablet_manager.cc:1397] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f: Time spent bootstrapping tablet: real 0.009s user 0.007s sys 0.001s
I20250901 08:00:54.050134 1813 raft_consensus.cc:357] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.050616 1813 raft_consensus.cc:383] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:54.050863 1813 raft_consensus.cc:738] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e157df0541da4d51b137423c6059db7f, State: Initialized, Role: FOLLOWER
I20250901 08:00:54.051348 1813 consensus_queue.cc:260] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.051748 1813 raft_consensus.cc:397] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:54.051997 1813 raft_consensus.cc:491] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:54.052309 1813 raft_consensus.cc:3058] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:54.056864 1813 raft_consensus.cc:513] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.057286 1813 leader_election.cc:304] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e157df0541da4d51b137423c6059db7f; no voters:
I20250901 08:00:54.057691 1813 leader_election.cc:290] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:54.057825 1817 raft_consensus.cc:2802] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:54.058544 1817 raft_consensus.cc:695] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Becoming Leader. State: Replica: e157df0541da4d51b137423c6059db7f, State: Running, Role: LEADER
I20250901 08:00:54.059017 1813 ts_tablet_manager.cc:1428] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f: Time spent starting tablet: real 0.011s user 0.011s sys 0.000s
I20250901 08:00:54.059154 1817 consensus_queue.cc:237] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.059541 1813 tablet_bootstrap.cc:492] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f: Bootstrap starting.
I20250901 08:00:54.063748 1813 tablet_bootstrap.cc:654] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:54.064359 1674 catalog_manager.cc:5582] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f reported cstate change: term changed from 0 to 1, leader changed from <none> to e157df0541da4d51b137423c6059db7f (127.0.232.65). New cstate: current_term: 1 leader_uuid: "e157df0541da4d51b137423c6059db7f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:54.067379 1813 tablet_bootstrap.cc:492] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f: No bootstrap required, opened a new log
I20250901 08:00:54.067704 1813 ts_tablet_manager.cc:1397] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f: Time spent bootstrapping tablet: real 0.008s user 0.006s sys 0.000s
I20250901 08:00:54.069229 1813 raft_consensus.cc:357] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.069648 1813 raft_consensus.cc:383] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:54.069868 1813 raft_consensus.cc:738] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e157df0541da4d51b137423c6059db7f, State: Initialized, Role: FOLLOWER
I20250901 08:00:54.070487 1813 consensus_queue.cc:260] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.071049 1813 raft_consensus.cc:397] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:54.071319 1813 raft_consensus.cc:491] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:54.071630 1813 raft_consensus.cc:3058] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:54.076359 1813 raft_consensus.cc:513] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.076901 1813 leader_election.cc:304] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e157df0541da4d51b137423c6059db7f; no voters:
I20250901 08:00:54.077291 1813 leader_election.cc:290] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:54.077399 1817 raft_consensus.cc:2802] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:54.077780 1817 raft_consensus.cc:695] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Becoming Leader. State: Replica: e157df0541da4d51b137423c6059db7f, State: Running, Role: LEADER
I20250901 08:00:54.078510 1813 ts_tablet_manager.cc:1428] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f: Time spent starting tablet: real 0.011s user 0.012s sys 0.000s
I20250901 08:00:54.078495 1817 consensus_queue.cc:237] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.079277 1813 tablet_bootstrap.cc:492] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f: Bootstrap starting.
I20250901 08:00:54.082983 1813 tablet_bootstrap.cc:654] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:54.084669 1673 catalog_manager.cc:5582] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f reported cstate change: term changed from 0 to 1, leader changed from <none> to e157df0541da4d51b137423c6059db7f (127.0.232.65). New cstate: current_term: 1 leader_uuid: "e157df0541da4d51b137423c6059db7f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:54.086911 1813 tablet_bootstrap.cc:492] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f: No bootstrap required, opened a new log
I20250901 08:00:54.087206 1813 ts_tablet_manager.cc:1397] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f: Time spent bootstrapping tablet: real 0.008s user 0.007s sys 0.000s
I20250901 08:00:54.088683 1813 raft_consensus.cc:357] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.089120 1813 raft_consensus.cc:383] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:54.089318 1813 raft_consensus.cc:738] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e157df0541da4d51b137423c6059db7f, State: Initialized, Role: FOLLOWER
I20250901 08:00:54.089735 1813 consensus_queue.cc:260] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.090087 1813 raft_consensus.cc:397] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:54.090303 1813 raft_consensus.cc:491] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:54.090543 1813 raft_consensus.cc:3058] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:54.095760 1813 raft_consensus.cc:513] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.096300 1813 leader_election.cc:304] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e157df0541da4d51b137423c6059db7f; no voters:
I20250901 08:00:54.096776 1813 leader_election.cc:290] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:54.096897 1817 raft_consensus.cc:2802] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:54.097592 1817 raft_consensus.cc:695] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Becoming Leader. State: Replica: e157df0541da4d51b137423c6059db7f, State: Running, Role: LEADER
I20250901 08:00:54.098002 1813 ts_tablet_manager.cc:1428] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f: Time spent starting tablet: real 0.011s user 0.008s sys 0.000s
I20250901 08:00:54.098047 1817 consensus_queue.cc:237] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.098608 1813 tablet_bootstrap.cc:492] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f: Bootstrap starting.
I20250901 08:00:54.103343 1813 tablet_bootstrap.cc:654] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:54.103789 1673 catalog_manager.cc:5582] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f reported cstate change: term changed from 0 to 1, leader changed from <none> to e157df0541da4d51b137423c6059db7f (127.0.232.65). New cstate: current_term: 1 leader_uuid: "e157df0541da4d51b137423c6059db7f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:54.106899 1813 tablet_bootstrap.cc:492] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f: No bootstrap required, opened a new log
I20250901 08:00:54.107220 1813 ts_tablet_manager.cc:1397] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f: Time spent bootstrapping tablet: real 0.009s user 0.007s sys 0.000s
I20250901 08:00:54.108947 1813 raft_consensus.cc:357] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.109382 1813 raft_consensus.cc:383] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:54.109630 1813 raft_consensus.cc:738] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e157df0541da4d51b137423c6059db7f, State: Initialized, Role: FOLLOWER
I20250901 08:00:54.110318 1813 consensus_queue.cc:260] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.110757 1813 raft_consensus.cc:397] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:54.110949 1813 raft_consensus.cc:491] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:54.111260 1813 raft_consensus.cc:3058] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:54.115466 1813 raft_consensus.cc:513] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.115952 1813 leader_election.cc:304] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e157df0541da4d51b137423c6059db7f; no voters:
I20250901 08:00:54.116298 1813 leader_election.cc:290] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:54.116441 1817 raft_consensus.cc:2802] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:54.116819 1817 raft_consensus.cc:695] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Becoming Leader. State: Replica: e157df0541da4d51b137423c6059db7f, State: Running, Role: LEADER
I20250901 08:00:54.117480 1813 ts_tablet_manager.cc:1428] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f: Time spent starting tablet: real 0.010s user 0.008s sys 0.000s
I20250901 08:00:54.117329 1817 consensus_queue.cc:237] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.117987 1813 tablet_bootstrap.cc:492] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f: Bootstrap starting.
I20250901 08:00:54.121906 1813 tablet_bootstrap.cc:654] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:54.122794 1673 catalog_manager.cc:5582] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f reported cstate change: term changed from 0 to 1, leader changed from <none> to e157df0541da4d51b137423c6059db7f (127.0.232.65). New cstate: current_term: 1 leader_uuid: "e157df0541da4d51b137423c6059db7f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:54.125959 1813 tablet_bootstrap.cc:492] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f: No bootstrap required, opened a new log
I20250901 08:00:54.126318 1813 ts_tablet_manager.cc:1397] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f: Time spent bootstrapping tablet: real 0.008s user 0.008s sys 0.000s
I20250901 08:00:54.127908 1813 raft_consensus.cc:357] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.128201 1813 raft_consensus.cc:383] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:54.128389 1813 raft_consensus.cc:738] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e157df0541da4d51b137423c6059db7f, State: Initialized, Role: FOLLOWER
I20250901 08:00:54.128813 1813 consensus_queue.cc:260] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.129161 1813 raft_consensus.cc:397] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:54.129398 1813 raft_consensus.cc:491] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:54.129666 1813 raft_consensus.cc:3058] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:54.134544 1813 raft_consensus.cc:513] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.135149 1813 leader_election.cc:304] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e157df0541da4d51b137423c6059db7f; no voters:
I20250901 08:00:54.135701 1813 leader_election.cc:290] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:54.135820 1817 raft_consensus.cc:2802] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:54.136453 1817 raft_consensus.cc:695] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Becoming Leader. State: Replica: e157df0541da4d51b137423c6059db7f, State: Running, Role: LEADER
I20250901 08:00:54.137475 1813 ts_tablet_manager.cc:1428] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f: Time spent starting tablet: real 0.011s user 0.005s sys 0.003s
I20250901 08:00:54.138211 1813 tablet_bootstrap.cc:492] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f: Bootstrap starting.
I20250901 08:00:54.137908 1817 consensus_queue.cc:237] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.143163 1813 tablet_bootstrap.cc:654] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:54.144726 1673 catalog_manager.cc:5582] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f reported cstate change: term changed from 0 to 1, leader changed from <none> to e157df0541da4d51b137423c6059db7f (127.0.232.65). New cstate: current_term: 1 leader_uuid: "e157df0541da4d51b137423c6059db7f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:54.148046 1813 tablet_bootstrap.cc:492] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f: No bootstrap required, opened a new log
I20250901 08:00:54.148321 1813 ts_tablet_manager.cc:1397] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f: Time spent bootstrapping tablet: real 0.010s user 0.009s sys 0.000s
I20250901 08:00:54.149735 1813 raft_consensus.cc:357] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.150117 1813 raft_consensus.cc:383] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:54.150315 1813 raft_consensus.cc:738] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e157df0541da4d51b137423c6059db7f, State: Initialized, Role: FOLLOWER
I20250901 08:00:54.150746 1813 consensus_queue.cc:260] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.151219 1813 raft_consensus.cc:397] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:54.151468 1813 raft_consensus.cc:491] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:54.151719 1813 raft_consensus.cc:3058] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:54.157349 1813 raft_consensus.cc:513] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.157842 1813 leader_election.cc:304] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e157df0541da4d51b137423c6059db7f; no voters:
I20250901 08:00:54.158320 1813 leader_election.cc:290] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:54.158600 1817 raft_consensus.cc:2802] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:54.158955 1817 raft_consensus.cc:695] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Becoming Leader. State: Replica: e157df0541da4d51b137423c6059db7f, State: Running, Role: LEADER
I20250901 08:00:54.159680 1813 ts_tablet_manager.cc:1428] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f: Time spent starting tablet: real 0.011s user 0.011s sys 0.000s
I20250901 08:00:54.159469 1817 consensus_queue.cc:237] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } }
I20250901 08:00:54.166534 1673 catalog_manager.cc:5582] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f reported cstate change: term changed from 0 to 1, leader changed from <none> to e157df0541da4d51b137423c6059db7f (127.0.232.65). New cstate: current_term: 1 leader_uuid: "e157df0541da4d51b137423c6059db7f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e157df0541da4d51b137423c6059db7f" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 39625 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:54.215999 929 catalog_manager.cc:3502] Servicing AlterTable request from internal request:
table { table_name: "test_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "int_val2" type: INT32 is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } }
I20250901 08:00:54.254469 929 tablet_server.cc:178] TabletServer@127.0.232.65:0 shutting down...
I20250901 08:00:54.276083 1848 tablet.cc:1722] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:54.284324 1843 tablet.cc:1722] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:54.287091 1852 tablet.cc:1722] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:54.282128 1841 tablet.cc:1722] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:54.289649 1848 tablet.cc:1620] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f: MemRowSet was empty: no flush needed.
I20250901 08:00:54.289806 1843 tablet.cc:1620] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f: MemRowSet was empty: no flush needed.
I20250901 08:00:54.290143 1852 tablet.cc:1620] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f: MemRowSet was empty: no flush needed.
I20250901 08:00:54.290293 1841 tablet.cc:1620] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f: MemRowSet was empty: no flush needed.
I20250901 08:00:54.301070 1841 tablet.cc:1722] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:54.302757 1841 tablet.cc:1620] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f: MemRowSet was empty: no flush needed.
I20250901 08:00:54.302757 1852 tablet.cc:1722] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:54.304714 1852 tablet.cc:1620] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f: MemRowSet was empty: no flush needed.
I20250901 08:00:54.348692 1848 tablet.cc:1722] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:54.347412 1843 tablet.cc:1722] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:54.353615 1848 tablet.cc:1620] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f: MemRowSet was empty: no flush needed.
I20250901 08:00:54.355790 1843 tablet.cc:1620] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f: MemRowSet was empty: no flush needed.
I20250901 08:00:54.358566 1841 tablet.cc:1722] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:54.360181 1841 tablet.cc:1620] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f: MemRowSet was empty: no flush needed.
I20250901 08:00:54.364202 1852 tablet.cc:1722] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:54.365959 929 ts_tablet_manager.cc:1500] Shutting down tablet manager...
I20250901 08:00:54.365952 1852 tablet.cc:1620] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f: MemRowSet was empty: no flush needed.
I20250901 08:00:54.367779 929 tablet_replica.cc:331] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f: stopping tablet replica
I20250901 08:00:54.368319 929 raft_consensus.cc:2241] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:54.368641 929 raft_consensus.cc:2270] T c6702adb68ff43ed9c9303ad36cf0709 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:54.376286 929 tablet_replica.cc:331] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f: stopping tablet replica
I20250901 08:00:54.377005 929 raft_consensus.cc:2241] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:54.377481 929 raft_consensus.cc:2270] T 0e27b40a10d9465d9a326582fef7ea9c P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:54.379427 929 tablet_replica.cc:331] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f: stopping tablet replica
I20250901 08:00:54.379930 929 raft_consensus.cc:2241] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:54.380278 929 raft_consensus.cc:2270] T d10c71c0a9554bb5a9eeffacf223f63f P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:54.381944 929 tablet_replica.cc:331] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f: stopping tablet replica
I20250901 08:00:54.382481 929 raft_consensus.cc:2241] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:54.382839 929 raft_consensus.cc:2270] T 4a0fdf28f1574910a568dd2ac7fb3ef9 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:54.384673 929 tablet_replica.cc:331] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f: stopping tablet replica
I20250901 08:00:54.385124 929 raft_consensus.cc:2241] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:54.385488 929 raft_consensus.cc:2270] T 72ffe2169403432183465bfc8b6211a7 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:54.387281 929 tablet_replica.cc:331] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f: stopping tablet replica
I20250901 08:00:54.387725 929 raft_consensus.cc:2241] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:54.388020 929 raft_consensus.cc:2270] T 07afa93082024fb698c19ff882507382 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:54.389655 929 tablet_replica.cc:331] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f: stopping tablet replica
I20250901 08:00:54.390038 929 raft_consensus.cc:2241] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:54.390378 929 raft_consensus.cc:2270] T ffdd04e5a86c459990f902ba33d16607 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:54.392462 929 tablet_replica.cc:331] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f: stopping tablet replica
I20250901 08:00:54.392846 929 raft_consensus.cc:2241] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:54.393175 929 raft_consensus.cc:2270] T ca287cba715e442a951e43be1e1a2016 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:54.395032 929 tablet_replica.cc:331] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f: stopping tablet replica
I20250901 08:00:54.395438 929 raft_consensus.cc:2241] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:54.395742 929 raft_consensus.cc:2270] T a75b3bb2c22c44d99e419ee594c3e8af P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:54.397226 929 tablet_replica.cc:331] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f: stopping tablet replica
I20250901 08:00:54.397594 929 raft_consensus.cc:2241] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:54.397886 929 raft_consensus.cc:2270] T ddc2ff1faf09480daa1cfbf5e61ae810 P e157df0541da4d51b137423c6059db7f [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:54.415673 929 tablet_server.cc:195] TabletServer@127.0.232.65:0 shutdown complete.
I20250901 08:00:54.427512 929 master.cc:561] Master@127.0.232.126:44127 shutting down...
I20250901 08:00:54.440189 929 raft_consensus.cc:2241] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:54.440572 929 raft_consensus.cc:2270] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:54.440809 929 tablet_replica.cc:331] T 00000000000000000000000000000000 P 7c4cba8999f940f2b954361943d196fb: stopping tablet replica
I20250901 08:00:54.458058 929 master.cc:583] Master@127.0.232.126:44127 shutdown complete.
[ OK ] CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc (911 ms)
[ RUN ] CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser
I20250901 08:00:54.479399 929 internal_mini_cluster.cc:156] Creating distributed mini masters. Addrs: 127.0.232.126:37115
I20250901 08:00:54.480319 929 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250901 08:00:54.484748 1854 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:54.485060 1855 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:54.486370 1857 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250901 08:00:54.487756 929 server_base.cc:1047] running on GCE node
I20250901 08:00:54.488660 929 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250901 08:00:54.488833 929 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250901 08:00:54.488991 929 hybrid_clock.cc:648] HybridClock initialized: now 1756713654488957 us; error 0 us; skew 500 ppm
I20250901 08:00:54.489465 929 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250901 08:00:54.491649 929 webserver.cc:480] Webserver started at http://127.0.232.126:40299/ using document root <none> and password file <none>
I20250901 08:00:54.492102 929 fs_manager.cc:362] Metadata directory not provided
I20250901 08:00:54.492269 929 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250901 08:00:54.492528 929 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
I20250901 08:00:54.493490 929 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1756713648558816-929-0/minicluster-data/master-0-root/instance:
uuid: "861b7645c49a40b08c5d6735d979f241"
format_stamp: "Formatted at 2025-09-01 08:00:54 on dist-test-slave-v0s2"
I20250901 08:00:54.497481 929 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.000s sys 0.005s
I20250901 08:00:54.500331 1862 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:54.500989 929 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20250901 08:00:54.501235 929 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1756713648558816-929-0/minicluster-data/master-0-root
uuid: "861b7645c49a40b08c5d6735d979f241"
format_stamp: "Formatted at 2025-09-01 08:00:54 on dist-test-slave-v0s2"
I20250901 08:00:54.501479 929 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1756713648558816-929-0/minicluster-data/master-0-root
metadata directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1756713648558816-929-0/minicluster-data/master-0-root
1 data directories: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1756713648558816-929-0/minicluster-data/master-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250901 08:00:54.514535 929 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250901 08:00:54.515403 929 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250901 08:00:54.547832 929 rpc_server.cc:307] RPC server started. Bound to: 127.0.232.126:37115
I20250901 08:00:54.547930 1913 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.232.126:37115 every 8 connection(s)
I20250901 08:00:54.551329 1914 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:54.560396 1914 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241: Bootstrap starting.
I20250901 08:00:54.564136 1914 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:54.567507 1914 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241: No bootstrap required, opened a new log
I20250901 08:00:54.569221 1914 raft_consensus.cc:357] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "861b7645c49a40b08c5d6735d979f241" member_type: VOTER }
I20250901 08:00:54.569547 1914 raft_consensus.cc:383] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:54.569725 1914 raft_consensus.cc:738] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 861b7645c49a40b08c5d6735d979f241, State: Initialized, Role: FOLLOWER
I20250901 08:00:54.570165 1914 consensus_queue.cc:260] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "861b7645c49a40b08c5d6735d979f241" member_type: VOTER }
I20250901 08:00:54.570573 1914 raft_consensus.cc:397] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:54.570772 1914 raft_consensus.cc:491] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:54.570971 1914 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:54.574746 1914 raft_consensus.cc:513] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "861b7645c49a40b08c5d6735d979f241" member_type: VOTER }
I20250901 08:00:54.575165 1914 leader_election.cc:304] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 861b7645c49a40b08c5d6735d979f241; no voters:
I20250901 08:00:54.576112 1914 leader_election.cc:290] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:54.576434 1917 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:54.577622 1917 raft_consensus.cc:695] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [term 1 LEADER]: Becoming Leader. State: Replica: 861b7645c49a40b08c5d6735d979f241, State: Running, Role: LEADER
I20250901 08:00:54.578117 1917 consensus_queue.cc:237] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "861b7645c49a40b08c5d6735d979f241" member_type: VOTER }
I20250901 08:00:54.578797 1914 sys_catalog.cc:564] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [sys.catalog]: configured and running, proceeding with master startup.
I20250901 08:00:54.582993 1918 sys_catalog.cc:455] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "861b7645c49a40b08c5d6735d979f241" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "861b7645c49a40b08c5d6735d979f241" member_type: VOTER } }
I20250901 08:00:54.583146 1919 sys_catalog.cc:455] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 861b7645c49a40b08c5d6735d979f241. Latest consensus state: current_term: 1 leader_uuid: "861b7645c49a40b08c5d6735d979f241" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "861b7645c49a40b08c5d6735d979f241" member_type: VOTER } }
I20250901 08:00:54.583635 1918 sys_catalog.cc:458] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [sys.catalog]: This master's current role is: LEADER
I20250901 08:00:54.583811 1919 sys_catalog.cc:458] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [sys.catalog]: This master's current role is: LEADER
I20250901 08:00:54.585541 1926 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250901 08:00:54.591096 1926 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250901 08:00:54.591995 929 internal_mini_cluster.cc:184] Waiting to initialize catalog manager on master 0
I20250901 08:00:54.598299 1926 catalog_manager.cc:1349] Generated new cluster ID: e3d82ccccf594c14a6e512cb1354da89
I20250901 08:00:54.598539 1926 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250901 08:00:54.608947 1926 catalog_manager.cc:1372] Generated new certificate authority record
I20250901 08:00:54.609983 1926 catalog_manager.cc:1506] Loading token signing keys...
I20250901 08:00:54.625463 1926 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241: Generated new TSK 0
I20250901 08:00:54.625970 1926 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250901 08:00:54.657708 929 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250901 08:00:54.662647 1935 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:54.663622 1936 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250901 08:00:54.665174 1938 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250901 08:00:54.665783 929 server_base.cc:1047] running on GCE node
I20250901 08:00:54.666443 929 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250901 08:00:54.666604 929 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250901 08:00:54.666759 929 hybrid_clock.cc:648] HybridClock initialized: now 1756713654666739 us; error 0 us; skew 500 ppm
I20250901 08:00:54.667294 929 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250901 08:00:54.669342 929 webserver.cc:480] Webserver started at http://127.0.232.65:34077/ using document root <none> and password file <none>
I20250901 08:00:54.669787 929 fs_manager.cc:362] Metadata directory not provided
I20250901 08:00:54.669943 929 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250901 08:00:54.670176 929 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
I20250901 08:00:54.671172 929 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1756713648558816-929-0/minicluster-data/ts-0-root/instance:
uuid: "6ca7c48cfba74c07a24e0e06ecf22132"
format_stamp: "Formatted at 2025-09-01 08:00:54 on dist-test-slave-v0s2"
I20250901 08:00:54.675046 929 fs_manager.cc:696] Time spent creating directory manager: real 0.003s user 0.003s sys 0.001s
I20250901 08:00:54.677798 1943 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:54.678457 929 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
I20250901 08:00:54.678699 929 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1756713648558816-929-0/minicluster-data/ts-0-root
uuid: "6ca7c48cfba74c07a24e0e06ecf22132"
format_stamp: "Formatted at 2025-09-01 08:00:54 on dist-test-slave-v0s2"
I20250901 08:00:54.678915 929 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1756713648558816-929-0/minicluster-data/ts-0-root
metadata directory: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1756713648558816-929-0/minicluster-data/ts-0-root
1 data directories: /tmp/dist-test-task0CfAQn/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1756713648558816-929-0/minicluster-data/ts-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250901 08:00:54.690778 929 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250901 08:00:54.691599 929 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250901 08:00:54.692845 929 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20250901 08:00:54.694808 929 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20250901 08:00:54.694990 929 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:54.695214 929 ts_tablet_manager.cc:610] Registered 0 tablets
I20250901 08:00:54.695384 929 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20250901 08:00:54.730099 929 rpc_server.cc:307] RPC server started. Bound to: 127.0.232.65:45873
I20250901 08:00:54.730194 2005 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.232.65:45873 every 8 connection(s)
I20250901 08:00:54.742653 2006 heartbeater.cc:344] Connected to a master server at 127.0.232.126:37115
I20250901 08:00:54.742928 2006 heartbeater.cc:461] Registering TS with master...
I20250901 08:00:54.743502 2006 heartbeater.cc:507] Master 127.0.232.126:37115 requested a full tablet report, sending...
I20250901 08:00:54.745102 1879 ts_manager.cc:194] Registered new tserver with Master: 6ca7c48cfba74c07a24e0e06ecf22132 (127.0.232.65:45873)
I20250901 08:00:54.746138 929 internal_mini_cluster.cc:371] 1 TS(s) registered with all masters after 0.01301731s
I20250901 08:00:54.746685 1879 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.0.1:52626
I20250901 08:00:54.765404 1879 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:52632:
name: "test_table"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
rows: "\004\001d\000\000\000\004\001\310\000\000\000\004\001,\001\000\000\004\001\220\001\000\000\004\001\364\001\000\000\004\001X\002\000\000\004\001\274\002\000\000\004\001 \003\000\000\004\001\204\003\000\000""\004\001d\000\000\000\004\001\310\000\000\000\004\001,\001\000\000\004\001\220\001\000\000\004\001\364\001\000\000\004\001X\002\000\000\004\001\274\002\000\000\004\001 \003\000\000\004\001\204\003\000\000"
indirect_data: """"
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
I20250901 08:00:54.810201 1971 tablet_service.cc:1468] Processing CreateTablet for tablet b0f3c3dbb774412ab31f41c924bac570 (DEFAULT_TABLE table=test_table [id=f389841ca3d4423497759cbd02f9d3f2]), partition=RANGE (key) PARTITION VALUES < 100
I20250901 08:00:54.810456 1970 tablet_service.cc:1468] Processing CreateTablet for tablet 60ed8de893c6428d9d84987b90395121 (DEFAULT_TABLE table=test_table [id=f389841ca3d4423497759cbd02f9d3f2]), partition=RANGE (key) PARTITION 100 <= VALUES < 200
I20250901 08:00:54.811301 1971 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b0f3c3dbb774412ab31f41c924bac570. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:54.812358 1967 tablet_service.cc:1468] Processing CreateTablet for tablet 62dc319bf25247f7864fc3b38950817e (DEFAULT_TABLE table=test_table [id=f389841ca3d4423497759cbd02f9d3f2]), partition=RANGE (key) PARTITION 400 <= VALUES < 500
I20250901 08:00:54.812417 1968 tablet_service.cc:1468] Processing CreateTablet for tablet 9c81c9ff7c504af7a0a5f2ff85f9af8e (DEFAULT_TABLE table=test_table [id=f389841ca3d4423497759cbd02f9d3f2]), partition=RANGE (key) PARTITION 300 <= VALUES < 400
I20250901 08:00:54.813330 1964 tablet_service.cc:1468] Processing CreateTablet for tablet abf7e9fa77444fcc8051a004f46701c2 (DEFAULT_TABLE table=test_table [id=f389841ca3d4423497759cbd02f9d3f2]), partition=RANGE (key) PARTITION 700 <= VALUES < 800
I20250901 08:00:54.813755 1967 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 62dc319bf25247f7864fc3b38950817e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:54.814023 1969 tablet_service.cc:1468] Processing CreateTablet for tablet 49fcff0588034be9bccc8e1ac04dc497 (DEFAULT_TABLE table=test_table [id=f389841ca3d4423497759cbd02f9d3f2]), partition=RANGE (key) PARTITION 200 <= VALUES < 300
I20250901 08:00:54.814390 1968 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9c81c9ff7c504af7a0a5f2ff85f9af8e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:54.815095 1970 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 60ed8de893c6428d9d84987b90395121. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:54.815138 1966 tablet_service.cc:1468] Processing CreateTablet for tablet 1cb8171e589747dd9de3f90e75b9166a (DEFAULT_TABLE table=test_table [id=f389841ca3d4423497759cbd02f9d3f2]), partition=RANGE (key) PARTITION 500 <= VALUES < 600
I20250901 08:00:54.816380 1966 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1cb8171e589747dd9de3f90e75b9166a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:54.816105 1962 tablet_service.cc:1468] Processing CreateTablet for tablet 4093f4fe5e554c58bdbe4a8e571531e9 (DEFAULT_TABLE table=test_table [id=f389841ca3d4423497759cbd02f9d3f2]), partition=RANGE (key) PARTITION 900 <= VALUES
I20250901 08:00:54.817252 1962 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4093f4fe5e554c58bdbe4a8e571531e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:54.818030 1965 tablet_service.cc:1468] Processing CreateTablet for tablet 2875b091d9254f8e9b17e7b4a0119471 (DEFAULT_TABLE table=test_table [id=f389841ca3d4423497759cbd02f9d3f2]), partition=RANGE (key) PARTITION 600 <= VALUES < 700
I20250901 08:00:54.819166 1965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2875b091d9254f8e9b17e7b4a0119471. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:54.823117 1963 tablet_service.cc:1468] Processing CreateTablet for tablet c2e02703d08d443c931d2ddb97cd2267 (DEFAULT_TABLE table=test_table [id=f389841ca3d4423497759cbd02f9d3f2]), partition=RANGE (key) PARTITION 800 <= VALUES < 900
I20250901 08:00:54.824288 1963 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c2e02703d08d443c931d2ddb97cd2267. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:54.826980 1964 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet abf7e9fa77444fcc8051a004f46701c2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:54.830652 1969 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 49fcff0588034be9bccc8e1ac04dc497. 1 dirs total, 0 dirs full, 0 dirs failed
I20250901 08:00:54.833302 2018 tablet_bootstrap.cc:492] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132: Bootstrap starting.
I20250901 08:00:54.838399 2018 tablet_bootstrap.cc:654] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:54.855687 2018 tablet_bootstrap.cc:492] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132: No bootstrap required, opened a new log
I20250901 08:00:54.856073 2018 ts_tablet_manager.cc:1397] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent bootstrapping tablet: real 0.023s user 0.008s sys 0.000s
I20250901 08:00:54.858206 2018 raft_consensus.cc:357] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.858724 2018 raft_consensus.cc:383] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:54.858987 2018 raft_consensus.cc:738] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Initialized, Role: FOLLOWER
I20250901 08:00:54.859558 2018 consensus_queue.cc:260] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.860119 2018 raft_consensus.cc:397] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:54.860383 2018 raft_consensus.cc:491] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:54.860635 2018 raft_consensus.cc:3058] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:54.864765 2018 raft_consensus.cc:513] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.865247 2018 leader_election.cc:304] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 6ca7c48cfba74c07a24e0e06ecf22132; no voters:
I20250901 08:00:54.866255 2018 leader_election.cc:290] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:54.866503 2020 raft_consensus.cc:2802] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:54.867914 2020 raft_consensus.cc:695] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Becoming Leader. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Running, Role: LEADER
I20250901 08:00:54.868113 2006 heartbeater.cc:499] Master 127.0.232.126:37115 was elected leader, sending a full tablet report...
I20250901 08:00:54.868654 2018 ts_tablet_manager.cc:1428] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent starting tablet: real 0.012s user 0.012s sys 0.001s
I20250901 08:00:54.868572 2020 consensus_queue.cc:237] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.869385 2018 tablet_bootstrap.cc:492] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132: Bootstrap starting.
I20250901 08:00:54.874027 2018 tablet_bootstrap.cc:654] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:54.878224 2018 tablet_bootstrap.cc:492] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132: No bootstrap required, opened a new log
I20250901 08:00:54.878608 2018 ts_tablet_manager.cc:1397] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent bootstrapping tablet: real 0.009s user 0.003s sys 0.006s
I20250901 08:00:54.880867 2018 raft_consensus.cc:357] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.881347 2018 raft_consensus.cc:383] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:54.881598 2018 raft_consensus.cc:738] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Initialized, Role: FOLLOWER
I20250901 08:00:54.881783 1879 catalog_manager.cc:5582] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6ca7c48cfba74c07a24e0e06ecf22132 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:54.882162 2018 consensus_queue.cc:260] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.882784 2018 raft_consensus.cc:397] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:54.883078 2018 raft_consensus.cc:491] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:54.883392 2018 raft_consensus.cc:3058] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:54.888494 2018 raft_consensus.cc:513] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.888963 2018 leader_election.cc:304] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 6ca7c48cfba74c07a24e0e06ecf22132; no voters:
I20250901 08:00:54.889506 2018 leader_election.cc:290] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:54.889714 2021 raft_consensus.cc:2802] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:54.890210 2021 raft_consensus.cc:695] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Becoming Leader. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Running, Role: LEADER
I20250901 08:00:54.891074 2018 ts_tablet_manager.cc:1428] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent starting tablet: real 0.012s user 0.009s sys 0.003s
I20250901 08:00:54.890961 2021 consensus_queue.cc:237] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.891831 2018 tablet_bootstrap.cc:492] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132: Bootstrap starting.
I20250901 08:00:54.897454 2018 tablet_bootstrap.cc:654] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:54.901666 2018 tablet_bootstrap.cc:492] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132: No bootstrap required, opened a new log
I20250901 08:00:54.902076 2018 ts_tablet_manager.cc:1397] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent bootstrapping tablet: real 0.010s user 0.008s sys 0.000s
I20250901 08:00:54.902245 1879 catalog_manager.cc:5582] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6ca7c48cfba74c07a24e0e06ecf22132 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:54.904453 2018 raft_consensus.cc:357] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.904873 2018 raft_consensus.cc:383] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:54.905095 2018 raft_consensus.cc:738] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Initialized, Role: FOLLOWER
I20250901 08:00:54.905625 2018 consensus_queue.cc:260] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.906030 2018 raft_consensus.cc:397] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:54.906227 2018 raft_consensus.cc:491] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:54.906533 2018 raft_consensus.cc:3058] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:54.911307 2018 raft_consensus.cc:513] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.911741 2018 leader_election.cc:304] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 6ca7c48cfba74c07a24e0e06ecf22132; no voters:
I20250901 08:00:54.912112 2018 leader_election.cc:290] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:54.912257 2021 raft_consensus.cc:2802] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:54.912746 2021 raft_consensus.cc:695] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Becoming Leader. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Running, Role: LEADER
I20250901 08:00:54.913375 2018 ts_tablet_manager.cc:1428] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent starting tablet: real 0.011s user 0.007s sys 0.001s
I20250901 08:00:54.913429 2021 consensus_queue.cc:237] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.914083 2018 tablet_bootstrap.cc:492] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132: Bootstrap starting.
I20250901 08:00:54.919123 2018 tablet_bootstrap.cc:654] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:54.919291 1879 catalog_manager.cc:5582] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6ca7c48cfba74c07a24e0e06ecf22132 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:54.922719 2018 tablet_bootstrap.cc:492] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132: No bootstrap required, opened a new log
I20250901 08:00:54.923064 2018 ts_tablet_manager.cc:1397] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent bootstrapping tablet: real 0.009s user 0.007s sys 0.001s
I20250901 08:00:54.924880 2018 raft_consensus.cc:357] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.925295 2018 raft_consensus.cc:383] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:54.925608 2018 raft_consensus.cc:738] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Initialized, Role: FOLLOWER
I20250901 08:00:54.926450 2018 consensus_queue.cc:260] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.926956 2018 raft_consensus.cc:397] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:54.927174 2018 raft_consensus.cc:491] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:54.927428 2018 raft_consensus.cc:3058] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:54.932780 2018 raft_consensus.cc:513] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.933210 2018 leader_election.cc:304] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 6ca7c48cfba74c07a24e0e06ecf22132; no voters:
I20250901 08:00:54.933606 2018 leader_election.cc:290] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:54.933756 2021 raft_consensus.cc:2802] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:54.934242 2021 raft_consensus.cc:695] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Becoming Leader. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Running, Role: LEADER
I20250901 08:00:54.934988 2021 consensus_queue.cc:237] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.935209 2018 ts_tablet_manager.cc:1428] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent starting tablet: real 0.012s user 0.010s sys 0.002s
I20250901 08:00:54.936064 2018 tablet_bootstrap.cc:492] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132: Bootstrap starting.
I20250901 08:00:54.940137 2018 tablet_bootstrap.cc:654] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:54.940711 1879 catalog_manager.cc:5582] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6ca7c48cfba74c07a24e0e06ecf22132 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:54.943864 2018 tablet_bootstrap.cc:492] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132: No bootstrap required, opened a new log
I20250901 08:00:54.944267 2018 ts_tablet_manager.cc:1397] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent bootstrapping tablet: real 0.008s user 0.006s sys 0.001s
I20250901 08:00:54.946240 2018 raft_consensus.cc:357] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.946765 2018 raft_consensus.cc:383] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:54.947027 2018 raft_consensus.cc:738] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Initialized, Role: FOLLOWER
I20250901 08:00:54.947603 2018 consensus_queue.cc:260] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.948087 2018 raft_consensus.cc:397] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:54.948292 2018 raft_consensus.cc:491] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:54.948513 2018 raft_consensus.cc:3058] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:54.953357 2018 raft_consensus.cc:513] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.953832 2018 leader_election.cc:304] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 6ca7c48cfba74c07a24e0e06ecf22132; no voters:
I20250901 08:00:54.954380 2018 leader_election.cc:290] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:54.954519 2021 raft_consensus.cc:2802] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:54.954969 2021 raft_consensus.cc:695] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Becoming Leader. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Running, Role: LEADER
I20250901 08:00:54.955639 2021 consensus_queue.cc:237] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.955964 2018 ts_tablet_manager.cc:1428] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent starting tablet: real 0.011s user 0.004s sys 0.007s
I20250901 08:00:54.956652 2018 tablet_bootstrap.cc:492] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132: Bootstrap starting.
I20250901 08:00:54.961668 2018 tablet_bootstrap.cc:654] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:54.962010 1879 catalog_manager.cc:5582] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6ca7c48cfba74c07a24e0e06ecf22132 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:54.965792 2018 tablet_bootstrap.cc:492] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132: No bootstrap required, opened a new log
I20250901 08:00:54.966198 2018 ts_tablet_manager.cc:1397] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent bootstrapping tablet: real 0.010s user 0.004s sys 0.004s
I20250901 08:00:54.968250 2018 raft_consensus.cc:357] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.968758 2018 raft_consensus.cc:383] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:54.969036 2018 raft_consensus.cc:738] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Initialized, Role: FOLLOWER
I20250901 08:00:54.969564 2018 consensus_queue.cc:260] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.970031 2018 raft_consensus.cc:397] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:54.970306 2018 raft_consensus.cc:491] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:54.970587 2018 raft_consensus.cc:3058] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:54.975194 2018 raft_consensus.cc:513] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.975623 2018 leader_election.cc:304] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 6ca7c48cfba74c07a24e0e06ecf22132; no voters:
I20250901 08:00:54.975982 2018 leader_election.cc:290] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:54.976126 2021 raft_consensus.cc:2802] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:54.976568 2021 raft_consensus.cc:695] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Becoming Leader. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Running, Role: LEADER
I20250901 08:00:54.977021 2018 ts_tablet_manager.cc:1428] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent starting tablet: real 0.011s user 0.008s sys 0.000s
I20250901 08:00:54.977191 2021 consensus_queue.cc:237] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.977592 2018 tablet_bootstrap.cc:492] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132: Bootstrap starting.
I20250901 08:00:54.981645 2018 tablet_bootstrap.cc:654] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:54.983291 1879 catalog_manager.cc:5582] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6ca7c48cfba74c07a24e0e06ecf22132 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:54.985648 2018 tablet_bootstrap.cc:492] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132: No bootstrap required, opened a new log
I20250901 08:00:54.986001 2018 ts_tablet_manager.cc:1397] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent bootstrapping tablet: real 0.009s user 0.006s sys 0.000s
I20250901 08:00:54.987474 2018 raft_consensus.cc:357] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.987922 2018 raft_consensus.cc:383] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:54.988163 2018 raft_consensus.cc:738] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Initialized, Role: FOLLOWER
I20250901 08:00:54.988680 2018 consensus_queue.cc:260] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.989176 2018 raft_consensus.cc:397] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:54.989413 2018 raft_consensus.cc:491] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:54.989687 2018 raft_consensus.cc:3058] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:54.993600 2018 raft_consensus.cc:513] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.994035 2018 leader_election.cc:304] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 6ca7c48cfba74c07a24e0e06ecf22132; no voters:
I20250901 08:00:54.994469 2018 leader_election.cc:290] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:54.994618 2021 raft_consensus.cc:2802] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:54.995038 2021 raft_consensus.cc:695] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Becoming Leader. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Running, Role: LEADER
I20250901 08:00:54.995729 2021 consensus_queue.cc:237] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:54.996044 2018 ts_tablet_manager.cc:1428] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent starting tablet: real 0.010s user 0.010s sys 0.000s
I20250901 08:00:54.996801 2018 tablet_bootstrap.cc:492] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132: Bootstrap starting.
I20250901 08:00:55.001271 2018 tablet_bootstrap.cc:654] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:55.001596 1879 catalog_manager.cc:5582] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6ca7c48cfba74c07a24e0e06ecf22132 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:55.004885 2018 tablet_bootstrap.cc:492] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132: No bootstrap required, opened a new log
I20250901 08:00:55.005270 2018 ts_tablet_manager.cc:1397] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent bootstrapping tablet: real 0.009s user 0.007s sys 0.000s
I20250901 08:00:55.007027 2018 raft_consensus.cc:357] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:55.007331 2018 raft_consensus.cc:383] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:55.007535 2018 raft_consensus.cc:738] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Initialized, Role: FOLLOWER
I20250901 08:00:55.008046 2018 consensus_queue.cc:260] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:55.008467 2018 raft_consensus.cc:397] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:55.008652 2018 raft_consensus.cc:491] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:55.008862 2018 raft_consensus.cc:3058] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:55.012609 2018 raft_consensus.cc:513] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:55.013024 2018 leader_election.cc:304] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 6ca7c48cfba74c07a24e0e06ecf22132; no voters:
I20250901 08:00:55.013399 2018 leader_election.cc:290] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:55.013550 2021 raft_consensus.cc:2802] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:55.014026 2021 raft_consensus.cc:695] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Becoming Leader. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Running, Role: LEADER
I20250901 08:00:55.014747 2018 ts_tablet_manager.cc:1428] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent starting tablet: real 0.009s user 0.010s sys 0.000s
I20250901 08:00:55.014806 2021 consensus_queue.cc:237] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:55.015396 2018 tablet_bootstrap.cc:492] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132: Bootstrap starting.
I20250901 08:00:55.019364 2018 tablet_bootstrap.cc:654] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:55.021239 1879 catalog_manager.cc:5582] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6ca7c48cfba74c07a24e0e06ecf22132 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:55.022830 2018 tablet_bootstrap.cc:492] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132: No bootstrap required, opened a new log
I20250901 08:00:55.023133 2018 ts_tablet_manager.cc:1397] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent bootstrapping tablet: real 0.008s user 0.007s sys 0.000s
I20250901 08:00:55.024539 2018 raft_consensus.cc:357] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:55.024894 2018 raft_consensus.cc:383] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:55.025142 2018 raft_consensus.cc:738] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Initialized, Role: FOLLOWER
I20250901 08:00:55.025689 2018 consensus_queue.cc:260] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:55.026206 2018 raft_consensus.cc:397] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:55.026496 2018 raft_consensus.cc:491] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:55.026777 2018 raft_consensus.cc:3058] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:55.030543 2018 raft_consensus.cc:513] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:55.031033 2018 leader_election.cc:304] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 6ca7c48cfba74c07a24e0e06ecf22132; no voters:
I20250901 08:00:55.031385 2018 leader_election.cc:290] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:55.031522 2021 raft_consensus.cc:2802] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:55.031924 2021 raft_consensus.cc:695] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Becoming Leader. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Running, Role: LEADER
I20250901 08:00:55.032495 2021 consensus_queue.cc:237] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:55.032971 2018 ts_tablet_manager.cc:1428] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent starting tablet: real 0.010s user 0.010s sys 0.000s
I20250901 08:00:55.033725 2018 tablet_bootstrap.cc:492] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132: Bootstrap starting.
I20250901 08:00:55.037237 2018 tablet_bootstrap.cc:654] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132: Neither blocks nor log segments found. Creating new log.
I20250901 08:00:55.038259 1879 catalog_manager.cc:5582] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6ca7c48cfba74c07a24e0e06ecf22132 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:55.040535 2018 tablet_bootstrap.cc:492] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132: No bootstrap required, opened a new log
I20250901 08:00:55.040827 2018 ts_tablet_manager.cc:1397] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent bootstrapping tablet: real 0.007s user 0.006s sys 0.000s
I20250901 08:00:55.042111 2018 raft_consensus.cc:357] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:55.042516 2018 raft_consensus.cc:383] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250901 08:00:55.042703 2018 raft_consensus.cc:738] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Initialized, Role: FOLLOWER
I20250901 08:00:55.043190 2018 consensus_queue.cc:260] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:55.043751 2018 raft_consensus.cc:397] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250901 08:00:55.043998 2018 raft_consensus.cc:491] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250901 08:00:55.044294 2018 raft_consensus.cc:3058] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 0 FOLLOWER]: Advancing to term 1
I20250901 08:00:55.048275 2018 raft_consensus.cc:513] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:55.048705 2018 leader_election.cc:304] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 6ca7c48cfba74c07a24e0e06ecf22132; no voters:
I20250901 08:00:55.049078 2018 leader_election.cc:290] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250901 08:00:55.049214 2021 raft_consensus.cc:2802] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Leader election won for term 1
I20250901 08:00:55.049700 2021 raft_consensus.cc:695] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Becoming Leader. State: Replica: 6ca7c48cfba74c07a24e0e06ecf22132, State: Running, Role: LEADER
I20250901 08:00:55.050380 2021 consensus_queue.cc:237] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } }
I20250901 08:00:55.050707 2018 ts_tablet_manager.cc:1428] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132: Time spent starting tablet: real 0.010s user 0.011s sys 0.000s
I20250901 08:00:55.056655 1879 catalog_manager.cc:5582] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6ca7c48cfba74c07a24e0e06ecf22132 (127.0.232.65). New cstate: current_term: 1 leader_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "6ca7c48cfba74c07a24e0e06ecf22132" member_type: VOTER last_known_addr { host: "127.0.232.65" port: 45873 } health_report { overall_health: HEALTHY } } }
I20250901 08:00:55.111370 929 catalog_manager.cc:3511] Servicing AlterTable request from test_user:
table { table_name: "test_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "int_val2" type: INT32 is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } }
I20250901 08:00:55.135915 929 tablet_server.cc:178] TabletServer@127.0.232.65:0 shutting down...
I20250901 08:00:55.190408 2056 tablet.cc:1722] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:55.192338 2051 tablet.cc:1722] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:55.192579 2049 tablet.cc:1722] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:55.198836 2051 tablet.cc:1620] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132: MemRowSet was empty: no flush needed.
I20250901 08:00:55.197618 2052 tablet.cc:1722] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:55.200438 2056 tablet.cc:1620] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132: MemRowSet was empty: no flush needed.
I20250901 08:00:55.202309 2052 tablet.cc:1620] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132: MemRowSet was empty: no flush needed.
I20250901 08:00:55.204123 2049 tablet.cc:1620] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132: MemRowSet was empty: no flush needed.
I20250901 08:00:55.216282 2052 tablet.cc:1722] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:55.217903 2052 tablet.cc:1620] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132: MemRowSet was empty: no flush needed.
I20250901 08:00:55.221823 929 ts_tablet_manager.cc:1500] Shutting down tablet manager...
I20250901 08:00:55.222112 2051 tablet.cc:1722] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:55.223114 929 tablet_replica.cc:331] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132: stopping tablet replica
I20250901 08:00:55.223965 929 raft_consensus.cc:2241] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:55.224607 929 raft_consensus.cc:2270] T 2875b091d9254f8e9b17e7b4a0119471 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:55.224836 2049 tablet.cc:1722] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:55.224009 2051 tablet.cc:1620] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132: MemRowSet was empty: no flush needed.
I20250901 08:00:55.226804 2056 tablet.cc:1722] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:55.227314 2049 tablet.cc:1620] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132: MemRowSet was empty: no flush needed.
I20250901 08:00:55.228152 2056 tablet.cc:1620] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132: MemRowSet was empty: no flush needed.
I20250901 08:00:55.229761 2052 tablet.cc:1722] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:55.231500 2052 tablet.cc:1620] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132: MemRowSet was empty: no flush needed.
I20250901 08:00:55.233144 929 tablet_replica.cc:331] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132: stopping tablet replica
I20250901 08:00:55.233990 929 raft_consensus.cc:2241] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:55.234642 929 raft_consensus.cc:2270] T 1cb8171e589747dd9de3f90e75b9166a P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:55.240224 2049 tablet.cc:1722] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250901 08:00:55.241925 2049 tablet.cc:1620] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132: MemRowSet was empty: no flush needed.
I20250901 08:00:55.245210 929 tablet_replica.cc:331] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132: stopping tablet replica
I20250901 08:00:55.245971 929 raft_consensus.cc:2241] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:55.246524 929 raft_consensus.cc:2270] T b0f3c3dbb774412ab31f41c924bac570 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:55.249171 929 tablet_replica.cc:331] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132: stopping tablet replica
I20250901 08:00:55.249853 929 raft_consensus.cc:2241] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:55.250340 929 raft_consensus.cc:2270] T abf7e9fa77444fcc8051a004f46701c2 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:55.252592 929 tablet_replica.cc:331] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132: stopping tablet replica
I20250901 08:00:55.253026 929 raft_consensus.cc:2241] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:55.253341 929 raft_consensus.cc:2270] T c2e02703d08d443c931d2ddb97cd2267 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:55.255216 929 tablet_replica.cc:331] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132: stopping tablet replica
I20250901 08:00:55.255672 929 raft_consensus.cc:2241] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:55.255972 929 raft_consensus.cc:2270] T 62dc319bf25247f7864fc3b38950817e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:55.257701 929 tablet_replica.cc:331] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132: stopping tablet replica
I20250901 08:00:55.258097 929 raft_consensus.cc:2241] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:55.258507 929 raft_consensus.cc:2270] T 9c81c9ff7c504af7a0a5f2ff85f9af8e P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:55.260239 929 tablet_replica.cc:331] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132: stopping tablet replica
I20250901 08:00:55.260722 929 raft_consensus.cc:2241] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:55.261178 929 raft_consensus.cc:2270] T 49fcff0588034be9bccc8e1ac04dc497 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:55.263118 929 tablet_replica.cc:331] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132: stopping tablet replica
I20250901 08:00:55.263571 929 raft_consensus.cc:2241] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:55.263921 929 raft_consensus.cc:2270] T 4093f4fe5e554c58bdbe4a8e571531e9 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:55.265832 929 tablet_replica.cc:331] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132: stopping tablet replica
I20250901 08:00:55.266209 929 raft_consensus.cc:2241] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:55.266593 929 raft_consensus.cc:2270] T 60ed8de893c6428d9d84987b90395121 P 6ca7c48cfba74c07a24e0e06ecf22132 [term 1 FOLLOWER]: Raft consensus is shut down!
==================
WARNING: ThreadSanitizer: data race (pid=929)
Write of size 8 at 0x7b44001f3b88 by main thread:
#0 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::__destruct_at_end(kudu::MemTracker**) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:429:12 (libkudu_util.so+0x3a6d30)
#1 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libkudu_util.so+0x3a6c44)
#2 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libkudu_util.so+0x3a6a3b)
#3 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libkudu_util.so+0x39ecf1)
#4 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x39e8a3)
#5 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x3a5b65)
#6 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libtablet_test_util.so+0x35485)
#7 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libtablet_test_util.so+0x35429)
#8 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#9 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41c8f4)
#10 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cf6d5)
#11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cf8e9)
#12 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1002d7)
#13 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x100299)
#14 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100259)
#15 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0x1001ba)
#16 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc669)
#17 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc640)
#18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc600)
#19 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b1096)
#20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b0fb4)
#21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b0d6b)
#22 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x289451)
#23 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b43be)
#24 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#25 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#26 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#27 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#28 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#29 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#30 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#31 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#32 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#33 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#34 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#35 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#36 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#37 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#38 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#39 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#40 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#41 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#42 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#43 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#44 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#45 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#46 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Previous read of size 8 at 0x7b44001f3b88 by thread T148 (mutexes: write M165642749574833856):
#0 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::end() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:1536:30 (libkudu_util.so+0x39fc0a)
#1 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:284:22 (libkudu_util.so+0x39d2e6)
#2 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d2c0)
#3 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416322)
#4 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415ab3)
#5 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419cc1)
#6 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419c79)
#7 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419c09)
#8 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419bd1)
#9 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x418dfd)
#10 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#11 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#12 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#13 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#14 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#15 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#16 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#17 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#18 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#19 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#20 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e85c2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d66a6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41dba0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d57e)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d13e7)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1a2e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b42b7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Location is heap block of size 264 at 0x7b44001f3b00 allocated by thread T187:
#0 operator new(unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:64 (catalog_manager-test+0x342a87)
#1 std::__1::__libcpp_allocate(unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:253:10 (libkudu_util.so+0x29a306)
#2 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::allocate(unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1789:34 (libkudu_util.so+0x3a5445)
#3 std::__1::enable_if<!(is_array<long&>::value), std::__1::shared_ptr<long&> >::type std::__1::make_shared<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler&&...) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4290:45 (libkudu_util.so+0x3a5221)
#4 std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/make_shared.h:61:12 (libkudu_util.so+0x39eb55)
#5 kudu::MemTracker::CreateTracker(long, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:54:34 (libkudu_util.so+0x39cb7e)
#6 kudu::tablet::OpTracker::StartMemoryTracking(std::__1::shared_ptr<kudu::MemTracker> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:311:20 (libtablet.so+0x41dd7b)
#7 kudu::tablet::TabletReplica::Start(kudu::consensus::ConsensusBootstrapInfo const&, std::__1::shared_ptr<kudu::tablet::Tablet>, kudu::clock::Clock*, std::__1::shared_ptr<kudu::rpc::Messenger>, scoped_refptr<kudu::rpc::ResultTracker>, scoped_refptr<kudu::log::Log>, kudu::ThreadPool*, kudu::DnsResolver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:277:19 (libtablet.so+0x3d0736)
#8 kudu::tserver::TSTabletManager::OpenTablet(scoped_refptr<kudu::tablet::TabletReplica> const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&, std::__1::atomic<int>*, std::__1::atomic<int>*, kudu::Timer*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1430:18 (libtserver.so+0x2b9b90)
#9 kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:703:3 (libtserver.so+0x2cdb01)
#10 decltype(std::__1::forward<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&>(fp)()) std::__1::__invoke<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&>(kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x2cdaa9)
#11 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&>(kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x2cda39)
#12 std::__1::__function::__alloc_func<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11, std::__1::allocator<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x2cda01)
#13 std::__1::__function::__func<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11, std::__1::allocator<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x2cccdd)
#14 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#15 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#16 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#17 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#18 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#19 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#20 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#21 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#22 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#23 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#24 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
Mutex M165642749574833856 is already destroyed.
Thread T148 'apply [worker]-' (tid=2051, running) created by thread T68 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c7095)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643:15 (libkudu_util.so+0x4423c8)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libmaster.so+0x2d4b89)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45ea4b)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45ccef)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45efb2)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x414e50)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x4153e9)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x41767a)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417608)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x417588)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417543)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41675c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3ea336)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4b88)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa2c0)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x1819a3)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b294)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138644)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1385f3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138568)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138523)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13780c)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x1614a6)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x14946b)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x133acd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x1374a5)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137439)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1373c9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137391)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13668d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
Thread T187 (tid=2018, finished) created by thread T125 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c7095)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643:15 (libkudu_util.so+0x4423c8)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libmaster.so+0x2d4b89)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45ea4b)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45ccef)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45efb2)
#6 kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:703:3 (libtserver.so+0x2b6d82)
#7 kudu::tserver::TabletServiceAdminImpl::CreateTablet(kudu::tserver::CreateTabletRequestPB const*, kudu::tserver::CreateTabletResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1475:34 (libtserver.so+0x267209)
#8 kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/tserver/tserver_admin.service.cc:189:13 (libtserver_admin_proto.so+0x68bf4)
#9 decltype(std::__1::forward<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver_admin_proto.so+0x68b82)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver_admin_proto.so+0x68ab1)
#11 std::__1::__function::__alloc_func<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver_admin_proto.so+0x68a2c)
#12 std::__1::__function::__func<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver_admin_proto.so+0x67ce2)
#13 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x1f3ccc)
#14 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x1f3106)
#15 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x1f2aaf)
#16 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:229:15 (libkrpc.so+0x1f5dc3)
#17 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:92:5 (libkrpc.so+0x1f70e1)
#18 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x1f7099)
#19 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x1f7029)
#20 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x1f6ff1)
#21 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x1f62ed)
#22 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#23 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#24 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:429:12 in std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::__destruct_at_end(kudu::MemTracker**)
==================
==================
WARNING: ThreadSanitizer: data race (pid=929)
Write of size 8 at 0x7b08000a3340 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x343359)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34f289)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34f229)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29aaa2)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29aa39)
#5 std::__1::allocator<kudu::MemTracker*>::deallocate(kudu::MemTracker**, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a6e0a)
#6 std::__1::allocator_traits<std::__1::allocator<kudu::MemTracker*> >::deallocate(std::__1::allocator<kudu::MemTracker*>&, kudu::MemTracker**, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1533:14 (libkudu_util.so+0x3a6c89)
#7 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:466:9 (libkudu_util.so+0x3a6a67)
#8 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libkudu_util.so+0x39ecf1)
#9 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x39e8a3)
#10 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x3a5b65)
#11 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libtablet_test_util.so+0x35485)
#12 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libtablet_test_util.so+0x35429)
#13 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#14 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41c8f4)
#15 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cf6d5)
#16 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cf8e9)
#17 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1002d7)
#18 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x100299)
#19 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100259)
#20 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0x1001ba)
#21 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc669)
#22 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc640)
#23 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc600)
#24 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b1096)
#25 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b0fb4)
#26 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b0d6b)
#27 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x289451)
#28 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b43be)
#29 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#30 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#31 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#32 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#33 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#34 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#35 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#36 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#37 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#38 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#39 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#40 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#41 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#42 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#43 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#44 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#45 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#46 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#47 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#48 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#49 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#50 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#51 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Previous read of size 8 at 0x7b08000a3340 by thread T148 (mutexes: write M165642749574833856):
#0 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:5 (libkudu_util.so+0x39d322)
#1 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d2c0)
#2 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416322)
#3 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415ab3)
#4 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419cc1)
#5 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419c79)
#6 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419c09)
#7 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419bd1)
#8 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x418dfd)
#9 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#10 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#11 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#12 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#13 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#14 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#15 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#16 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#17 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#18 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#19 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e85c2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d66a6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41dba0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d57e)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d13e7)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1a2e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b42b7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Mutex M165642749574833856 is already destroyed.
Thread T148 'apply [worker]-' (tid=2051, running) created by thread T68 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c7095)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643:15 (libkudu_util.so+0x4423c8)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libmaster.so+0x2d4b89)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45ea4b)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45ccef)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45efb2)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x414e50)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x4153e9)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x41767a)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417608)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x417588)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417543)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41675c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3ea336)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4b88)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa2c0)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x1819a3)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b294)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138644)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1385f3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138568)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138523)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13780c)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x1614a6)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x14946b)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x133acd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x1374a5)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137439)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1373c9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137391)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13668d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 in operator delete(void*)
==================
==================
WARNING: ThreadSanitizer: data race (pid=929)
Write of size 8 at 0x7b44001f32b0 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x343359)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34f289)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34f229)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29aaa2)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29aa39)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a622d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a5bd0)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x39e8ab)
#11 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x3a5b65)
#12 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libtablet_test_util.so+0x35485)
#13 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libtablet_test_util.so+0x35429)
#14 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#15 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41c8f4)
#16 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cf6d5)
#17 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cf8e9)
#18 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1002d7)
#19 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x100299)
#20 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100259)
#21 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0x1001ba)
#22 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc669)
#23 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc640)
#24 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc600)
#25 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b1096)
#26 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b0fb4)
#27 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b0d6b)
#28 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x289451)
#29 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b43be)
#30 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#31 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#32 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#33 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#34 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#35 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#36 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#37 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#38 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#39 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#40 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#41 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#42 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#43 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#44 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#45 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#46 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#47 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#48 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#49 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#50 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#51 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#52 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Previous atomic write of size 8 at 0x7b44001f32b0 by thread T148 (mutexes: write M165642749574833856):
#0 __tsan_atomic64_fetch_add /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:620 (catalog_manager-test+0x316635)
#1 long std::__1::__cxx_atomic_fetch_add<long>(std::__1::__cxx_atomic_base_impl<long>*, long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1042:12 (libkudu_util.so+0x2a5263)
#2 std::__1::__atomic_base<long, true>::fetch_add(long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1706:17 (libkudu_util.so+0x2a51a9)
#3 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:30 (libkudu_util.so+0x39fcc2)
#4 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x39d331)
#5 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d2c0)
#6 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416322)
#7 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415ab3)
#8 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419cc1)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419c79)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419c09)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419bd1)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x418dfd)
#13 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#14 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#15 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#16 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#17 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#18 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#19 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#20 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#21 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#22 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#23 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e85c2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d66a6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41dba0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d57e)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d13e7)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1a2e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b42b7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Mutex M165642749574833856 is already destroyed.
Thread T148 'apply [worker]-' (tid=2051, running) created by thread T68 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c7095)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643:15 (libkudu_util.so+0x4423c8)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libmaster.so+0x2d4b89)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45ea4b)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45ccef)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45efb2)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x414e50)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x4153e9)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x41767a)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417608)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x417588)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417543)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41675c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3ea336)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4b88)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa2c0)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x1819a3)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b294)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138644)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1385f3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138568)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138523)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13780c)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x1614a6)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x14946b)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x133acd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x1374a5)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137439)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1373c9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137391)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13668d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 in operator delete(void*)
==================
==================
WARNING: ThreadSanitizer: data race (pid=929)
Write of size 8 at 0x7b44001f32b8 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x343359)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34f289)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34f229)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29aaa2)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29aa39)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a622d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a5bd0)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x39e8ab)
#11 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x3a5b65)
#12 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libtablet_test_util.so+0x35485)
#13 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libtablet_test_util.so+0x35429)
#14 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#15 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41c8f4)
#16 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cf6d5)
#17 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cf8e9)
#18 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1002d7)
#19 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x100299)
#20 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100259)
#21 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0x1001ba)
#22 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc669)
#23 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc640)
#24 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc600)
#25 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b1096)
#26 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b0fb4)
#27 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b0d6b)
#28 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x289451)
#29 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b43be)
#30 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#31 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#32 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#33 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#34 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#35 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#36 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#37 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#38 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#39 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#40 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#41 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#42 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#43 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#44 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#45 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#46 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#47 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#48 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#49 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#50 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#51 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#52 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Previous atomic write of size 8 at 0x7b44001f32b8 by thread T148 (mutexes: write M165642749574833856):
#0 __tsan_atomic64_compare_exchange_val /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:850 (catalog_manager-test+0x31d708)
#1 bool std::__1::__cxx_atomic_compare_exchange_weak<long>(std::__1::__cxx_atomic_base_impl<long>*, long*, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1031:12 (liblog.so+0x8bd24)
#2 std::__1::__atomic_base<long, false>::compare_exchange_weak(long&, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1622:17 (liblog.so+0x8bb49)
#3 void kudu::AtomicStoreMax<long>(std::__1::atomic<long>&, long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/atomic-utils.h:33:13 (liblog.so+0x7eaa0)
#4 kudu::HighWaterMark::UpdateMax(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:80:5 (libkudu_util.so+0x3a4e64)
#5 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:5 (libkudu_util.so+0x39fccd)
#6 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x39d331)
#7 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d2c0)
#8 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416322)
#9 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415ab3)
#10 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419cc1)
#11 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419c79)
#12 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419c09)
#13 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419bd1)
#14 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x418dfd)
#15 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#16 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#17 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#18 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#19 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#21 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#22 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#23 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#24 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#25 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e85c2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d66a6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41dba0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d57e)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d13e7)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1a2e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b42b7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Mutex M165642749574833856 is already destroyed.
Thread T148 'apply [worker]-' (tid=2051, running) created by thread T68 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c7095)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643:15 (libkudu_util.so+0x4423c8)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libmaster.so+0x2d4b89)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45ea4b)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45ccef)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45efb2)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x414e50)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x4153e9)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x41767a)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417608)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x417588)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417543)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41675c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3ea336)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4b88)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa2c0)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x1819a3)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b294)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138644)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1385f3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138568)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138523)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13780c)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x1614a6)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x14946b)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x133acd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x1374a5)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137439)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1373c9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137391)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13668d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 in operator delete(void*)
==================
==================
WARNING: ThreadSanitizer: data race (pid=929)
Write of size 8 at 0x7b44001f3b70 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x343359)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34f289)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34f229)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29aaa2)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29aa39)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a622d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a5bd0)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41c8f4)
#11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cf6d5)
#12 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cf8e9)
#13 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1002d7)
#14 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x100299)
#15 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100259)
#16 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0x1001ba)
#17 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc669)
#18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc640)
#19 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc600)
#20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b1096)
#21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b0fb4)
#22 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b0d6b)
#23 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x289451)
#24 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b43be)
#25 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#26 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#27 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#28 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#29 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#30 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#31 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#32 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#33 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#34 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#35 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#36 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#37 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#38 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#39 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#40 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#41 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#42 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#43 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#44 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#45 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#46 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#47 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Previous atomic write of size 8 at 0x7b44001f3b70 by thread T148 (mutexes: write M165642749574833856):
#0 __tsan_atomic64_fetch_add /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:620 (catalog_manager-test+0x316635)
#1 long std::__1::__cxx_atomic_fetch_add<long>(std::__1::__cxx_atomic_base_impl<long>*, long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1042:12 (libkudu_util.so+0x2a5263)
#2 std::__1::__atomic_base<long, true>::fetch_add(long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1706:17 (libkudu_util.so+0x2a51a9)
#3 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:30 (libkudu_util.so+0x39fcc2)
#4 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x39d331)
#5 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d2c0)
#6 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416322)
#7 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415ab3)
#8 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419cc1)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419c79)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419c09)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419bd1)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x418dfd)
#13 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#14 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#15 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#16 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#17 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#18 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#19 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#20 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#21 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#22 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#23 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e85c2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d66a6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41dba0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d57e)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d13e7)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1a2e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b42b7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Mutex M165642749574833856 is already destroyed.
Thread T148 'apply [worker]-' (tid=2051, running) created by thread T68 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c7095)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643:15 (libkudu_util.so+0x4423c8)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libmaster.so+0x2d4b89)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45ea4b)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45ccef)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45efb2)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x414e50)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x4153e9)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x41767a)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417608)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x417588)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417543)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41675c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3ea336)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4b88)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa2c0)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x1819a3)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b294)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138644)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1385f3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138568)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138523)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13780c)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x1614a6)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x14946b)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x133acd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x1374a5)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137439)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1373c9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137391)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13668d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 in operator delete(void*)
==================
==================
WARNING: ThreadSanitizer: data race (pid=929)
Write of size 8 at 0x7b44001f3b78 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x343359)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34f289)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34f229)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29aaa2)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29aa39)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a622d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a5bd0)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41c8f4)
#11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cf6d5)
#12 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cf8e9)
#13 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1002d7)
#14 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x100299)
#15 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100259)
#16 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0x1001ba)
#17 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc669)
#18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc640)
#19 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc600)
#20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b1096)
#21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b0fb4)
#22 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b0d6b)
#23 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x289451)
#24 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b43be)
#25 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#26 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#27 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#28 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#29 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#30 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#31 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#32 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#33 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#34 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#35 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#36 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#37 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#38 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#39 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#40 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#41 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#42 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#43 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#44 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#45 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#46 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#47 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Previous atomic write of size 8 at 0x7b44001f3b78 by thread T148 (mutexes: write M165642749574833856):
#0 __tsan_atomic64_compare_exchange_val /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:850 (catalog_manager-test+0x31d708)
#1 bool std::__1::__cxx_atomic_compare_exchange_weak<long>(std::__1::__cxx_atomic_base_impl<long>*, long*, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1031:12 (liblog.so+0x8bd24)
#2 std::__1::__atomic_base<long, false>::compare_exchange_weak(long&, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1622:17 (liblog.so+0x8bb49)
#3 void kudu::AtomicStoreMax<long>(std::__1::atomic<long>&, long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/atomic-utils.h:33:13 (liblog.so+0x7eaa0)
#4 kudu::HighWaterMark::UpdateMax(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:80:5 (libkudu_util.so+0x3a4e64)
#5 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:5 (libkudu_util.so+0x39fccd)
#6 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x39d331)
#7 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d2c0)
#8 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416322)
#9 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415ab3)
#10 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419cc1)
#11 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419c79)
#12 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419c09)
#13 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419bd1)
#14 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x418dfd)
#15 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#16 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#17 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#18 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#19 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#21 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#22 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#23 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#24 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#25 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e85c2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d66a6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41dba0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d57e)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d13e7)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1a2e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b42b7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Mutex M165642749574833856 is already destroyed.
Thread T148 'apply [worker]-' (tid=2051, running) created by thread T68 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c7095)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643:15 (libkudu_util.so+0x4423c8)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libmaster.so+0x2d4b89)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45ea4b)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45ccef)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45efb2)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x414e50)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x4153e9)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x41767a)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417608)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x417588)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417543)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41675c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3ea336)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4b88)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa2c0)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x1819a3)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b294)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138644)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1385f3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138568)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138523)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13780c)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x1614a6)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x14946b)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x133acd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x1374a5)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137439)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1373c9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137391)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13668d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 in operator delete(void*)
==================
==================
WARNING: ThreadSanitizer: data race (pid=929)
Write of size 8 at 0x7b44001f3b80 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x343359)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34f289)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34f229)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29aaa2)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29aa39)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a622d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a5bd0)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41c8f4)
#11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cf6d5)
#12 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cf8e9)
#13 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1002d7)
#14 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x100299)
#15 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100259)
#16 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0x1001ba)
#17 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc669)
#18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc640)
#19 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc600)
#20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b1096)
#21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b0fb4)
#22 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b0d6b)
#23 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x289451)
#24 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b43be)
#25 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#26 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#27 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#28 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#29 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#30 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#31 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#32 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#33 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#34 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#35 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#36 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#37 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#38 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#39 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#40 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#41 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#42 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#43 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#44 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#45 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#46 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#47 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Previous read of size 8 at 0x7b44001f3b80 by thread T148 (mutexes: write M165642749574833856):
#0 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::begin() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:1520:30 (libkudu_util.so+0x39fbc9)
#1 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:284:22 (libkudu_util.so+0x39d2da)
#2 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d2c0)
#3 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416322)
#4 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415ab3)
#5 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419cc1)
#6 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419c79)
#7 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419c09)
#8 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419bd1)
#9 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x418dfd)
#10 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#11 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#12 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#13 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#14 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#15 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#16 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#17 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#18 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#19 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#20 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e85c2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d66a6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41dba0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d57e)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d13e7)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1a2e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b42b7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x25602e)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b7f9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x3135a2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd7b47)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe39e4)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd6543)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd65d9)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x351f27)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x351e8d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x351e1b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34bffd)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x34c059)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x360d4b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:115:10 (catalog_manager-test+0x35f95c)
Mutex M165642749574833856 is already destroyed.
Thread T148 'apply [worker]-' (tid=2051, running) created by thread T68 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c7095)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643:15 (libkudu_util.so+0x4423c8)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libmaster.so+0x2d4b89)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45ea4b)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45ccef)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45efb2)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x414e50)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x4153e9)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x41767a)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417608)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x417588)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417543)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41675c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3ea336)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4b88)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa2c0)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x1819a3)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b294)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138644)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1385f3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138568)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138523)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13780c)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x1614a6)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x14946b)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x133acd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x1374a5)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137439)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1373c9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137391)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13668d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x45f814)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x461b61)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x461b19)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x461aa9)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x461a71)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x460d6d)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (catalog_manager-test+0x360fe4)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (catalog_manager-test+0x360e29)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3 (libkudu_util.so+0x442b1a)
SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 in operator delete(void*)
==================
I20250901 08:00:55.659466 929 tablet_server.cc:195] TabletServer@127.0.232.65:0 shutdown complete.
I20250901 08:00:55.674324 929 master.cc:561] Master@127.0.232.126:37115 shutting down...
I20250901 08:00:55.688601 929 raft_consensus.cc:2241] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [term 1 LEADER]: Raft consensus shutting down.
I20250901 08:00:55.689080 929 raft_consensus.cc:2270] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250901 08:00:55.689335 929 tablet_replica.cc:331] T 00000000000000000000000000000000 P 861b7645c49a40b08c5d6735d979f241: stopping tablet replica
I20250901 08:00:55.705722 929 master.cc:583] Master@127.0.232.126:37115 shutdown complete.
[ OK ] CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser (1247 ms)
[----------] 6 tests from CatalogManagerRpcAndUserFunctionsTest (6934 ms total)
[----------] Global test environment tear-down
[==========] 10 tests from 3 test suites ran. (7000 ms total)
[ PASSED ] 10 tests.
I20250901 08:00:55.731341 929 logging.cc:424] LogThrottler /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/proxy.cc:239: suppressed but not reported on 72 messages since previous log ~4 seconds ago
ThreadSanitizer: reported 7 warnings