CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser: WARNING: ThreadSanitizer: data race (pid=4824) Write of size 8 at 0x7b4400112048 by main thread:
#0 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::__destruct_at_end(kudu::MemTracker**) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:429:12 (libkudu_util.so+0x3a7aa0)
#1 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libkudu_util.so+0x3a79b4)
#2 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libkudu_util.so+0x3a77ab)
#3 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libkudu_util.so+0x39fa61)
#4 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x39f613)
#5 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x3a68d5)
#6 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libtablet_test_util.so+0x35485)
#7 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libtablet_test_util.so+0x35429)
#8 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#9 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41cc24)
#10 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cfa05)
#11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cfc19)
#12 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1000f7)
#13 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x1000b9)
#14 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100079)
#15 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0xfffda)
#16 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc329)
#17 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc300)
#18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc2c0)
#19 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b09a6)
#20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b08c4)
#21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b067b)
#22 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x288d61)
#23 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b3cce)
#24 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#25 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#26 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#27 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#28 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#29 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#30 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#31 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#32 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#33 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#34 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#35 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#36 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#37 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#38 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#39 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#40 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#41 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#42 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#43 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#44 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#45 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#46 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Previous read of size 8 at 0x7b4400112048 by thread T75 (mutexes: write M197167946966685248):
#0 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::end() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:1536:30 (libkudu_util.so+0x3a097a)
#1 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:284:22 (libkudu_util.so+0x39e056)
#2 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d5f0)
#3 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416652)
#4 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415de3)
#5 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419ff1)
#6 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419fa9)
#7 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419f39)
#8 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419f01)
#9 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41912d)
#10 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#11 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#12 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#13 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#14 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#15 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#16 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#17 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#18 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#19 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#20 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e4ed2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d72d6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41ded0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d8ae)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d1717)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1d5e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b3bc7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Location is heap block of size 264 at 0x7b4400111fc0 allocated by thread T158:
#0 operator new(unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:64 (catalog_manager-test+0x33f397)
#1 std::__1::__libcpp_allocate(unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:253:10 (libkudu_util.so+0x29b1e6)
#2 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::allocate(unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1789:34 (libkudu_util.so+0x3a61b5)
#3 std::__1::enable_if<!(is_array<long&>::value), std::__1::shared_ptr<long&> >::type std::__1::make_shared<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler&&...) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4290:45 (libkudu_util.so+0x3a5f91)
#4 std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/make_shared.h:61:12 (libkudu_util.so+0x39f8c5)
#5 kudu::MemTracker::CreateTracker(long, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:54:34 (libkudu_util.so+0x39d8ee)
#6 kudu::tablet::OpTracker::StartMemoryTracking(std::__1::shared_ptr<kudu::MemTracker> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:311:20 (libtablet.so+0x41e0ab)
#7 kudu::tablet::TabletReplica::Start(kudu::consensus::ConsensusBootstrapInfo const&, std::__1::shared_ptr<kudu::tablet::Tablet>, kudu::clock::Clock*, std::__1::shared_ptr<kudu::rpc::Messenger>, scoped_refptr<kudu::rpc::ResultTracker>, scoped_refptr<kudu::log::Log>, kudu::ThreadPool*, kudu::DnsResolver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:277:19 (libtablet.so+0x3d0a66)
#8 kudu::tserver::TSTabletManager::OpenTablet(scoped_refptr<kudu::tablet::TabletReplica> const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&, std::__1::atomic<int>*, std::__1::atomic<int>*, kudu::Timer*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1430:18 (libtserver.so+0x2b94a0)
#9 kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:703:3 (libtserver.so+0x2cd411)
#10 decltype(std::__1::forward<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&>(fp)()) std::__1::__invoke<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&>(kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x2cd3b9)
#11 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&>(kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x2cd349)
#12 std::__1::__function::__alloc_func<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11, std::__1::allocator<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x2cd311)
#13 std::__1::__function::__func<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11, std::__1::allocator<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x2cc5ed)
#14 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#15 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#16 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#17 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#18 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#19 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#20 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#21 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#22 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#23 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#24 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
Mutex M197167946966685248 is already destroyed.
Thread T75 'apply [worker]-' (tid=5947, running) created by thread T95 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c39a5)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642:15 (libkudu_util.so+0x442eca)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:146:12 (libmaster.so+0x2d49b9)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45f7cb)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45da6f)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45fd32)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x415180)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x415719)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x4179aa)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417938)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x4178b8)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417873)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x416a8c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3e9ff6)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4848)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa040)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x181723)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b014)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138534)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1384e3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138458)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138413)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x1376fc)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x161226)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x1491eb)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x1339bd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x137395)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137329)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1372b9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137281)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13657d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
Thread T158 (tid=5911, finished) created by thread T194 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c39a5)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642:15 (libkudu_util.so+0x442eca)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:146:12 (libmaster.so+0x2d49b9)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45f7cb)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45da6f)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45fd32)
#6 kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:703:3 (libtserver.so+0x2b6692)
#7 kudu::tserver::TabletServiceAdminImpl::CreateTablet(kudu::tserver::CreateTabletRequestPB const*, kudu::tserver::CreateTabletResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1475:34 (libtserver.so+0x266c99)
#8 kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/tserver/tserver_admin.service.cc:189:13 (libtserver_admin_proto.so+0x67d34)
#9 decltype(std::__1::forward<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver_admin_proto.so+0x67cc2)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver_admin_proto.so+0x67bf1)
#11 std::__1::__function::__alloc_func<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver_admin_proto.so+0x67b6c)
#12 std::__1::__function::__func<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver_admin_proto.so+0x66e22)
#13 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x1f2dcc)
#14 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x1f2206)
#15 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x1f1baf)
#16 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:229:15 (libkrpc.so+0x1f4ec3)
#17 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:92:5 (libkrpc.so+0x1f61e1)
#18 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x1f6199)
#19 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x1f6129)
#20 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x1f60f1)
#21 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x1f53ed)
#22 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#23 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#24 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser: WARNING: ThreadSanitizer: data race (pid=4824) Write of size 8 at 0x7b0800098780 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x33fc69)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34bb99)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34bb39)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29b982)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29b919)
#5 std::__1::allocator<kudu::MemTracker*>::deallocate(kudu::MemTracker**, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a7b7a)
#6 std::__1::allocator_traits<std::__1::allocator<kudu::MemTracker*> >::deallocate(std::__1::allocator<kudu::MemTracker*>&, kudu::MemTracker**, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1533:14 (libkudu_util.so+0x3a79f9)
#7 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:466:9 (libkudu_util.so+0x3a77d7)
#8 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libkudu_util.so+0x39fa61)
#9 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x39f613)
#10 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x3a68d5)
#11 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libtablet_test_util.so+0x35485)
#12 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libtablet_test_util.so+0x35429)
#13 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#14 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41cc24)
#15 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cfa05)
#16 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cfc19)
#17 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1000f7)
#18 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x1000b9)
#19 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100079)
#20 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0xfffda)
#21 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc329)
#22 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc300)
#23 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc2c0)
#24 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b09a6)
#25 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b08c4)
#26 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b067b)
#27 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x288d61)
#28 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b3cce)
#29 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#30 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#31 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#32 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#33 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#34 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#35 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#36 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#37 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#38 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#39 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#40 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#41 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#42 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#43 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#44 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#45 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#46 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#47 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#48 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#49 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#50 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#51 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Previous read of size 8 at 0x7b0800098780 by thread T75 (mutexes: write M197167946966685248):
#0 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:5 (libkudu_util.so+0x39e092)
#1 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d5f0)
#2 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416652)
#3 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415de3)
#4 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419ff1)
#5 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419fa9)
#6 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419f39)
#7 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419f01)
#8 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41912d)
#9 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#10 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#11 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#12 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#13 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#14 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#15 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#16 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#17 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#18 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#19 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e4ed2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d72d6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41ded0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d8ae)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d1717)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1d5e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b3bc7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Mutex M197167946966685248 is already destroyed.
Thread T75 'apply [worker]-' (tid=5947, running) created by thread T95 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c39a5)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642:15 (libkudu_util.so+0x442eca)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:146:12 (libmaster.so+0x2d49b9)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45f7cb)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45da6f)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45fd32)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x415180)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x415719)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x4179aa)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417938)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x4178b8)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417873)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x416a8c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3e9ff6)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4848)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa040)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x181723)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b014)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138534)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1384e3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138458)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138413)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x1376fc)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x161226)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x1491eb)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x1339bd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x137395)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137329)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1372b9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137281)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13657d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser: WARNING: ThreadSanitizer: data race (pid=4824) Write of size 8 at 0x7b44000cfd70 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x33fc69)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34bb99)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34bb39)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29b982)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29b919)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a6f9d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a6940)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x39f61b)
#11 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x3a68d5)
#12 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libtablet_test_util.so+0x35485)
#13 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libtablet_test_util.so+0x35429)
#14 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#15 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41cc24)
#16 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cfa05)
#17 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cfc19)
#18 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1000f7)
#19 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x1000b9)
#20 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100079)
#21 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0xfffda)
#22 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc329)
#23 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc300)
#24 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc2c0)
#25 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b09a6)
#26 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b08c4)
#27 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b067b)
#28 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x288d61)
#29 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b3cce)
#30 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#31 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#32 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#33 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#34 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#35 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#36 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#37 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#38 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#39 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#40 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#41 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#42 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#43 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#44 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#45 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#46 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#47 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#48 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#49 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#50 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#51 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#52 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Previous atomic write of size 8 at 0x7b44000cfd70 by thread T75 (mutexes: write M197167946966685248):
#0 __tsan_atomic64_fetch_add /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:620 (catalog_manager-test+0x312f45)
#1 long std::__1::__cxx_atomic_fetch_add<long>(std::__1::__cxx_atomic_base_impl<long>*, long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1042:12 (libkudu_util.so+0x2a5fd3)
#2 std::__1::__atomic_base<long, true>::fetch_add(long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1706:17 (libkudu_util.so+0x2a5f19)
#3 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:30 (libkudu_util.so+0x3a0a32)
#4 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x39e0a1)
#5 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d5f0)
#6 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416652)
#7 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415de3)
#8 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419ff1)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419fa9)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419f39)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419f01)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41912d)
#13 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#14 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#15 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#16 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#17 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#18 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#19 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#20 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#21 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#22 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#23 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e4ed2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d72d6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41ded0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d8ae)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d1717)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1d5e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b3bc7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Mutex M197167946966685248 is already destroyed.
Thread T75 'apply [worker]-' (tid=5947, running) created by thread T95 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c39a5)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642:15 (libkudu_util.so+0x442eca)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:146:12 (libmaster.so+0x2d49b9)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45f7cb)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45da6f)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45fd32)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x415180)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x415719)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x4179aa)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417938)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x4178b8)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417873)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x416a8c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3e9ff6)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4848)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa040)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x181723)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b014)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138534)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1384e3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138458)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138413)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x1376fc)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x161226)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x1491eb)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x1339bd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x137395)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137329)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1372b9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137281)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13657d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser: WARNING: ThreadSanitizer: data race (pid=4824) Write of size 8 at 0x7b44000cfd78 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x33fc69)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34bb99)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34bb39)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29b982)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29b919)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a6f9d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a6940)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x39f61b)
#11 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x3a68d5)
#12 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libtablet_test_util.so+0x35485)
#13 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libtablet_test_util.so+0x35429)
#14 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#15 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41cc24)
#16 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cfa05)
#17 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cfc19)
#18 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1000f7)
#19 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x1000b9)
#20 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100079)
#21 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0xfffda)
#22 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc329)
#23 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc300)
#24 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc2c0)
#25 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b09a6)
#26 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b08c4)
#27 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b067b)
#28 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x288d61)
#29 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b3cce)
#30 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#31 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#32 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#33 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#34 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#35 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#36 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#37 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#38 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#39 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#40 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#41 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#42 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#43 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#44 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#45 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#46 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#47 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#48 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#49 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#50 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#51 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#52 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Previous atomic write of size 8 at 0x7b44000cfd78 by thread T75 (mutexes: write M197167946966685248):
#0 __tsan_atomic64_compare_exchange_val /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:850 (catalog_manager-test+0x31a018)
#1 bool std::__1::__cxx_atomic_compare_exchange_weak<long>(std::__1::__cxx_atomic_base_impl<long>*, long*, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1031:12 (liblog.so+0x8bad4)
#2 std::__1::__atomic_base<long, false>::compare_exchange_weak(long&, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1622:17 (liblog.so+0x8b8f9)
#3 void kudu::AtomicStoreMax<long>(std::__1::atomic<long>&, long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/atomic-utils.h:33:13 (liblog.so+0x7e850)
#4 kudu::HighWaterMark::UpdateMax(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:80:5 (libkudu_util.so+0x3a5bd4)
#5 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:5 (libkudu_util.so+0x3a0a3d)
#6 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x39e0a1)
#7 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d5f0)
#8 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416652)
#9 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415de3)
#10 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419ff1)
#11 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419fa9)
#12 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419f39)
#13 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419f01)
#14 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41912d)
#15 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#16 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#17 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#18 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#19 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#21 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#22 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#23 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#24 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#25 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e4ed2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d72d6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41ded0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d8ae)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d1717)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1d5e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b3bc7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Mutex M197167946966685248 is already destroyed.
Thread T75 'apply [worker]-' (tid=5947, running) created by thread T95 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c39a5)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642:15 (libkudu_util.so+0x442eca)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:146:12 (libmaster.so+0x2d49b9)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45f7cb)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45da6f)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45fd32)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x415180)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x415719)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x4179aa)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417938)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x4178b8)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417873)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x416a8c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3e9ff6)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4848)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa040)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x181723)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b014)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138534)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1384e3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138458)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138413)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x1376fc)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x161226)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x1491eb)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x1339bd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x137395)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137329)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1372b9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137281)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13657d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser: WARNING: ThreadSanitizer: data race (pid=4824) Write of size 8 at 0x7b4400112030 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x33fc69)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34bb99)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34bb39)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29b982)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29b919)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a6f9d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a6940)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41cc24)
#11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cfa05)
#12 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cfc19)
#13 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1000f7)
#14 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x1000b9)
#15 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100079)
#16 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0xfffda)
#17 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc329)
#18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc300)
#19 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc2c0)
#20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b09a6)
#21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b08c4)
#22 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b067b)
#23 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x288d61)
#24 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b3cce)
#25 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#26 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#27 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#28 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#29 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#30 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#31 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#32 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#33 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#34 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#35 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#36 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#37 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#38 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#39 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#40 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#41 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#42 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#43 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#44 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#45 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#46 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#47 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Previous atomic write of size 8 at 0x7b4400112030 by thread T75 (mutexes: write M197167946966685248):
#0 __tsan_atomic64_fetch_add /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:620 (catalog_manager-test+0x312f45)
#1 long std::__1::__cxx_atomic_fetch_add<long>(std::__1::__cxx_atomic_base_impl<long>*, long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1042:12 (libkudu_util.so+0x2a5fd3)
#2 std::__1::__atomic_base<long, true>::fetch_add(long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1706:17 (libkudu_util.so+0x2a5f19)
#3 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:30 (libkudu_util.so+0x3a0a32)
#4 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x39e0a1)
#5 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d5f0)
#6 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416652)
#7 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415de3)
#8 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419ff1)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419fa9)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419f39)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419f01)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41912d)
#13 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#14 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#15 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#16 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#17 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#18 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#19 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#20 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#21 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#22 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#23 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e4ed2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d72d6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41ded0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d8ae)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d1717)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1d5e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b3bc7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Mutex M197167946966685248 is already destroyed.
Thread T75 'apply [worker]-' (tid=5947, running) created by thread T95 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c39a5)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642:15 (libkudu_util.so+0x442eca)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:146:12 (libmaster.so+0x2d49b9)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45f7cb)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45da6f)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45fd32)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x415180)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x415719)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x4179aa)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417938)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x4178b8)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417873)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x416a8c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3e9ff6)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4848)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa040)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x181723)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b014)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138534)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1384e3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138458)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138413)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x1376fc)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x161226)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x1491eb)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x1339bd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x137395)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137329)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1372b9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137281)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13657d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser: WARNING: ThreadSanitizer: data race (pid=4824) Write of size 8 at 0x7b4400112038 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x33fc69)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34bb99)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34bb39)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29b982)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29b919)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a6f9d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a6940)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41cc24)
#11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cfa05)
#12 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cfc19)
#13 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1000f7)
#14 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x1000b9)
#15 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100079)
#16 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0xfffda)
#17 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc329)
#18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc300)
#19 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc2c0)
#20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b09a6)
#21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b08c4)
#22 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b067b)
#23 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x288d61)
#24 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b3cce)
#25 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#26 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#27 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#28 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#29 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#30 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#31 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#32 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#33 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#34 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#35 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#36 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#37 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#38 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#39 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#40 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#41 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#42 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#43 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#44 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#45 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#46 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#47 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Previous atomic write of size 8 at 0x7b4400112038 by thread T75 (mutexes: write M197167946966685248):
#0 __tsan_atomic64_compare_exchange_val /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:850 (catalog_manager-test+0x31a018)
#1 bool std::__1::__cxx_atomic_compare_exchange_weak<long>(std::__1::__cxx_atomic_base_impl<long>*, long*, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1031:12 (liblog.so+0x8bad4)
#2 std::__1::__atomic_base<long, false>::compare_exchange_weak(long&, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1622:17 (liblog.so+0x8b8f9)
#3 void kudu::AtomicStoreMax<long>(std::__1::atomic<long>&, long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/atomic-utils.h:33:13 (liblog.so+0x7e850)
#4 kudu::HighWaterMark::UpdateMax(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:80:5 (libkudu_util.so+0x3a5bd4)
#5 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:5 (libkudu_util.so+0x3a0a3d)
#6 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x39e0a1)
#7 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d5f0)
#8 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416652)
#9 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415de3)
#10 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419ff1)
#11 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419fa9)
#12 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419f39)
#13 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419f01)
#14 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41912d)
#15 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#16 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#17 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#18 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#19 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#21 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#22 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#23 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#24 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#25 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e4ed2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d72d6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41ded0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d8ae)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d1717)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1d5e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b3bc7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Mutex M197167946966685248 is already destroyed.
Thread T75 'apply [worker]-' (tid=5947, running) created by thread T95 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c39a5)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642:15 (libkudu_util.so+0x442eca)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:146:12 (libmaster.so+0x2d49b9)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45f7cb)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45da6f)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45fd32)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x415180)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x415719)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x4179aa)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417938)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x4178b8)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417873)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x416a8c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3e9ff6)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4848)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa040)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x181723)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b014)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138534)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1384e3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138458)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138413)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x1376fc)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x161226)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x1491eb)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x1339bd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x137395)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137329)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1372b9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137281)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13657d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser: WARNING: ThreadSanitizer: data race (pid=4824) Write of size 8 at 0x7b4400112040 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x33fc69)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34bb99)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34bb39)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29b982)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29b919)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a6f9d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a6940)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41cc24)
#11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cfa05)
#12 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cfc19)
#13 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1000f7)
#14 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x1000b9)
#15 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100079)
#16 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0xfffda)
#17 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc329)
#18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc300)
#19 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc2c0)
#20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b09a6)
#21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b08c4)
#22 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b067b)
#23 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x288d61)
#24 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b3cce)
#25 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#26 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#27 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#28 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#29 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#30 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#31 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#32 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#33 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#34 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#35 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#36 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#37 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#38 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#39 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#40 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#41 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#42 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#43 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#44 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#45 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#46 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#47 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Previous read of size 8 at 0x7b4400112040 by thread T75 (mutexes: write M197167946966685248):
#0 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::begin() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:1520:30 (libkudu_util.so+0x3a0939)
#1 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:284:22 (libkudu_util.so+0x39e04a)
#2 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d5f0)
#3 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416652)
#4 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415de3)
#5 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419ff1)
#6 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419fa9)
#7 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419f39)
#8 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419f01)
#9 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41912d)
#10 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#11 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#12 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#13 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#14 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#15 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#16 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#17 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#18 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#19 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#20 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e4ed2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d72d6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41ded0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d8ae)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d1717)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1d5e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b3bc7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Mutex M197167946966685248 is already destroyed.
Thread T75 'apply [worker]-' (tid=5947, running) created by thread T95 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c39a5)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642:15 (libkudu_util.so+0x442eca)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:146:12 (libmaster.so+0x2d49b9)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45f7cb)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45da6f)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45fd32)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x415180)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x415719)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x4179aa)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417938)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x4178b8)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417873)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x416a8c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3e9ff6)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4848)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa040)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x181723)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b014)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138534)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1384e3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138458)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138413)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x1376fc)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x161226)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x1491eb)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x1339bd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x137395)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137329)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1372b9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137281)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13657d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
[==========] Running 10 tests from 3 test suites.
[----------] Global test environment set-up.
[----------] 3 tests from TableInfoTest
[ RUN ] TableInfoTest.TestAssignmentRanges
WARNING: Logging before InitGoogleLogging() is written to STDERR
I20250628 07:54:24.255412 4824 catalog_manager-test.cc:126] Key found in tablet tablet--a
I20250628 07:54:24.255937 4824 catalog_manager-test.cc:126] Key a found in tablet tablet-a-b
I20250628 07:54:24.256206 4824 catalog_manager-test.cc:126] Key b found in tablet tablet-b-c
I20250628 07:54:24.256453 4824 catalog_manager-test.cc:126] Key c found in tablet tablet-c-
[ OK ] TableInfoTest.TestAssignmentRanges (7 ms)
[ RUN ] TableInfoTest.GetTableLocationsLegacyCustomHashSchemas
[ OK ] TableInfoTest.GetTableLocationsLegacyCustomHashSchemas (1 ms)
[ RUN ] TableInfoTest.MaxReturnedLocationsNotSpecified
[ OK ] TableInfoTest.MaxReturnedLocationsNotSpecified (46 ms)
[----------] 3 tests from TableInfoTest (55 ms total)
[----------] 1 test from TestTSDescriptor
[ RUN ] TestTSDescriptor.TestReplicaCreationsDecay
[ OK ] TestTSDescriptor.TestReplicaCreationsDecay (10 ms)
[----------] 1 test from TestTSDescriptor (10 ms total)
[----------] 6 tests from CatalogManagerRpcAndUserFunctionsTest
[ RUN ] CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable
I20250628 07:54:24.321696 4824 internal_mini_cluster.cc:156] Creating distributed mini masters. Addrs: 127.4.182.62:42061
I20250628 07:54:24.323076 4824 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20250628 07:54:24.324162 4824 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:54:24.338053 4831 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:24.339535 4833 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:24.338012 4830 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:25.521131 4832 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
I20250628 07:54:25.521148 4824 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
I20250628 07:54:25.525508 4824 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250628 07:54:25.525749 4824 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250628 07:54:25.525902 4824 hybrid_clock.cc:648] HybridClock initialized: now 1751097265525884 us; error 0 us; skew 500 ppm
I20250628 07:54:25.526681 4824 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:54:25.537281 4824 webserver.cc:469] Webserver started at http://127.4.182.62:40225/ using document root <none> and password file <none>
I20250628 07:54:25.538143 4824 fs_manager.cc:362] Metadata directory not provided
I20250628 07:54:25.538331 4824 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:54:25.538704 4824 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:54:25.544406 4824 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1751097264090342-4824-0/minicluster-data/master-0-root/instance:
uuid: "87f43170b89d4d59814b739f53b3c8d0"
format_stamp: "Formatted at 2025-06-28 07:54:25 on dist-test-slave-v4r1"
I20250628 07:54:25.552099 4824 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.003s sys 0.004s
I20250628 07:54:25.557389 4839 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:25.558511 4824 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
I20250628 07:54:25.558866 4824 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1751097264090342-4824-0/minicluster-data/master-0-root
uuid: "87f43170b89d4d59814b739f53b3c8d0"
format_stamp: "Formatted at 2025-06-28 07:54:25 on dist-test-slave-v4r1"
I20250628 07:54:25.559224 4824 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1751097264090342-4824-0/minicluster-data/master-0-root
metadata directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1751097264090342-4824-0/minicluster-data/master-0-root
1 data directories: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1751097264090342-4824-0/minicluster-data/master-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:54:25.618896 4824 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250628 07:54:25.620513 4824 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20250628 07:54:25.620967 4824 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:54:25.691998 4824 rpc_server.cc:307] RPC server started. Bound to: 127.4.182.62:42061
I20250628 07:54:25.692103 4890 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.182.62:42061 every 8 connection(s)
I20250628 07:54:25.699103 4891 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:25.721120 4891 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0: Bootstrap starting.
I20250628 07:54:25.726998 4891 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:25.728826 4891 log.cc:826] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0: Log is configured to *not* fsync() on all Append() calls
I20250628 07:54:25.733260 4891 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0: No bootstrap required, opened a new log
I20250628 07:54:25.753724 4891 raft_consensus.cc:357] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "87f43170b89d4d59814b739f53b3c8d0" member_type: VOTER }
I20250628 07:54:25.754267 4891 raft_consensus.cc:383] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:25.754460 4891 raft_consensus.cc:738] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 87f43170b89d4d59814b739f53b3c8d0, State: Initialized, Role: FOLLOWER
I20250628 07:54:25.755231 4891 consensus_queue.cc:260] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "87f43170b89d4d59814b739f53b3c8d0" member_type: VOTER }
I20250628 07:54:25.755744 4891 raft_consensus.cc:397] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:25.755985 4891 raft_consensus.cc:491] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:25.756295 4891 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:25.760360 4891 raft_consensus.cc:513] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "87f43170b89d4d59814b739f53b3c8d0" member_type: VOTER }
I20250628 07:54:25.761044 4891 leader_election.cc:304] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 87f43170b89d4d59814b739f53b3c8d0; no voters:
I20250628 07:54:25.762719 4891 leader_election.cc:290] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:25.763044 4894 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:25.765751 4894 raft_consensus.cc:695] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [term 1 LEADER]: Becoming Leader. State: Replica: 87f43170b89d4d59814b739f53b3c8d0, State: Running, Role: LEADER
I20250628 07:54:25.766631 4894 consensus_queue.cc:237] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "87f43170b89d4d59814b739f53b3c8d0" member_type: VOTER }
I20250628 07:54:25.767745 4891 sys_catalog.cc:564] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [sys.catalog]: configured and running, proceeding with master startup.
I20250628 07:54:25.774120 4895 sys_catalog.cc:455] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 87f43170b89d4d59814b739f53b3c8d0. Latest consensus state: current_term: 1 leader_uuid: "87f43170b89d4d59814b739f53b3c8d0" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "87f43170b89d4d59814b739f53b3c8d0" member_type: VOTER } }
I20250628 07:54:25.774911 4895 sys_catalog.cc:458] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [sys.catalog]: This master's current role is: LEADER
I20250628 07:54:25.774783 4896 sys_catalog.cc:455] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "87f43170b89d4d59814b739f53b3c8d0" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "87f43170b89d4d59814b739f53b3c8d0" member_type: VOTER } }
I20250628 07:54:25.775710 4896 sys_catalog.cc:458] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [sys.catalog]: This master's current role is: LEADER
I20250628 07:54:25.779950 4901 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250628 07:54:25.792446 4901 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250628 07:54:25.794682 4824 internal_mini_cluster.cc:184] Waiting to initialize catalog manager on master 0
I20250628 07:54:25.807914 4901 catalog_manager.cc:1349] Generated new cluster ID: a91b8142b7c84809a89bc54e8e0af575
I20250628 07:54:25.808231 4901 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250628 07:54:25.850266 4901 catalog_manager.cc:1372] Generated new certificate authority record
I20250628 07:54:25.851820 4901 catalog_manager.cc:1506] Loading token signing keys...
I20250628 07:54:25.863724 4901 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0: Generated new TSK 0
I20250628 07:54:25.864626 4901 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250628 07:54:25.925580 4824 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:54:25.933714 4913 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:25.934803 4914 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:25.938741 4916 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:54:25.939162 4824 server_base.cc:1048] running on GCE node
I20250628 07:54:25.939985 4824 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250628 07:54:25.940201 4824 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250628 07:54:25.940384 4824 hybrid_clock.cc:648] HybridClock initialized: now 1751097265940365 us; error 0 us; skew 500 ppm
I20250628 07:54:25.940941 4824 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:54:25.945390 4824 webserver.cc:469] Webserver started at http://127.4.182.1:44309/ using document root <none> and password file <none>
I20250628 07:54:25.945930 4824 fs_manager.cc:362] Metadata directory not provided
I20250628 07:54:25.946115 4824 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:54:25.946389 4824 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:54:25.947520 4824 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1751097264090342-4824-0/minicluster-data/ts-0-root/instance:
uuid: "ff68db7d7d94460abbc1812869223d47"
format_stamp: "Formatted at 2025-06-28 07:54:25 on dist-test-slave-v4r1"
I20250628 07:54:25.953053 4824 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20250628 07:54:25.957528 4921 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:25.958305 4824 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20250628 07:54:25.958581 4824 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1751097264090342-4824-0/minicluster-data/ts-0-root
uuid: "ff68db7d7d94460abbc1812869223d47"
format_stamp: "Formatted at 2025-06-28 07:54:25 on dist-test-slave-v4r1"
I20250628 07:54:25.958855 4824 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1751097264090342-4824-0/minicluster-data/ts-0-root
metadata directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1751097264090342-4824-0/minicluster-data/ts-0-root
1 data directories: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1751097264090342-4824-0/minicluster-data/ts-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:54:25.977207 4824 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250628 07:54:25.978504 4824 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:54:25.981241 4824 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20250628 07:54:25.985913 4824 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20250628 07:54:25.986135 4824 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:25.986411 4824 ts_tablet_manager.cc:610] Registered 0 tablets
I20250628 07:54:25.986565 4824 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:26.090669 4824 rpc_server.cc:307] RPC server started. Bound to: 127.4.182.1:42291
I20250628 07:54:26.090746 4983 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.182.1:42291 every 8 connection(s)
I20250628 07:54:26.115676 4984 heartbeater.cc:344] Connected to a master server at 127.4.182.62:42061
I20250628 07:54:26.116179 4984 heartbeater.cc:461] Registering TS with master...
I20250628 07:54:26.117350 4984 heartbeater.cc:507] Master 127.4.182.62:42061 requested a full tablet report, sending...
I20250628 07:54:26.120536 4856 ts_manager.cc:194] Registered new tserver with Master: ff68db7d7d94460abbc1812869223d47 (127.4.182.1:42291)
I20250628 07:54:26.121199 4824 internal_mini_cluster.cc:371] 1 TS(s) registered with all masters after 0.025009901s
I20250628 07:54:26.123576 4856 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.0.1:47674
I20250628 07:54:26.153928 4856 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47688:
name: "test_table"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
rows: "\004\001d\000\000\000\004\001\310\000\000\000\004\001,\001\000\000\004\001\220\001\000\000\004\001\364\001\000\000\004\001X\002\000\000\004\001\274\002\000\000\004\001 \003\000\000\004\001\204\003\000\000""\004\001d\000\000\000\004\001\310\000\000\000\004\001,\001\000\000\004\001\220\001\000\000\004\001\364\001\000\000\004\001X\002\000\000\004\001\274\002\000\000\004\001 \003\000\000\004\001\204\003\000\000"
indirect_data: """"
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
I20250628 07:54:26.229334 4949 tablet_service.cc:1468] Processing CreateTablet for tablet 78bb9f01effc4aa8824a85d9ff8bc783 (DEFAULT_TABLE table=test_table [id=37152aa14bbe44ee9b4bc9e1cdd68381]), partition=RANGE (key) PARTITION VALUES < 100
I20250628 07:54:26.229310 4948 tablet_service.cc:1468] Processing CreateTablet for tablet 48989e93540a47b993f55765275ab9a8 (DEFAULT_TABLE table=test_table [id=37152aa14bbe44ee9b4bc9e1cdd68381]), partition=RANGE (key) PARTITION 100 <= VALUES < 200
I20250628 07:54:26.233659 4949 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 78bb9f01effc4aa8824a85d9ff8bc783. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:26.235982 4946 tablet_service.cc:1468] Processing CreateTablet for tablet d4661b10e740408b8a4404eef06f5385 (DEFAULT_TABLE table=test_table [id=37152aa14bbe44ee9b4bc9e1cdd68381]), partition=RANGE (key) PARTITION 300 <= VALUES < 400
I20250628 07:54:26.237306 4948 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 48989e93540a47b993f55765275ab9a8. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:26.236202 4947 tablet_service.cc:1468] Processing CreateTablet for tablet b017f60bc4c0412ca69f1f57df5f6622 (DEFAULT_TABLE table=test_table [id=37152aa14bbe44ee9b4bc9e1cdd68381]), partition=RANGE (key) PARTITION 200 <= VALUES < 300
I20250628 07:54:26.239010 4942 tablet_service.cc:1468] Processing CreateTablet for tablet 5e4bce40be5f4ef89c3eb46ea5d5cd90 (DEFAULT_TABLE table=test_table [id=37152aa14bbe44ee9b4bc9e1cdd68381]), partition=RANGE (key) PARTITION 700 <= VALUES < 800
I20250628 07:54:26.240154 4946 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d4661b10e740408b8a4404eef06f5385. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:26.237185 4944 tablet_service.cc:1468] Processing CreateTablet for tablet 5541c71c3bea4bdf8d17310c674b6c18 (DEFAULT_TABLE table=test_table [id=37152aa14bbe44ee9b4bc9e1cdd68381]), partition=RANGE (key) PARTITION 500 <= VALUES < 600
I20250628 07:54:26.234997 4943 tablet_service.cc:1468] Processing CreateTablet for tablet ed28c5860a784f809cb98c2a78cce85a (DEFAULT_TABLE table=test_table [id=37152aa14bbe44ee9b4bc9e1cdd68381]), partition=RANGE (key) PARTITION 600 <= VALUES < 700
I20250628 07:54:26.241695 4940 tablet_service.cc:1468] Processing CreateTablet for tablet cbef5daaa279424298191f17d9e7c430 (DEFAULT_TABLE table=test_table [id=37152aa14bbe44ee9b4bc9e1cdd68381]), partition=RANGE (key) PARTITION 900 <= VALUES
I20250628 07:54:26.238029 4941 tablet_service.cc:1468] Processing CreateTablet for tablet 2ca1de1a3241424ea10b453450de8f56 (DEFAULT_TABLE table=test_table [id=37152aa14bbe44ee9b4bc9e1cdd68381]), partition=RANGE (key) PARTITION 800 <= VALUES < 900
I20250628 07:54:26.236155 4945 tablet_service.cc:1468] Processing CreateTablet for tablet 596e010d71324e01825f5e0cf3197a3e (DEFAULT_TABLE table=test_table [id=37152aa14bbe44ee9b4bc9e1cdd68381]), partition=RANGE (key) PARTITION 400 <= VALUES < 500
I20250628 07:54:26.242866 4942 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5e4bce40be5f4ef89c3eb46ea5d5cd90. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:26.245909 4947 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b017f60bc4c0412ca69f1f57df5f6622. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:26.248497 4940 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet cbef5daaa279424298191f17d9e7c430. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:26.250560 4945 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 596e010d71324e01825f5e0cf3197a3e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:26.252261 4943 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ed28c5860a784f809cb98c2a78cce85a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:26.254666 4941 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2ca1de1a3241424ea10b453450de8f56. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:26.257004 4944 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5541c71c3bea4bdf8d17310c674b6c18. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:26.300009 4996 tablet_bootstrap.cc:492] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47: Bootstrap starting.
I20250628 07:54:26.310572 4996 tablet_bootstrap.cc:654] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:26.321520 4996 tablet_bootstrap.cc:492] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47: No bootstrap required, opened a new log
I20250628 07:54:26.321880 4996 ts_tablet_manager.cc:1397] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47: Time spent bootstrapping tablet: real 0.022s user 0.012s sys 0.005s
I20250628 07:54:26.323880 4996 raft_consensus.cc:357] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.324321 4996 raft_consensus.cc:383] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:26.324540 4996 raft_consensus.cc:738] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Initialized, Role: FOLLOWER
I20250628 07:54:26.325042 4996 consensus_queue.cc:260] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.325484 4996 raft_consensus.cc:397] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:26.325735 4996 raft_consensus.cc:491] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:26.326046 4996 raft_consensus.cc:3058] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:26.330395 4996 raft_consensus.cc:513] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.330955 4996 leader_election.cc:304] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: ff68db7d7d94460abbc1812869223d47; no voters:
I20250628 07:54:26.332887 4996 leader_election.cc:290] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:26.333173 4998 raft_consensus.cc:2802] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:26.335981 4984 heartbeater.cc:499] Master 127.4.182.62:42061 was elected leader, sending a full tablet report...
I20250628 07:54:26.336647 4996 ts_tablet_manager.cc:1428] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47: Time spent starting tablet: real 0.015s user 0.010s sys 0.005s
I20250628 07:54:26.336750 4998 raft_consensus.cc:695] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Becoming Leader. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Running, Role: LEADER
I20250628 07:54:26.337553 4996 tablet_bootstrap.cc:492] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47: Bootstrap starting.
I20250628 07:54:26.337615 4998 consensus_queue.cc:237] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.344635 4996 tablet_bootstrap.cc:654] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47: Neither blocks nor log segments found. Creating new log.
W20250628 07:54:26.346854 4985 tablet.cc:2378] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20250628 07:54:26.356977 4996 tablet_bootstrap.cc:492] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47: No bootstrap required, opened a new log
I20250628 07:54:26.357410 4996 ts_tablet_manager.cc:1397] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47: Time spent bootstrapping tablet: real 0.020s user 0.009s sys 0.008s
I20250628 07:54:26.360045 4996 raft_consensus.cc:357] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.360733 4996 raft_consensus.cc:383] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:26.361075 4996 raft_consensus.cc:738] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Initialized, Role: FOLLOWER
I20250628 07:54:26.360800 4856 catalog_manager.cc:5582] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47 reported cstate change: term changed from 0 to 1, leader changed from <none> to ff68db7d7d94460abbc1812869223d47 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "ff68db7d7d94460abbc1812869223d47" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:26.361699 4996 consensus_queue.cc:260] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.362287 4996 raft_consensus.cc:397] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:26.362598 4996 raft_consensus.cc:491] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:26.362931 4996 raft_consensus.cc:3058] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:26.369450 4996 raft_consensus.cc:513] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.370092 4996 leader_election.cc:304] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: ff68db7d7d94460abbc1812869223d47; no voters:
I20250628 07:54:26.370793 5000 raft_consensus.cc:2802] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:26.371219 5000 raft_consensus.cc:695] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Becoming Leader. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Running, Role: LEADER
I20250628 07:54:26.371907 5000 consensus_queue.cc:237] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.376691 4996 leader_election.cc:290] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:26.379204 4996 ts_tablet_manager.cc:1428] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47: Time spent starting tablet: real 0.022s user 0.015s sys 0.001s
I20250628 07:54:26.380021 4996 tablet_bootstrap.cc:492] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47: Bootstrap starting.
I20250628 07:54:26.386138 4996 tablet_bootstrap.cc:654] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:26.386834 4856 catalog_manager.cc:5582] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47 reported cstate change: term changed from 0 to 1, leader changed from <none> to ff68db7d7d94460abbc1812869223d47 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "ff68db7d7d94460abbc1812869223d47" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:26.391994 4996 tablet_bootstrap.cc:492] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47: No bootstrap required, opened a new log
I20250628 07:54:26.392423 4996 ts_tablet_manager.cc:1397] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47: Time spent bootstrapping tablet: real 0.013s user 0.007s sys 0.004s
I20250628 07:54:26.394982 4996 raft_consensus.cc:357] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.395488 4996 raft_consensus.cc:383] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:26.395758 4996 raft_consensus.cc:738] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Initialized, Role: FOLLOWER
I20250628 07:54:26.396364 4996 consensus_queue.cc:260] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.397007 4996 raft_consensus.cc:397] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:26.397295 4996 raft_consensus.cc:491] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:26.397630 4996 raft_consensus.cc:3058] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:26.403800 4996 raft_consensus.cc:513] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.404444 4996 leader_election.cc:304] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: ff68db7d7d94460abbc1812869223d47; no voters:
I20250628 07:54:26.405026 4996 leader_election.cc:290] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:26.405146 5000 raft_consensus.cc:2802] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:26.405586 5000 raft_consensus.cc:695] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Becoming Leader. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Running, Role: LEADER
I20250628 07:54:26.406252 5000 consensus_queue.cc:237] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.407019 4996 ts_tablet_manager.cc:1428] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47: Time spent starting tablet: real 0.014s user 0.014s sys 0.000s
I20250628 07:54:26.407815 4996 tablet_bootstrap.cc:492] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47: Bootstrap starting.
I20250628 07:54:26.413535 4856 catalog_manager.cc:5582] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47 reported cstate change: term changed from 0 to 1, leader changed from <none> to ff68db7d7d94460abbc1812869223d47 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "ff68db7d7d94460abbc1812869223d47" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:26.413887 4996 tablet_bootstrap.cc:654] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:26.419762 4996 tablet_bootstrap.cc:492] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47: No bootstrap required, opened a new log
I20250628 07:54:26.420188 4996 ts_tablet_manager.cc:1397] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47: Time spent bootstrapping tablet: real 0.013s user 0.008s sys 0.003s
I20250628 07:54:26.422698 4996 raft_consensus.cc:357] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.423223 4996 raft_consensus.cc:383] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:26.423532 4996 raft_consensus.cc:738] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Initialized, Role: FOLLOWER
I20250628 07:54:26.424659 4996 consensus_queue.cc:260] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.425266 4996 raft_consensus.cc:397] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:26.425558 4996 raft_consensus.cc:491] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:26.425865 4996 raft_consensus.cc:3058] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:26.432214 4996 raft_consensus.cc:513] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.432885 4996 leader_election.cc:304] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: ff68db7d7d94460abbc1812869223d47; no voters:
I20250628 07:54:26.433468 4996 leader_election.cc:290] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:26.433913 5000 raft_consensus.cc:2802] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:26.434376 5000 raft_consensus.cc:695] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Becoming Leader. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Running, Role: LEADER
I20250628 07:54:26.435246 4996 ts_tablet_manager.cc:1428] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47: Time spent starting tablet: real 0.015s user 0.012s sys 0.000s
I20250628 07:54:26.435039 5000 consensus_queue.cc:237] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.436160 4996 tablet_bootstrap.cc:492] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47: Bootstrap starting.
I20250628 07:54:26.442654 4996 tablet_bootstrap.cc:654] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:26.443976 4855 catalog_manager.cc:5582] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47 reported cstate change: term changed from 0 to 1, leader changed from <none> to ff68db7d7d94460abbc1812869223d47 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "ff68db7d7d94460abbc1812869223d47" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:26.448717 4996 tablet_bootstrap.cc:492] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47: No bootstrap required, opened a new log
I20250628 07:54:26.449120 4996 ts_tablet_manager.cc:1397] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47: Time spent bootstrapping tablet: real 0.013s user 0.007s sys 0.004s
I20250628 07:54:26.452888 4996 raft_consensus.cc:357] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.453437 4996 raft_consensus.cc:383] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:26.453732 4996 raft_consensus.cc:738] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Initialized, Role: FOLLOWER
I20250628 07:54:26.454319 4996 consensus_queue.cc:260] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.454928 4996 raft_consensus.cc:397] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:26.455224 4996 raft_consensus.cc:491] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:26.455574 4996 raft_consensus.cc:3058] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:26.461824 4996 raft_consensus.cc:513] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.462483 4996 leader_election.cc:304] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: ff68db7d7d94460abbc1812869223d47; no voters:
I20250628 07:54:26.463063 4996 leader_election.cc:290] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:26.463191 5000 raft_consensus.cc:2802] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:26.463655 5000 raft_consensus.cc:695] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Becoming Leader. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Running, Role: LEADER
I20250628 07:54:26.464347 5000 consensus_queue.cc:237] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.464876 4996 ts_tablet_manager.cc:1428] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47: Time spent starting tablet: real 0.015s user 0.015s sys 0.000s
I20250628 07:54:26.465713 4996 tablet_bootstrap.cc:492] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47: Bootstrap starting.
I20250628 07:54:26.471042 4855 catalog_manager.cc:5582] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47 reported cstate change: term changed from 0 to 1, leader changed from <none> to ff68db7d7d94460abbc1812869223d47 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "ff68db7d7d94460abbc1812869223d47" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:26.474454 4996 tablet_bootstrap.cc:654] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:26.479822 4996 tablet_bootstrap.cc:492] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47: No bootstrap required, opened a new log
I20250628 07:54:26.480206 4996 ts_tablet_manager.cc:1397] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47: Time spent bootstrapping tablet: real 0.015s user 0.011s sys 0.000s
I20250628 07:54:26.482460 4996 raft_consensus.cc:357] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.482935 4996 raft_consensus.cc:383] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:26.483177 4996 raft_consensus.cc:738] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Initialized, Role: FOLLOWER
I20250628 07:54:26.483752 4996 consensus_queue.cc:260] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.484334 4996 raft_consensus.cc:397] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:26.484606 4996 raft_consensus.cc:491] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:26.484874 4996 raft_consensus.cc:3058] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:26.490360 4996 raft_consensus.cc:513] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.490954 4996 leader_election.cc:304] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: ff68db7d7d94460abbc1812869223d47; no voters:
I20250628 07:54:26.491542 4996 leader_election.cc:290] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:26.491663 5000 raft_consensus.cc:2802] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:26.492089 5000 raft_consensus.cc:695] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Becoming Leader. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Running, Role: LEADER
I20250628 07:54:26.492789 5000 consensus_queue.cc:237] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.493067 4996 ts_tablet_manager.cc:1428] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47: Time spent starting tablet: real 0.013s user 0.012s sys 0.000s
I20250628 07:54:26.493813 4996 tablet_bootstrap.cc:492] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47: Bootstrap starting.
I20250628 07:54:26.499704 4996 tablet_bootstrap.cc:654] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:26.500442 4855 catalog_manager.cc:5582] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47 reported cstate change: term changed from 0 to 1, leader changed from <none> to ff68db7d7d94460abbc1812869223d47 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "ff68db7d7d94460abbc1812869223d47" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:26.505220 4996 tablet_bootstrap.cc:492] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47: No bootstrap required, opened a new log
I20250628 07:54:26.505611 4996 ts_tablet_manager.cc:1397] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47: Time spent bootstrapping tablet: real 0.012s user 0.010s sys 0.000s
I20250628 07:54:26.507902 4996 raft_consensus.cc:357] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.508378 4996 raft_consensus.cc:383] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:26.508643 4996 raft_consensus.cc:738] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Initialized, Role: FOLLOWER
I20250628 07:54:26.509166 4996 consensus_queue.cc:260] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.509722 4996 raft_consensus.cc:397] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:26.509976 4996 raft_consensus.cc:491] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:26.510257 4996 raft_consensus.cc:3058] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:26.515767 4996 raft_consensus.cc:513] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.516402 4996 leader_election.cc:304] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: ff68db7d7d94460abbc1812869223d47; no voters:
I20250628 07:54:26.516947 4996 leader_election.cc:290] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:26.517094 5000 raft_consensus.cc:2802] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:26.517625 5000 raft_consensus.cc:695] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Becoming Leader. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Running, Role: LEADER
I20250628 07:54:26.518311 5000 consensus_queue.cc:237] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.518533 4996 ts_tablet_manager.cc:1428] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47: Time spent starting tablet: real 0.013s user 0.011s sys 0.000s
I20250628 07:54:26.519728 4996 tablet_bootstrap.cc:492] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47: Bootstrap starting.
I20250628 07:54:26.525121 4996 tablet_bootstrap.cc:654] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:26.525792 4855 catalog_manager.cc:5582] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47 reported cstate change: term changed from 0 to 1, leader changed from <none> to ff68db7d7d94460abbc1812869223d47 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "ff68db7d7d94460abbc1812869223d47" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:26.530750 4996 tablet_bootstrap.cc:492] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47: No bootstrap required, opened a new log
I20250628 07:54:26.531136 4996 ts_tablet_manager.cc:1397] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47: Time spent bootstrapping tablet: real 0.012s user 0.009s sys 0.000s
I20250628 07:54:26.533373 4996 raft_consensus.cc:357] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.533847 4996 raft_consensus.cc:383] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:26.534094 4996 raft_consensus.cc:738] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Initialized, Role: FOLLOWER
I20250628 07:54:26.534619 4996 consensus_queue.cc:260] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.535152 4996 raft_consensus.cc:397] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:26.535441 4996 raft_consensus.cc:491] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:26.535755 4996 raft_consensus.cc:3058] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:26.541322 4996 raft_consensus.cc:513] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.541921 4996 leader_election.cc:304] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: ff68db7d7d94460abbc1812869223d47; no voters:
I20250628 07:54:26.542448 4996 leader_election.cc:290] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:26.542560 5000 raft_consensus.cc:2802] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:26.543200 5000 raft_consensus.cc:695] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Becoming Leader. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Running, Role: LEADER
I20250628 07:54:26.543833 5000 consensus_queue.cc:237] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.544050 4996 ts_tablet_manager.cc:1428] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47: Time spent starting tablet: real 0.013s user 0.014s sys 0.000s
I20250628 07:54:26.544845 4996 tablet_bootstrap.cc:492] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47: Bootstrap starting.
I20250628 07:54:26.550612 4996 tablet_bootstrap.cc:654] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:26.550904 4855 catalog_manager.cc:5582] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47 reported cstate change: term changed from 0 to 1, leader changed from <none> to ff68db7d7d94460abbc1812869223d47 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "ff68db7d7d94460abbc1812869223d47" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:26.555974 4996 tablet_bootstrap.cc:492] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47: No bootstrap required, opened a new log
I20250628 07:54:26.556363 4996 ts_tablet_manager.cc:1397] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47: Time spent bootstrapping tablet: real 0.012s user 0.002s sys 0.008s
I20250628 07:54:26.558117 4996 raft_consensus.cc:357] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.558485 4996 raft_consensus.cc:383] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:26.558676 4996 raft_consensus.cc:738] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Initialized, Role: FOLLOWER
I20250628 07:54:26.559244 4996 consensus_queue.cc:260] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.559724 4996 raft_consensus.cc:397] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:26.559919 4996 raft_consensus.cc:491] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:26.560171 4996 raft_consensus.cc:3058] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:26.565363 4996 raft_consensus.cc:513] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.565897 4996 leader_election.cc:304] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: ff68db7d7d94460abbc1812869223d47; no voters:
I20250628 07:54:26.566287 4996 leader_election.cc:290] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:26.566417 5000 raft_consensus.cc:2802] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:26.566829 5000 raft_consensus.cc:695] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Becoming Leader. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Running, Role: LEADER
I20250628 07:54:26.567507 5000 consensus_queue.cc:237] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.568140 4996 ts_tablet_manager.cc:1428] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47: Time spent starting tablet: real 0.012s user 0.012s sys 0.000s
I20250628 07:54:26.568883 4996 tablet_bootstrap.cc:492] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47: Bootstrap starting.
I20250628 07:54:26.574672 4996 tablet_bootstrap.cc:654] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:26.575235 4855 catalog_manager.cc:5582] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47 reported cstate change: term changed from 0 to 1, leader changed from <none> to ff68db7d7d94460abbc1812869223d47 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "ff68db7d7d94460abbc1812869223d47" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:26.580387 4996 tablet_bootstrap.cc:492] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47: No bootstrap required, opened a new log
I20250628 07:54:26.580708 4996 ts_tablet_manager.cc:1397] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47: Time spent bootstrapping tablet: real 0.012s user 0.006s sys 0.004s
I20250628 07:54:26.582536 4996 raft_consensus.cc:357] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.582890 4996 raft_consensus.cc:383] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:26.583081 4996 raft_consensus.cc:738] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Initialized, Role: FOLLOWER
I20250628 07:54:26.583640 4996 consensus_queue.cc:260] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.584201 4996 raft_consensus.cc:397] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:26.584476 4996 raft_consensus.cc:491] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:26.584767 4996 raft_consensus.cc:3058] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:26.588814 4996 raft_consensus.cc:513] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.589282 4996 leader_election.cc:304] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: ff68db7d7d94460abbc1812869223d47; no voters:
I20250628 07:54:26.589687 4996 leader_election.cc:290] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:26.589819 5000 raft_consensus.cc:2802] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:26.591265 4996 ts_tablet_manager.cc:1428] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47: Time spent starting tablet: real 0.010s user 0.008s sys 0.000s
I20250628 07:54:26.591341 5000 raft_consensus.cc:695] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Becoming Leader. State: Replica: ff68db7d7d94460abbc1812869223d47, State: Running, Role: LEADER
I20250628 07:54:26.592007 5000 consensus_queue.cc:237] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } }
I20250628 07:54:26.598901 4855 catalog_manager.cc:5582] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47 reported cstate change: term changed from 0 to 1, leader changed from <none> to ff68db7d7d94460abbc1812869223d47 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "ff68db7d7d94460abbc1812869223d47" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "ff68db7d7d94460abbc1812869223d47" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 42291 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:26.606118 4824 catalog_manager.cc:2730] Servicing DeleteTable request from internal request:
table { table_name: "test_table" }
I20250628 07:54:26.623634 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0: Sending DeleteTablet for 1 replicas of tablet 78bb9f01effc4aa8824a85d9ff8bc783
I20250628 07:54:26.624862 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0: Sending DeleteTablet for 1 replicas of tablet 48989e93540a47b993f55765275ab9a8
I20250628 07:54:26.625335 4943 tablet_service.cc:1515] Processing DeleteTablet for tablet 78bb9f01effc4aa8824a85d9ff8bc783 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:26 UTC) from {username='slave'} at 127.0.0.1:36700
I20250628 07:54:26.625510 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0: Sending DeleteTablet for 1 replicas of tablet b017f60bc4c0412ca69f1f57df5f6622
I20250628 07:54:26.626508 4942 tablet_service.cc:1515] Processing DeleteTablet for tablet 48989e93540a47b993f55765275ab9a8 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:26 UTC) from {username='slave'} at 127.0.0.1:36700
I20250628 07:54:26.626112 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0: Sending DeleteTablet for 1 replicas of tablet d4661b10e740408b8a4404eef06f5385
I20250628 07:54:26.627341 4942 tablet_service.cc:1515] Processing DeleteTablet for tablet b017f60bc4c0412ca69f1f57df5f6622 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:26 UTC) from {username='slave'} at 127.0.0.1:36700
I20250628 07:54:26.628227 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0: Sending DeleteTablet for 1 replicas of tablet 596e010d71324e01825f5e0cf3197a3e
I20250628 07:54:26.628350 5021 tablet_replica.cc:331] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47: stopping tablet replica
I20250628 07:54:26.628911 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0: Sending DeleteTablet for 1 replicas of tablet 5541c71c3bea4bdf8d17310c674b6c18
I20250628 07:54:26.629529 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0: Sending DeleteTablet for 1 replicas of tablet ed28c5860a784f809cb98c2a78cce85a
I20250628 07:54:26.630271 4942 tablet_service.cc:1515] Processing DeleteTablet for tablet 596e010d71324e01825f5e0cf3197a3e with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:26 UTC) from {username='slave'} at 127.0.0.1:36700
I20250628 07:54:26.630885 4940 tablet_service.cc:1515] Processing DeleteTablet for tablet 5541c71c3bea4bdf8d17310c674b6c18 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:26 UTC) from {username='slave'} at 127.0.0.1:36700
I20250628 07:54:26.631021 4943 tablet_service.cc:1515] Processing DeleteTablet for tablet d4661b10e740408b8a4404eef06f5385 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:26 UTC) from {username='slave'} at 127.0.0.1:36700
I20250628 07:54:26.630795 5021 raft_consensus.cc:2241] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:26.631582 4942 tablet_service.cc:1515] Processing DeleteTablet for tablet ed28c5860a784f809cb98c2a78cce85a with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:26 UTC) from {username='slave'} at 127.0.0.1:36700
I20250628 07:54:26.631743 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0: Sending DeleteTablet for 1 replicas of tablet 5e4bce40be5f4ef89c3eb46ea5d5cd90
I20250628 07:54:26.631933 5021 raft_consensus.cc:2270] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:26.632447 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0: Sending DeleteTablet for 1 replicas of tablet 2ca1de1a3241424ea10b453450de8f56
I20250628 07:54:26.633107 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0: Sending DeleteTablet for 1 replicas of tablet cbef5daaa279424298191f17d9e7c430
I20250628 07:54:26.633401 4942 tablet_service.cc:1515] Processing DeleteTablet for tablet 5e4bce40be5f4ef89c3eb46ea5d5cd90 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:26 UTC) from {username='slave'} at 127.0.0.1:36700
I20250628 07:54:26.635362 4943 tablet_service.cc:1515] Processing DeleteTablet for tablet cbef5daaa279424298191f17d9e7c430 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:26 UTC) from {username='slave'} at 127.0.0.1:36700
I20250628 07:54:26.635885 5021 ts_tablet_manager.cc:1905] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:54:26.636104 4942 tablet_service.cc:1515] Processing DeleteTablet for tablet 2ca1de1a3241424ea10b453450de8f56 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:26 UTC) from {username='slave'} at 127.0.0.1:36700
I20250628 07:54:26.638020 4824 tablet_server.cc:178] TabletServer@127.4.182.1:0 shutting down...
I20250628 07:54:26.650331 5021 ts_tablet_manager.cc:1918] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:54:26.650956 5021 log.cc:1199] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47: Deleting WAL directory at /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1751097264090342-4824-0/minicluster-data/ts-0-root/wals/78bb9f01effc4aa8824a85d9ff8bc783
I20250628 07:54:26.651724 5021 ts_tablet_manager.cc:1939] T 78bb9f01effc4aa8824a85d9ff8bc783 P ff68db7d7d94460abbc1812869223d47: Deleting consensus metadata
I20250628 07:54:26.656355 5021 tablet_replica.cc:331] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47: stopping tablet replica
I20250628 07:54:26.656347 4842 catalog_manager.cc:4928] TS ff68db7d7d94460abbc1812869223d47 (127.4.182.1:42291): tablet 78bb9f01effc4aa8824a85d9ff8bc783 (table test_table [id=37152aa14bbe44ee9b4bc9e1cdd68381]) successfully deleted
I20250628 07:54:26.657397 5021 raft_consensus.cc:2241] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:26.659137 5021 raft_consensus.cc:2270] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:26.662820 5021 ts_tablet_manager.cc:1905] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:54:26.670389 4824 ts_tablet_manager.cc:1500] Shutting down tablet manager...
I20250628 07:54:26.677842 5021 ts_tablet_manager.cc:1918] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:54:26.678306 5021 log.cc:1199] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47: Deleting WAL directory at /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable.1751097264090342-4824-0/minicluster-data/ts-0-root/wals/48989e93540a47b993f55765275ab9a8
I20250628 07:54:26.679090 5021 ts_tablet_manager.cc:1939] T 48989e93540a47b993f55765275ab9a8 P ff68db7d7d94460abbc1812869223d47: Deleting consensus metadata
W20250628 07:54:26.681177 4842 catalog_manager.cc:4920] TS ff68db7d7d94460abbc1812869223d47 (127.4.182.1:42291): delete failed for tablet 48989e93540a47b993f55765275ab9a8 with error code TABLET_NOT_RUNNING: Service unavailable: Tablet Manager is not running: MANAGER_QUIESCING
W20250628 07:54:26.681906 4842 proxy.cc:239] Call had error, refreshing address and retrying: Remote error: Service unavailable: Tablet server shutting down
I20250628 07:54:26.683580 4824 tablet_replica.cc:331] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47: stopping tablet replica
I20250628 07:54:26.684321 4824 raft_consensus.cc:2241] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:26.684814 4824 raft_consensus.cc:2270] T cbef5daaa279424298191f17d9e7c430 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:54:26.687796 4842 catalog_manager.cc:4671] TS ff68db7d7d94460abbc1812869223d47 (127.4.182.1:42291): DeleteTablet:TABLET_DATA_DELETED RPC failed for tablet b017f60bc4c0412ca69f1f57df5f6622: Remote error: Service unavailable: service kudu.tserver.TabletServerAdminService not registered on TabletServer
I20250628 07:54:26.688302 4824 tablet_replica.cc:331] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47: stopping tablet replica
I20250628 07:54:26.688810 4824 raft_consensus.cc:2241] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:26.689167 4824 raft_consensus.cc:2270] T b017f60bc4c0412ca69f1f57df5f6622 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:26.691629 4824 tablet_replica.cc:331] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47: stopping tablet replica
I20250628 07:54:26.692072 4824 raft_consensus.cc:2241] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:26.692406 4824 raft_consensus.cc:2270] T ed28c5860a784f809cb98c2a78cce85a P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:26.694420 4824 tablet_replica.cc:331] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47: stopping tablet replica
I20250628 07:54:26.694873 4824 raft_consensus.cc:2241] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:26.695199 4824 raft_consensus.cc:2270] T 5e4bce40be5f4ef89c3eb46ea5d5cd90 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:26.696965 4824 tablet_replica.cc:331] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47: stopping tablet replica
I20250628 07:54:26.697408 4824 raft_consensus.cc:2241] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:26.697734 4824 raft_consensus.cc:2270] T 596e010d71324e01825f5e0cf3197a3e P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:26.699648 4824 tablet_replica.cc:331] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47: stopping tablet replica
I20250628 07:54:26.700057 4824 raft_consensus.cc:2241] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:26.700398 4824 raft_consensus.cc:2270] T 2ca1de1a3241424ea10b453450de8f56 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:26.702293 4824 tablet_replica.cc:331] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47: stopping tablet replica
I20250628 07:54:26.702705 4824 raft_consensus.cc:2241] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:26.703013 4824 raft_consensus.cc:2270] T d4661b10e740408b8a4404eef06f5385 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:26.704772 4824 tablet_replica.cc:331] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47: stopping tablet replica
I20250628 07:54:26.705111 4824 raft_consensus.cc:2241] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:26.705399 4824 raft_consensus.cc:2270] T 5541c71c3bea4bdf8d17310c674b6c18 P ff68db7d7d94460abbc1812869223d47 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:26.868399 4824 tablet_server.cc:195] TabletServer@127.4.182.1:0 shutdown complete.
I20250628 07:54:26.883872 4824 master.cc:561] Master@127.4.182.62:42061 shutting down...
I20250628 07:54:27.104805 4824 raft_consensus.cc:2241] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:27.105281 4824 raft_consensus.cc:2270] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:27.105607 4824 tablet_replica.cc:331] T 00000000000000000000000000000000 P 87f43170b89d4d59814b739f53b3c8d0: stopping tablet replica
I20250628 07:54:27.124742 4824 master.cc:583] Master@127.4.182.62:42061 shutdown complete.
[ OK ] CatalogManagerRpcAndUserFunctionsTest.TestDeleteTable (2834 ms)
[ RUN ] CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser
I20250628 07:54:27.154524 4824 internal_mini_cluster.cc:156] Creating distributed mini masters. Addrs: 127.4.182.62:34207
I20250628 07:54:27.155843 4824 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:54:27.160914 5023 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:27.161329 5024 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:27.163072 5026 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:54:27.164327 4824 server_base.cc:1048] running on GCE node
I20250628 07:54:27.165324 4824 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250628 07:54:27.165576 4824 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250628 07:54:27.165757 4824 hybrid_clock.cc:648] HybridClock initialized: now 1751097267165741 us; error 0 us; skew 500 ppm
I20250628 07:54:27.166409 4824 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:54:27.169092 4824 webserver.cc:469] Webserver started at http://127.4.182.62:35441/ using document root <none> and password file <none>
I20250628 07:54:27.169695 4824 fs_manager.cc:362] Metadata directory not provided
I20250628 07:54:27.169919 4824 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:54:27.170245 4824 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:54:27.171599 4824 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1751097264090342-4824-0/minicluster-data/master-0-root/instance:
uuid: "3591ddc4ecfb4db19467e4b5f9fea55f"
format_stamp: "Formatted at 2025-06-28 07:54:27 on dist-test-slave-v4r1"
I20250628 07:54:27.177096 4824 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20250628 07:54:27.181051 5031 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:27.181833 4824 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20250628 07:54:27.182083 4824 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1751097264090342-4824-0/minicluster-data/master-0-root
uuid: "3591ddc4ecfb4db19467e4b5f9fea55f"
format_stamp: "Formatted at 2025-06-28 07:54:27 on dist-test-slave-v4r1"
I20250628 07:54:27.182324 4824 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1751097264090342-4824-0/minicluster-data/master-0-root
metadata directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1751097264090342-4824-0/minicluster-data/master-0-root
1 data directories: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1751097264090342-4824-0/minicluster-data/master-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:54:27.192399 4824 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250628 07:54:27.193264 4824 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:54:27.231359 4824 rpc_server.cc:307] RPC server started. Bound to: 127.4.182.62:34207
I20250628 07:54:27.231539 5082 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.182.62:34207 every 8 connection(s)
I20250628 07:54:27.235500 5083 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:27.244768 5083 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f: Bootstrap starting.
I20250628 07:54:27.248699 5083 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:27.252476 5083 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f: No bootstrap required, opened a new log
I20250628 07:54:27.254160 5083 raft_consensus.cc:357] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3591ddc4ecfb4db19467e4b5f9fea55f" member_type: VOTER }
I20250628 07:54:27.254559 5083 raft_consensus.cc:383] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:27.254763 5083 raft_consensus.cc:738] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3591ddc4ecfb4db19467e4b5f9fea55f, State: Initialized, Role: FOLLOWER
I20250628 07:54:27.255227 5083 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3591ddc4ecfb4db19467e4b5f9fea55f" member_type: VOTER }
I20250628 07:54:27.255681 5083 raft_consensus.cc:397] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:27.255889 5083 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:27.256134 5083 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:27.260105 5083 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3591ddc4ecfb4db19467e4b5f9fea55f" member_type: VOTER }
I20250628 07:54:27.260630 5083 leader_election.cc:304] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 3591ddc4ecfb4db19467e4b5f9fea55f; no voters:
I20250628 07:54:27.261847 5083 leader_election.cc:290] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:27.262140 5086 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:27.263679 5086 raft_consensus.cc:695] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [term 1 LEADER]: Becoming Leader. State: Replica: 3591ddc4ecfb4db19467e4b5f9fea55f, State: Running, Role: LEADER
I20250628 07:54:27.264449 5086 consensus_queue.cc:237] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3591ddc4ecfb4db19467e4b5f9fea55f" member_type: VOTER }
I20250628 07:54:27.265262 5083 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [sys.catalog]: configured and running, proceeding with master startup.
I20250628 07:54:27.267328 5087 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "3591ddc4ecfb4db19467e4b5f9fea55f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3591ddc4ecfb4db19467e4b5f9fea55f" member_type: VOTER } }
I20250628 07:54:27.267894 5087 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [sys.catalog]: This master's current role is: LEADER
I20250628 07:54:27.273130 5088 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3591ddc4ecfb4db19467e4b5f9fea55f. Latest consensus state: current_term: 1 leader_uuid: "3591ddc4ecfb4db19467e4b5f9fea55f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3591ddc4ecfb4db19467e4b5f9fea55f" member_type: VOTER } }
I20250628 07:54:27.273674 5088 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [sys.catalog]: This master's current role is: LEADER
I20250628 07:54:27.273952 5091 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250628 07:54:27.282064 5091 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250628 07:54:27.283042 4824 internal_mini_cluster.cc:184] Waiting to initialize catalog manager on master 0
I20250628 07:54:27.291126 5091 catalog_manager.cc:1349] Generated new cluster ID: 6977ed34bf8847af99a2be7be2be3eb1
I20250628 07:54:27.291432 5091 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250628 07:54:27.315306 5091 catalog_manager.cc:1372] Generated new certificate authority record
I20250628 07:54:27.316478 5091 catalog_manager.cc:1506] Loading token signing keys...
I20250628 07:54:27.329725 5091 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f: Generated new TSK 0
I20250628 07:54:27.330245 5091 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250628 07:54:27.348831 4824 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:54:27.354457 5104 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:27.355994 5105 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:27.358348 5107 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:54:27.358407 4824 server_base.cc:1048] running on GCE node
I20250628 07:54:27.359572 4824 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250628 07:54:27.359814 4824 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250628 07:54:27.359995 4824 hybrid_clock.cc:648] HybridClock initialized: now 1751097267359980 us; error 0 us; skew 500 ppm
I20250628 07:54:27.360669 4824 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:54:27.363332 4824 webserver.cc:469] Webserver started at http://127.4.182.1:35069/ using document root <none> and password file <none>
I20250628 07:54:27.363960 4824 fs_manager.cc:362] Metadata directory not provided
I20250628 07:54:27.364183 4824 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:54:27.364497 4824 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:54:27.365862 4824 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1751097264090342-4824-0/minicluster-data/ts-0-root/instance:
uuid: "407b8f0cb3024d4c94323b7e79dd8600"
format_stamp: "Formatted at 2025-06-28 07:54:27 on dist-test-slave-v4r1"
I20250628 07:54:27.370479 4824 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.006s sys 0.000s
I20250628 07:54:27.373761 5112 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:27.374572 4824 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20250628 07:54:27.374861 4824 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1751097264090342-4824-0/minicluster-data/ts-0-root
uuid: "407b8f0cb3024d4c94323b7e79dd8600"
format_stamp: "Formatted at 2025-06-28 07:54:27 on dist-test-slave-v4r1"
I20250628 07:54:27.375131 4824 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1751097264090342-4824-0/minicluster-data/ts-0-root
metadata directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1751097264090342-4824-0/minicluster-data/ts-0-root
1 data directories: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1751097264090342-4824-0/minicluster-data/ts-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:54:27.430009 4824 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250628 07:54:27.430965 4824 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:54:27.432601 4824 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20250628 07:54:27.434844 4824 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20250628 07:54:27.435007 4824 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:27.435228 4824 ts_tablet_manager.cc:610] Registered 0 tablets
I20250628 07:54:27.435365 4824 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:27.485550 4824 rpc_server.cc:307] RPC server started. Bound to: 127.4.182.1:40401
I20250628 07:54:27.485636 5174 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.182.1:40401 every 8 connection(s)
I20250628 07:54:27.499765 5175 heartbeater.cc:344] Connected to a master server at 127.4.182.62:34207
I20250628 07:54:27.500083 5175 heartbeater.cc:461] Registering TS with master...
I20250628 07:54:27.500754 5175 heartbeater.cc:507] Master 127.4.182.62:34207 requested a full tablet report, sending...
I20250628 07:54:27.502491 5048 ts_manager.cc:194] Registered new tserver with Master: 407b8f0cb3024d4c94323b7e79dd8600 (127.4.182.1:40401)
I20250628 07:54:27.502735 4824 internal_mini_cluster.cc:371] 1 TS(s) registered with all masters after 0.013934073s
I20250628 07:54:27.504134 5048 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.0.1:38534
I20250628 07:54:27.526381 5048 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38550:
name: "test_table"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
rows: "\004\001d\000\000\000\004\001\310\000\000\000\004\001,\001\000\000\004\001\220\001\000\000\004\001\364\001\000\000\004\001X\002\000\000\004\001\274\002\000\000\004\001 \003\000\000\004\001\204\003\000\000""\004\001d\000\000\000\004\001\310\000\000\000\004\001,\001\000\000\004\001\220\001\000\000\004\001\364\001\000\000\004\001X\002\000\000\004\001\274\002\000\000\004\001 \003\000\000\004\001\204\003\000\000"
indirect_data: """"
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
I20250628 07:54:27.580096 5140 tablet_service.cc:1468] Processing CreateTablet for tablet 2cc4129a5ef9405d913a93f0ec351eae (DEFAULT_TABLE table=test_table [id=a38837c585e34580ad65764e0f8805ff]), partition=RANGE (key) PARTITION VALUES < 100
I20250628 07:54:27.580350 5139 tablet_service.cc:1468] Processing CreateTablet for tablet 2e24101def284c78adb8da97a6c55f5c (DEFAULT_TABLE table=test_table [id=a38837c585e34580ad65764e0f8805ff]), partition=RANGE (key) PARTITION 100 <= VALUES < 200
I20250628 07:54:27.581017 5137 tablet_service.cc:1468] Processing CreateTablet for tablet aca138b125614a2e9fbf817669df22c7 (DEFAULT_TABLE table=test_table [id=a38837c585e34580ad65764e0f8805ff]), partition=RANGE (key) PARTITION 300 <= VALUES < 400
I20250628 07:54:27.581422 5140 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2cc4129a5ef9405d913a93f0ec351eae. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:27.582228 5137 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet aca138b125614a2e9fbf817669df22c7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:27.582157 5138 tablet_service.cc:1468] Processing CreateTablet for tablet d396c8b9f507404a8bc324ddd83408ce (DEFAULT_TABLE table=test_table [id=a38837c585e34580ad65764e0f8805ff]), partition=RANGE (key) PARTITION 200 <= VALUES < 300
I20250628 07:54:27.583467 5139 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2e24101def284c78adb8da97a6c55f5c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:27.587867 5138 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d396c8b9f507404a8bc324ddd83408ce. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:27.590101 5136 tablet_service.cc:1468] Processing CreateTablet for tablet c11c99a9810d44f2992e2c6352aa44b2 (DEFAULT_TABLE table=test_table [id=a38837c585e34580ad65764e0f8805ff]), partition=RANGE (key) PARTITION 400 <= VALUES < 500
I20250628 07:54:27.591356 5136 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c11c99a9810d44f2992e2c6352aa44b2. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:27.596928 5135 tablet_service.cc:1468] Processing CreateTablet for tablet ed0e24badb1f45118c05873c68630f24 (DEFAULT_TABLE table=test_table [id=a38837c585e34580ad65764e0f8805ff]), partition=RANGE (key) PARTITION 500 <= VALUES < 600
I20250628 07:54:27.598179 5135 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ed0e24badb1f45118c05873c68630f24. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:27.600330 5134 tablet_service.cc:1468] Processing CreateTablet for tablet 8d10e117388d4af48a4ab58bc6ebef66 (DEFAULT_TABLE table=test_table [id=a38837c585e34580ad65764e0f8805ff]), partition=RANGE (key) PARTITION 600 <= VALUES < 700
I20250628 07:54:27.601557 5134 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8d10e117388d4af48a4ab58bc6ebef66. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:27.609381 5133 tablet_service.cc:1468] Processing CreateTablet for tablet c2ec930cfa304c8abbf64cf97aa38d44 (DEFAULT_TABLE table=test_table [id=a38837c585e34580ad65764e0f8805ff]), partition=RANGE (key) PARTITION 800 <= VALUES < 900
I20250628 07:54:27.611061 5133 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c2ec930cfa304c8abbf64cf97aa38d44. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:27.622164 5132 tablet_service.cc:1468] Processing CreateTablet for tablet 60109d005bb346d1b90301381712282d (DEFAULT_TABLE table=test_table [id=a38837c585e34580ad65764e0f8805ff]), partition=RANGE (key) PARTITION 900 <= VALUES
I20250628 07:54:27.624331 5132 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 60109d005bb346d1b90301381712282d. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:27.631887 5187 tablet_bootstrap.cc:492] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600: Bootstrap starting.
I20250628 07:54:27.638041 5187 tablet_bootstrap.cc:654] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:27.639285 5140 tablet_service.cc:1468] Processing CreateTablet for tablet 0d1ffcf6f5c346668d1fc976a9cb56f9 (DEFAULT_TABLE table=test_table [id=a38837c585e34580ad65764e0f8805ff]), partition=RANGE (key) PARTITION 700 <= VALUES < 800
I20250628 07:54:27.653033 5140 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0d1ffcf6f5c346668d1fc976a9cb56f9. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:27.663132 5187 tablet_bootstrap.cc:492] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600: No bootstrap required, opened a new log
I20250628 07:54:27.663616 5187 ts_tablet_manager.cc:1397] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600: Time spent bootstrapping tablet: real 0.032s user 0.015s sys 0.007s
I20250628 07:54:27.665998 5187 raft_consensus.cc:357] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.666579 5187 raft_consensus.cc:383] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:27.666857 5187 raft_consensus.cc:738] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Initialized, Role: FOLLOWER
I20250628 07:54:27.667590 5187 consensus_queue.cc:260] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.668198 5187 raft_consensus.cc:397] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:27.668567 5187 raft_consensus.cc:491] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:27.668949 5187 raft_consensus.cc:3058] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:27.675143 5187 raft_consensus.cc:513] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.675738 5187 leader_election.cc:304] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 407b8f0cb3024d4c94323b7e79dd8600; no voters:
I20250628 07:54:27.677114 5187 leader_election.cc:290] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:27.677395 5189 raft_consensus.cc:2802] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:27.678977 5175 heartbeater.cc:499] Master 127.4.182.62:34207 was elected leader, sending a full tablet report...
I20250628 07:54:27.679235 5189 raft_consensus.cc:695] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Becoming Leader. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Running, Role: LEADER
I20250628 07:54:27.679948 5189 consensus_queue.cc:237] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.680675 5187 ts_tablet_manager.cc:1428] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600: Time spent starting tablet: real 0.017s user 0.014s sys 0.002s
I20250628 07:54:27.681562 5187 tablet_bootstrap.cc:492] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600: Bootstrap starting.
I20250628 07:54:27.687472 5187 tablet_bootstrap.cc:654] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:27.693138 5187 tablet_bootstrap.cc:492] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600: No bootstrap required, opened a new log
I20250628 07:54:27.693771 5187 ts_tablet_manager.cc:1397] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600: Time spent bootstrapping tablet: real 0.012s user 0.001s sys 0.010s
I20250628 07:54:27.693478 5048 catalog_manager.cc:5582] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600 reported cstate change: term changed from 0 to 1, leader changed from <none> to 407b8f0cb3024d4c94323b7e79dd8600 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "407b8f0cb3024d4c94323b7e79dd8600" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:27.697151 5187 raft_consensus.cc:357] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.697988 5187 raft_consensus.cc:383] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:27.698408 5187 raft_consensus.cc:738] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Initialized, Role: FOLLOWER
I20250628 07:54:27.699470 5187 consensus_queue.cc:260] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.700430 5187 raft_consensus.cc:397] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:27.700870 5187 raft_consensus.cc:491] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:27.701256 5187 raft_consensus.cc:3058] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:27.708552 5187 raft_consensus.cc:513] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.709388 5187 leader_election.cc:304] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 407b8f0cb3024d4c94323b7e79dd8600; no voters:
I20250628 07:54:27.710009 5187 leader_election.cc:290] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:27.710813 5191 raft_consensus.cc:2802] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:27.711244 5191 raft_consensus.cc:695] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Becoming Leader. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Running, Role: LEADER
I20250628 07:54:27.711830 5187 ts_tablet_manager.cc:1428] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600: Time spent starting tablet: real 0.018s user 0.016s sys 0.000s
I20250628 07:54:27.711957 5191 consensus_queue.cc:237] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.712731 5187 tablet_bootstrap.cc:492] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600: Bootstrap starting.
I20250628 07:54:27.718641 5187 tablet_bootstrap.cc:654] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:27.722263 5048 catalog_manager.cc:5582] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600 reported cstate change: term changed from 0 to 1, leader changed from <none> to 407b8f0cb3024d4c94323b7e79dd8600 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "407b8f0cb3024d4c94323b7e79dd8600" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:27.724535 5187 tablet_bootstrap.cc:492] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600: No bootstrap required, opened a new log
I20250628 07:54:27.725014 5187 ts_tablet_manager.cc:1397] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600: Time spent bootstrapping tablet: real 0.012s user 0.011s sys 0.000s
I20250628 07:54:27.727599 5187 raft_consensus.cc:357] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.728005 5187 raft_consensus.cc:383] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:27.728222 5187 raft_consensus.cc:738] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Initialized, Role: FOLLOWER
I20250628 07:54:27.728796 5187 consensus_queue.cc:260] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.729362 5187 raft_consensus.cc:397] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:27.729593 5187 raft_consensus.cc:491] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:27.729847 5187 raft_consensus.cc:3058] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:27.734577 5187 raft_consensus.cc:513] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.735157 5187 leader_election.cc:304] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 407b8f0cb3024d4c94323b7e79dd8600; no voters:
I20250628 07:54:27.736032 5191 raft_consensus.cc:2802] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:27.736479 5191 raft_consensus.cc:695] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Becoming Leader. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Running, Role: LEADER
I20250628 07:54:27.737135 5191 consensus_queue.cc:237] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.739137 5187 leader_election.cc:290] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:27.740732 5187 ts_tablet_manager.cc:1428] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600: Time spent starting tablet: real 0.015s user 0.012s sys 0.000s
I20250628 07:54:27.741526 5187 tablet_bootstrap.cc:492] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600: Bootstrap starting.
I20250628 07:54:27.743643 5048 catalog_manager.cc:5582] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600 reported cstate change: term changed from 0 to 1, leader changed from <none> to 407b8f0cb3024d4c94323b7e79dd8600 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "407b8f0cb3024d4c94323b7e79dd8600" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:27.748284 5187 tablet_bootstrap.cc:654] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:27.753455 5187 tablet_bootstrap.cc:492] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600: No bootstrap required, opened a new log
I20250628 07:54:27.753850 5187 ts_tablet_manager.cc:1397] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600: Time spent bootstrapping tablet: real 0.012s user 0.009s sys 0.000s
I20250628 07:54:27.755966 5187 raft_consensus.cc:357] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.756363 5187 raft_consensus.cc:383] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:27.756572 5187 raft_consensus.cc:738] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Initialized, Role: FOLLOWER
I20250628 07:54:27.757113 5187 consensus_queue.cc:260] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.757561 5187 raft_consensus.cc:397] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:27.757774 5187 raft_consensus.cc:491] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:27.758016 5187 raft_consensus.cc:3058] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:27.763463 5187 raft_consensus.cc:513] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.764178 5187 leader_election.cc:304] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 407b8f0cb3024d4c94323b7e79dd8600; no voters:
I20250628 07:54:27.764650 5187 leader_election.cc:290] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:27.764788 5191 raft_consensus.cc:2802] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:27.765224 5191 raft_consensus.cc:695] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Becoming Leader. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Running, Role: LEADER
I20250628 07:54:27.765930 5191 consensus_queue.cc:237] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.766672 5187 ts_tablet_manager.cc:1428] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600: Time spent starting tablet: real 0.013s user 0.012s sys 0.000s
I20250628 07:54:27.767611 5187 tablet_bootstrap.cc:492] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600: Bootstrap starting.
I20250628 07:54:27.773063 5048 catalog_manager.cc:5582] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600 reported cstate change: term changed from 0 to 1, leader changed from <none> to 407b8f0cb3024d4c94323b7e79dd8600 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "407b8f0cb3024d4c94323b7e79dd8600" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:27.774205 5187 tablet_bootstrap.cc:654] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:27.778740 5187 tablet_bootstrap.cc:492] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600: No bootstrap required, opened a new log
I20250628 07:54:27.779152 5187 ts_tablet_manager.cc:1397] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600: Time spent bootstrapping tablet: real 0.012s user 0.010s sys 0.000s
I20250628 07:54:27.781692 5187 raft_consensus.cc:357] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.782163 5187 raft_consensus.cc:383] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:27.782469 5187 raft_consensus.cc:738] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Initialized, Role: FOLLOWER
I20250628 07:54:27.783349 5187 consensus_queue.cc:260] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.784510 5187 raft_consensus.cc:397] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:27.784829 5187 raft_consensus.cc:491] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:27.785171 5187 raft_consensus.cc:3058] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:27.791412 5187 raft_consensus.cc:513] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.791993 5187 leader_election.cc:304] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 407b8f0cb3024d4c94323b7e79dd8600; no voters:
I20250628 07:54:27.792477 5187 leader_election.cc:290] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:27.792650 5191 raft_consensus.cc:2802] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:27.793123 5191 raft_consensus.cc:695] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Becoming Leader. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Running, Role: LEADER
I20250628 07:54:27.794150 5187 ts_tablet_manager.cc:1428] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600: Time spent starting tablet: real 0.015s user 0.010s sys 0.001s
I20250628 07:54:27.793833 5191 consensus_queue.cc:237] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.794847 5187 tablet_bootstrap.cc:492] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600: Bootstrap starting.
I20250628 07:54:27.800253 5187 tablet_bootstrap.cc:654] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:27.800424 5048 catalog_manager.cc:5582] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600 reported cstate change: term changed from 0 to 1, leader changed from <none> to 407b8f0cb3024d4c94323b7e79dd8600 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "407b8f0cb3024d4c94323b7e79dd8600" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:27.804431 5187 tablet_bootstrap.cc:492] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600: No bootstrap required, opened a new log
I20250628 07:54:27.804805 5187 ts_tablet_manager.cc:1397] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600: Time spent bootstrapping tablet: real 0.010s user 0.008s sys 0.000s
I20250628 07:54:27.807842 5187 raft_consensus.cc:357] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.808523 5187 raft_consensus.cc:383] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:27.808874 5187 raft_consensus.cc:738] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Initialized, Role: FOLLOWER
I20250628 07:54:27.809638 5187 consensus_queue.cc:260] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.810434 5187 raft_consensus.cc:397] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:27.810820 5187 raft_consensus.cc:491] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:27.811302 5187 raft_consensus.cc:3058] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:27.817133 5187 raft_consensus.cc:513] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.817734 5187 leader_election.cc:304] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 407b8f0cb3024d4c94323b7e79dd8600; no voters:
I20250628 07:54:27.818240 5187 leader_election.cc:290] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:27.818359 5191 raft_consensus.cc:2802] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:27.818789 5191 raft_consensus.cc:695] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Becoming Leader. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Running, Role: LEADER
I20250628 07:54:27.819504 5191 consensus_queue.cc:237] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.819806 5187 ts_tablet_manager.cc:1428] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600: Time spent starting tablet: real 0.015s user 0.016s sys 0.000s
I20250628 07:54:27.820528 5187 tablet_bootstrap.cc:492] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600: Bootstrap starting.
I20250628 07:54:27.825430 5187 tablet_bootstrap.cc:654] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:27.825956 5048 catalog_manager.cc:5582] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600 reported cstate change: term changed from 0 to 1, leader changed from <none> to 407b8f0cb3024d4c94323b7e79dd8600 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "407b8f0cb3024d4c94323b7e79dd8600" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:27.831554 5187 tablet_bootstrap.cc:492] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600: No bootstrap required, opened a new log
I20250628 07:54:27.832005 5187 ts_tablet_manager.cc:1397] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600: Time spent bootstrapping tablet: real 0.012s user 0.007s sys 0.004s
I20250628 07:54:27.834183 5187 raft_consensus.cc:357] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.834564 5187 raft_consensus.cc:383] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:27.834777 5187 raft_consensus.cc:738] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Initialized, Role: FOLLOWER
I20250628 07:54:27.835455 5187 consensus_queue.cc:260] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.836041 5187 raft_consensus.cc:397] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:27.836333 5187 raft_consensus.cc:491] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:27.836645 5187 raft_consensus.cc:3058] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:27.840629 5187 raft_consensus.cc:513] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.841109 5187 leader_election.cc:304] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 407b8f0cb3024d4c94323b7e79dd8600; no voters:
I20250628 07:54:27.841655 5187 leader_election.cc:290] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:27.841789 5191 raft_consensus.cc:2802] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:27.842269 5191 raft_consensus.cc:695] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Becoming Leader. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Running, Role: LEADER
I20250628 07:54:27.843135 5187 ts_tablet_manager.cc:1428] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600: Time spent starting tablet: real 0.011s user 0.009s sys 0.000s
I20250628 07:54:27.843025 5191 consensus_queue.cc:237] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.843835 5187 tablet_bootstrap.cc:492] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600: Bootstrap starting.
I20250628 07:54:27.848521 5187 tablet_bootstrap.cc:654] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:27.850339 5048 catalog_manager.cc:5582] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600 reported cstate change: term changed from 0 to 1, leader changed from <none> to 407b8f0cb3024d4c94323b7e79dd8600 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "407b8f0cb3024d4c94323b7e79dd8600" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:27.852119 5187 tablet_bootstrap.cc:492] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600: No bootstrap required, opened a new log
I20250628 07:54:27.852473 5187 ts_tablet_manager.cc:1397] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600: Time spent bootstrapping tablet: real 0.009s user 0.007s sys 0.000s
I20250628 07:54:27.854569 5187 raft_consensus.cc:357] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.855032 5187 raft_consensus.cc:383] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:27.855288 5187 raft_consensus.cc:738] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Initialized, Role: FOLLOWER
I20250628 07:54:27.855818 5187 consensus_queue.cc:260] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.856269 5187 raft_consensus.cc:397] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:27.856551 5187 raft_consensus.cc:491] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:27.856791 5187 raft_consensus.cc:3058] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:27.860891 5187 raft_consensus.cc:513] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.861379 5187 leader_election.cc:304] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 407b8f0cb3024d4c94323b7e79dd8600; no voters:
I20250628 07:54:27.861910 5187 leader_election.cc:290] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:27.862043 5191 raft_consensus.cc:2802] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:27.862428 5191 raft_consensus.cc:695] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Becoming Leader. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Running, Role: LEADER
I20250628 07:54:27.863488 5187 ts_tablet_manager.cc:1428] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600: Time spent starting tablet: real 0.011s user 0.012s sys 0.000s
I20250628 07:54:27.863063 5191 consensus_queue.cc:237] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.864183 5187 tablet_bootstrap.cc:492] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600: Bootstrap starting.
I20250628 07:54:27.869684 5187 tablet_bootstrap.cc:654] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:27.872202 5048 catalog_manager.cc:5582] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600 reported cstate change: term changed from 0 to 1, leader changed from <none> to 407b8f0cb3024d4c94323b7e79dd8600 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "407b8f0cb3024d4c94323b7e79dd8600" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:27.873517 5187 tablet_bootstrap.cc:492] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600: No bootstrap required, opened a new log
I20250628 07:54:27.873824 5187 ts_tablet_manager.cc:1397] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600: Time spent bootstrapping tablet: real 0.010s user 0.004s sys 0.004s
I20250628 07:54:27.875739 5187 raft_consensus.cc:357] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.876232 5187 raft_consensus.cc:383] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:27.876521 5187 raft_consensus.cc:738] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Initialized, Role: FOLLOWER
I20250628 07:54:27.877058 5187 consensus_queue.cc:260] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.877478 5187 raft_consensus.cc:397] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:27.877647 5187 raft_consensus.cc:491] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:27.877867 5187 raft_consensus.cc:3058] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:27.882027 5187 raft_consensus.cc:513] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.882503 5187 leader_election.cc:304] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 407b8f0cb3024d4c94323b7e79dd8600; no voters:
I20250628 07:54:27.882880 5187 leader_election.cc:290] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:27.883031 5189 raft_consensus.cc:2802] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:27.884136 5187 ts_tablet_manager.cc:1428] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600: Time spent starting tablet: real 0.010s user 0.011s sys 0.000s
I20250628 07:54:27.884091 5189 raft_consensus.cc:695] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Becoming Leader. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Running, Role: LEADER
I20250628 07:54:27.884822 5187 tablet_bootstrap.cc:492] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600: Bootstrap starting.
I20250628 07:54:27.884860 5189 consensus_queue.cc:237] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.889648 5187 tablet_bootstrap.cc:654] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:27.892442 5048 catalog_manager.cc:5582] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600 reported cstate change: term changed from 0 to 1, leader changed from <none> to 407b8f0cb3024d4c94323b7e79dd8600 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "407b8f0cb3024d4c94323b7e79dd8600" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:27.895433 5187 tablet_bootstrap.cc:492] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600: No bootstrap required, opened a new log
I20250628 07:54:27.895841 5187 ts_tablet_manager.cc:1397] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600: Time spent bootstrapping tablet: real 0.011s user 0.010s sys 0.000s
I20250628 07:54:27.897815 5187 raft_consensus.cc:357] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.898229 5187 raft_consensus.cc:383] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:27.898497 5187 raft_consensus.cc:738] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Initialized, Role: FOLLOWER
I20250628 07:54:27.898916 5187 consensus_queue.cc:260] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.899273 5187 raft_consensus.cc:397] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:27.899569 5187 raft_consensus.cc:491] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:27.899858 5187 raft_consensus.cc:3058] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:27.903898 5187 raft_consensus.cc:513] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.904498 5187 leader_election.cc:304] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 407b8f0cb3024d4c94323b7e79dd8600; no voters:
I20250628 07:54:27.904932 5187 leader_election.cc:290] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:27.905066 5189 raft_consensus.cc:2802] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:27.905479 5189 raft_consensus.cc:695] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Becoming Leader. State: Replica: 407b8f0cb3024d4c94323b7e79dd8600, State: Running, Role: LEADER
I20250628 07:54:27.906762 5187 ts_tablet_manager.cc:1428] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600: Time spent starting tablet: real 0.011s user 0.011s sys 0.000s
I20250628 07:54:27.906198 5189 consensus_queue.cc:237] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } }
I20250628 07:54:27.912676 5048 catalog_manager.cc:5582] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600 reported cstate change: term changed from 0 to 1, leader changed from <none> to 407b8f0cb3024d4c94323b7e79dd8600 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "407b8f0cb3024d4c94323b7e79dd8600" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "407b8f0cb3024d4c94323b7e79dd8600" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 40401 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:27.965425 4824 catalog_manager.cc:2739] Servicing DeleteTable request from test_user:
table { table_name: "test_table" }
I20250628 07:54:27.977994 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f: Sending DeleteTablet for 1 replicas of tablet 2cc4129a5ef9405d913a93f0ec351eae
I20250628 07:54:27.978622 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f: Sending DeleteTablet for 1 replicas of tablet 2e24101def284c78adb8da97a6c55f5c
I20250628 07:54:27.979192 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f: Sending DeleteTablet for 1 replicas of tablet d396c8b9f507404a8bc324ddd83408ce
I20250628 07:54:27.979524 5140 tablet_service.cc:1515] Processing DeleteTablet for tablet 2cc4129a5ef9405d913a93f0ec351eae with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:27 UTC) from {username='slave'} at 127.0.0.1:54406
I20250628 07:54:27.979768 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f: Sending DeleteTablet for 1 replicas of tablet aca138b125614a2e9fbf817669df22c7
I20250628 07:54:27.980126 5133 tablet_service.cc:1515] Processing DeleteTablet for tablet 2e24101def284c78adb8da97a6c55f5c with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:27 UTC) from {username='slave'} at 127.0.0.1:54406
I20250628 07:54:27.980397 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f: Sending DeleteTablet for 1 replicas of tablet c11c99a9810d44f2992e2c6352aa44b2
I20250628 07:54:27.980860 5132 tablet_service.cc:1515] Processing DeleteTablet for tablet d396c8b9f507404a8bc324ddd83408ce with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:27 UTC) from {username='slave'} at 127.0.0.1:54406
I20250628 07:54:27.981698 5133 tablet_service.cc:1515] Processing DeleteTablet for tablet aca138b125614a2e9fbf817669df22c7 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:27 UTC) from {username='slave'} at 127.0.0.1:54406
I20250628 07:54:27.981768 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f: Sending DeleteTablet for 1 replicas of tablet ed0e24badb1f45118c05873c68630f24
I20250628 07:54:27.982051 5212 tablet_replica.cc:331] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600: stopping tablet replica
I20250628 07:54:27.982252 5132 tablet_service.cc:1515] Processing DeleteTablet for tablet c11c99a9810d44f2992e2c6352aa44b2 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:27 UTC) from {username='slave'} at 127.0.0.1:54406
I20250628 07:54:27.982440 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f: Sending DeleteTablet for 1 replicas of tablet 8d10e117388d4af48a4ab58bc6ebef66
I20250628 07:54:27.982915 5212 raft_consensus.cc:2241] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:27.983093 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f: Sending DeleteTablet for 1 replicas of tablet 0d1ffcf6f5c346668d1fc976a9cb56f9
I20250628 07:54:27.983355 5132 tablet_service.cc:1515] Processing DeleteTablet for tablet ed0e24badb1f45118c05873c68630f24 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:27 UTC) from {username='slave'} at 127.0.0.1:54406
I20250628 07:54:27.983523 5212 raft_consensus.cc:2270] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:27.983742 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f: Sending DeleteTablet for 1 replicas of tablet c2ec930cfa304c8abbf64cf97aa38d44
I20250628 07:54:27.984256 5132 tablet_service.cc:1515] Processing DeleteTablet for tablet 8d10e117388d4af48a4ab58bc6ebef66 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:27 UTC) from {username='slave'} at 127.0.0.1:54406
I20250628 07:54:27.984390 4824 catalog_manager.cc:5869] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f: Sending DeleteTablet for 1 replicas of tablet 60109d005bb346d1b90301381712282d
I20250628 07:54:27.985808 5132 tablet_service.cc:1515] Processing DeleteTablet for tablet 0d1ffcf6f5c346668d1fc976a9cb56f9 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:27 UTC) from {username='slave'} at 127.0.0.1:54406
I20250628 07:54:27.986380 5140 tablet_service.cc:1515] Processing DeleteTablet for tablet 60109d005bb346d1b90301381712282d with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:27 UTC) from {username='slave'} at 127.0.0.1:54406
I20250628 07:54:27.986495 5133 tablet_service.cc:1515] Processing DeleteTablet for tablet c2ec930cfa304c8abbf64cf97aa38d44 with delete_type TABLET_DATA_DELETED (Table deleted at 2025-06-28 07:54:27 UTC) from {username='slave'} at 127.0.0.1:54406
I20250628 07:54:27.987314 4824 tablet_server.cc:178] TabletServer@127.4.182.1:0 shutting down...
I20250628 07:54:27.987291 5212 ts_tablet_manager.cc:1905] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:54:28.005982 5212 ts_tablet_manager.cc:1918] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:54:28.006513 5212 log.cc:1199] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600: Deleting WAL directory at /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1751097264090342-4824-0/minicluster-data/ts-0-root/wals/2cc4129a5ef9405d913a93f0ec351eae
I20250628 07:54:28.007151 5212 ts_tablet_manager.cc:1939] T 2cc4129a5ef9405d913a93f0ec351eae P 407b8f0cb3024d4c94323b7e79dd8600: Deleting consensus metadata
I20250628 07:54:28.009729 5212 tablet_replica.cc:331] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600: stopping tablet replica
I20250628 07:54:28.009804 5033 catalog_manager.cc:4928] TS 407b8f0cb3024d4c94323b7e79dd8600 (127.4.182.1:40401): tablet 2cc4129a5ef9405d913a93f0ec351eae (table test_table [id=a38837c585e34580ad65764e0f8805ff]) successfully deleted
I20250628 07:54:28.010653 4824 ts_tablet_manager.cc:1500] Shutting down tablet manager...
I20250628 07:54:28.010641 5212 raft_consensus.cc:2241] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:28.011155 5212 raft_consensus.cc:2270] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:28.013537 5212 ts_tablet_manager.cc:1905] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600: Deleting tablet data with delete state TABLET_DATA_DELETED
I20250628 07:54:28.022845 5212 ts_tablet_manager.cc:1918] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20250628 07:54:28.023133 5212 log.cc:1199] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600: Deleting WAL directory at /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser.1751097264090342-4824-0/minicluster-data/ts-0-root/wals/2e24101def284c78adb8da97a6c55f5c
I20250628 07:54:28.023715 5212 ts_tablet_manager.cc:1939] T 2e24101def284c78adb8da97a6c55f5c P 407b8f0cb3024d4c94323b7e79dd8600: Deleting consensus metadata
W20250628 07:54:28.024946 5033 catalog_manager.cc:4920] TS 407b8f0cb3024d4c94323b7e79dd8600 (127.4.182.1:40401): delete failed for tablet 2e24101def284c78adb8da97a6c55f5c with error code TABLET_NOT_RUNNING: Service unavailable: Tablet Manager is not running: MANAGER_QUIESCING
I20250628 07:54:28.026836 4824 tablet_replica.cc:331] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600: stopping tablet replica
I20250628 07:54:28.027428 4824 raft_consensus.cc:2241] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:28.027822 4824 raft_consensus.cc:2270] T 0d1ffcf6f5c346668d1fc976a9cb56f9 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:28.030179 4824 tablet_replica.cc:331] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600: stopping tablet replica
W20250628 07:54:28.030855 5033 catalog_manager.cc:4671] TS 407b8f0cb3024d4c94323b7e79dd8600 (127.4.182.1:40401): DeleteTablet:TABLET_DATA_DELETED RPC failed for tablet d396c8b9f507404a8bc324ddd83408ce: Remote error: Service unavailable: service kudu.tserver.TabletServerAdminService not registered on TabletServer
I20250628 07:54:28.030877 4824 raft_consensus.cc:2241] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:28.031443 4824 raft_consensus.cc:2270] T c2ec930cfa304c8abbf64cf97aa38d44 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:28.033929 4824 tablet_replica.cc:331] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600: stopping tablet replica
I20250628 07:54:28.034407 4824 raft_consensus.cc:2241] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:28.034757 4824 raft_consensus.cc:2270] T 60109d005bb346d1b90301381712282d P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:28.036518 4824 tablet_replica.cc:331] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600: stopping tablet replica
I20250628 07:54:28.036974 4824 raft_consensus.cc:2241] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:28.037307 4824 raft_consensus.cc:2270] T 8d10e117388d4af48a4ab58bc6ebef66 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:28.038985 4824 tablet_replica.cc:331] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600: stopping tablet replica
I20250628 07:54:28.039448 4824 raft_consensus.cc:2241] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:28.039808 4824 raft_consensus.cc:2270] T c11c99a9810d44f2992e2c6352aa44b2 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:28.041837 4824 tablet_replica.cc:331] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600: stopping tablet replica
I20250628 07:54:28.042275 4824 raft_consensus.cc:2241] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:28.042615 4824 raft_consensus.cc:2270] T ed0e24badb1f45118c05873c68630f24 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:28.044495 4824 tablet_replica.cc:331] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600: stopping tablet replica
I20250628 07:54:28.044934 4824 raft_consensus.cc:2241] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:28.045265 4824 raft_consensus.cc:2270] T d396c8b9f507404a8bc324ddd83408ce P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:28.046993 4824 tablet_replica.cc:331] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600: stopping tablet replica
I20250628 07:54:28.047426 4824 raft_consensus.cc:2241] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:28.047765 4824 raft_consensus.cc:2270] T aca138b125614a2e9fbf817669df22c7 P 407b8f0cb3024d4c94323b7e79dd8600 [term 1 FOLLOWER]: Raft consensus is shut down!
W20250628 07:54:28.058296 4824 connection.cc:441] server connection from 127.0.0.1:54406 torn down before Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:54406 (request call id 29) could send its response
I20250628 07:54:28.062096 4824 tablet_server.cc:195] TabletServer@127.4.182.1:0 shutdown complete.
I20250628 07:54:28.073740 4824 master.cc:561] Master@127.4.182.62:34207 shutting down...
I20250628 07:54:28.173224 4824 raft_consensus.cc:2241] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:28.173743 4824 raft_consensus.cc:2270] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:28.174041 4824 tablet_replica.cc:331] T 00000000000000000000000000000000 P 3591ddc4ecfb4db19467e4b5f9fea55f: stopping tablet replica
I20250628 07:54:28.191022 4824 master.cc:583] Master@127.4.182.62:34207 shutdown complete.
[ OK ] CatalogManagerRpcAndUserFunctionsTest.TestDeleteTableWithUser (1057 ms)
[ RUN ] CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc
I20250628 07:54:28.210747 4824 internal_mini_cluster.cc:156] Creating distributed mini masters. Addrs: 127.4.182.62:39203
I20250628 07:54:28.211755 4824 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:54:28.215888 5214 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:28.216780 5215 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:28.217655 5217 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:54:28.218304 4824 server_base.cc:1048] running on GCE node
I20250628 07:54:28.218964 4824 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250628 07:54:28.219135 4824 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250628 07:54:28.219271 4824 hybrid_clock.cc:648] HybridClock initialized: now 1751097268219255 us; error 0 us; skew 500 ppm
I20250628 07:54:28.219794 4824 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:54:28.221933 4824 webserver.cc:469] Webserver started at http://127.4.182.62:44071/ using document root <none> and password file <none>
I20250628 07:54:28.222359 4824 fs_manager.cc:362] Metadata directory not provided
I20250628 07:54:28.222520 4824 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:54:28.222752 4824 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:54:28.223747 4824 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1751097264090342-4824-0/minicluster-data/master-0-root/instance:
uuid: "0ab4209550154dd5ba34fd5c2d78b1b5"
format_stamp: "Formatted at 2025-06-28 07:54:28 on dist-test-slave-v4r1"
I20250628 07:54:28.227831 4824 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.004s sys 0.000s
I20250628 07:54:28.230798 5222 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:28.231535 4824 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
I20250628 07:54:28.231779 4824 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1751097264090342-4824-0/minicluster-data/master-0-root
uuid: "0ab4209550154dd5ba34fd5c2d78b1b5"
format_stamp: "Formatted at 2025-06-28 07:54:28 on dist-test-slave-v4r1"
I20250628 07:54:28.232018 4824 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1751097264090342-4824-0/minicluster-data/master-0-root
metadata directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1751097264090342-4824-0/minicluster-data/master-0-root
1 data directories: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1751097264090342-4824-0/minicluster-data/master-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:54:28.249032 4824 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250628 07:54:28.250001 4824 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:54:28.282694 4824 rpc_server.cc:307] RPC server started. Bound to: 127.4.182.62:39203
I20250628 07:54:28.282784 5273 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.182.62:39203 every 8 connection(s)
I20250628 07:54:28.286314 5274 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:28.295308 5274 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5: Bootstrap starting.
I20250628 07:54:28.299155 5274 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:28.302659 5274 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5: No bootstrap required, opened a new log
I20250628 07:54:28.304419 5274 raft_consensus.cc:357] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "0ab4209550154dd5ba34fd5c2d78b1b5" member_type: VOTER }
I20250628 07:54:28.304747 5274 raft_consensus.cc:383] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:28.304914 5274 raft_consensus.cc:738] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0ab4209550154dd5ba34fd5c2d78b1b5, State: Initialized, Role: FOLLOWER
I20250628 07:54:28.305363 5274 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "0ab4209550154dd5ba34fd5c2d78b1b5" member_type: VOTER }
I20250628 07:54:28.305765 5274 raft_consensus.cc:397] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:28.305946 5274 raft_consensus.cc:491] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:28.306142 5274 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:28.310127 5274 raft_consensus.cc:513] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "0ab4209550154dd5ba34fd5c2d78b1b5" member_type: VOTER }
I20250628 07:54:28.310614 5274 leader_election.cc:304] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 0ab4209550154dd5ba34fd5c2d78b1b5; no voters:
I20250628 07:54:28.311609 5274 leader_election.cc:290] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:28.311863 5277 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:28.313647 5277 raft_consensus.cc:695] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [term 1 LEADER]: Becoming Leader. State: Replica: 0ab4209550154dd5ba34fd5c2d78b1b5, State: Running, Role: LEADER
I20250628 07:54:28.314302 5277 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "0ab4209550154dd5ba34fd5c2d78b1b5" member_type: VOTER }
I20250628 07:54:28.314800 5274 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [sys.catalog]: configured and running, proceeding with master startup.
I20250628 07:54:28.325033 5278 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "0ab4209550154dd5ba34fd5c2d78b1b5" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "0ab4209550154dd5ba34fd5c2d78b1b5" member_type: VOTER } }
I20250628 07:54:28.325659 5278 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [sys.catalog]: This master's current role is: LEADER
I20250628 07:54:28.326053 5279 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0ab4209550154dd5ba34fd5c2d78b1b5. Latest consensus state: current_term: 1 leader_uuid: "0ab4209550154dd5ba34fd5c2d78b1b5" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "0ab4209550154dd5ba34fd5c2d78b1b5" member_type: VOTER } }
I20250628 07:54:28.326680 5279 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [sys.catalog]: This master's current role is: LEADER
I20250628 07:54:28.331255 5283 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250628 07:54:28.336830 5283 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250628 07:54:28.340186 4824 internal_mini_cluster.cc:184] Waiting to initialize catalog manager on master 0
I20250628 07:54:28.344736 5283 catalog_manager.cc:1349] Generated new cluster ID: 5f5dc9f9f4914d8fb051dfde790e2400
I20250628 07:54:28.344973 5283 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250628 07:54:28.359308 5283 catalog_manager.cc:1372] Generated new certificate authority record
I20250628 07:54:28.360466 5283 catalog_manager.cc:1506] Loading token signing keys...
I20250628 07:54:28.377110 5283 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5: Generated new TSK 0
I20250628 07:54:28.377714 5283 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250628 07:54:28.406287 4824 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:54:28.411469 5296 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:28.412412 5297 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:28.414319 5299 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:54:28.415187 4824 server_base.cc:1048] running on GCE node
I20250628 07:54:28.416018 4824 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250628 07:54:28.416188 4824 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250628 07:54:28.416352 4824 hybrid_clock.cc:648] HybridClock initialized: now 1751097268416335 us; error 0 us; skew 500 ppm
I20250628 07:54:28.416826 4824 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:54:28.418990 4824 webserver.cc:469] Webserver started at http://127.4.182.1:37185/ using document root <none> and password file <none>
I20250628 07:54:28.419489 4824 fs_manager.cc:362] Metadata directory not provided
I20250628 07:54:28.419667 4824 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:54:28.419905 4824 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:54:28.420872 4824 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1751097264090342-4824-0/minicluster-data/ts-0-root/instance:
uuid: "ffcb7508b5614f86b64633ede642d729"
format_stamp: "Formatted at 2025-06-28 07:54:28 on dist-test-slave-v4r1"
I20250628 07:54:28.425015 4824 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
I20250628 07:54:28.427929 5304 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:28.428714 4824 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
I20250628 07:54:28.429087 4824 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1751097264090342-4824-0/minicluster-data/ts-0-root
uuid: "ffcb7508b5614f86b64633ede642d729"
format_stamp: "Formatted at 2025-06-28 07:54:28 on dist-test-slave-v4r1"
I20250628 07:54:28.429495 4824 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1751097264090342-4824-0/minicluster-data/ts-0-root
metadata directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1751097264090342-4824-0/minicluster-data/ts-0-root
1 data directories: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc.1751097264090342-4824-0/minicluster-data/ts-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:54:28.461519 4824 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250628 07:54:28.462471 4824 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:54:28.463812 4824 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20250628 07:54:28.465883 4824 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20250628 07:54:28.466074 4824 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:28.466296 4824 ts_tablet_manager.cc:610] Registered 0 tablets
I20250628 07:54:28.466452 4824 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:28.501276 4824 rpc_server.cc:307] RPC server started. Bound to: 127.4.182.1:41915
I20250628 07:54:28.501366 5366 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.182.1:41915 every 8 connection(s)
I20250628 07:54:28.514408 5367 heartbeater.cc:344] Connected to a master server at 127.4.182.62:39203
I20250628 07:54:28.514753 5367 heartbeater.cc:461] Registering TS with master...
I20250628 07:54:28.515437 5367 heartbeater.cc:507] Master 127.4.182.62:39203 requested a full tablet report, sending...
I20250628 07:54:28.517158 5239 ts_manager.cc:194] Registered new tserver with Master: ffcb7508b5614f86b64633ede642d729 (127.4.182.1:41915)
I20250628 07:54:28.518105 4824 internal_mini_cluster.cc:371] 1 TS(s) registered with all masters after 0.013826835s
I20250628 07:54:28.518781 5239 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.0.1:42290
I20250628 07:54:28.536516 4824 catalog_manager.cc:2232] Servicing CreateTable request from internal request:
name: "test_table"
schema {
columns {
name: "key"
type: INT32
is_key: true
}
columns {
name: "int_val"
type: INT32
}
}
num_replicas: 1
owner: "default"
I20250628 07:54:28.545244 4824 tablet_server.cc:178] TabletServer@127.4.182.1:0 shutting down...
I20250628 07:54:28.565106 4824 ts_tablet_manager.cc:1500] Shutting down tablet manager...
W20250628 07:54:28.568531 5305 connection.cc:441] server connection from 127.0.0.1:46758 torn down before Call kudu.tserver.TabletServerAdminService.CreateTablet from 127.0.0.1:46758 (request call id 0) could send its response
I20250628 07:54:28.582927 4824 tablet_server.cc:195] TabletServer@127.4.182.1:0 shutdown complete.
I20250628 07:54:28.589501 4824 master.cc:561] Master@127.4.182.62:39203 shutting down...
I20250628 07:54:28.605893 4824 raft_consensus.cc:2241] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:28.606370 4824 raft_consensus.cc:2270] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:28.606756 4824 tablet_replica.cc:331] T 00000000000000000000000000000000 P 0ab4209550154dd5ba34fd5c2d78b1b5: stopping tablet replica
I20250628 07:54:28.615515 4824 master.cc:583] Master@127.4.182.62:39203 shutdown complete.
[ OK ] CatalogManagerRpcAndUserFunctionsTest.TestCreateTableRpc (421 ms)
[ RUN ] CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser
I20250628 07:54:28.632182 4824 internal_mini_cluster.cc:156] Creating distributed mini masters. Addrs: 127.4.182.62:35369
I20250628 07:54:28.633095 4824 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:54:28.637316 5380 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:28.638859 5381 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:28.638979 5383 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:54:28.639294 4824 server_base.cc:1048] running on GCE node
I20250628 07:54:28.639999 4824 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250628 07:54:28.640157 4824 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250628 07:54:28.640261 4824 hybrid_clock.cc:648] HybridClock initialized: now 1751097268640252 us; error 0 us; skew 500 ppm
I20250628 07:54:28.640690 4824 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:54:28.642856 4824 webserver.cc:469] Webserver started at http://127.4.182.62:32929/ using document root <none> and password file <none>
I20250628 07:54:28.643292 4824 fs_manager.cc:362] Metadata directory not provided
I20250628 07:54:28.643489 4824 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:54:28.643734 4824 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:54:28.644709 4824 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1751097264090342-4824-0/minicluster-data/master-0-root/instance:
uuid: "f43b1a0935e549019ab0fc3627b81574"
format_stamp: "Formatted at 2025-06-28 07:54:28 on dist-test-slave-v4r1"
I20250628 07:54:28.648737 4824 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
I20250628 07:54:28.651820 5388 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:28.652513 4824 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
I20250628 07:54:28.652774 4824 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1751097264090342-4824-0/minicluster-data/master-0-root
uuid: "f43b1a0935e549019ab0fc3627b81574"
format_stamp: "Formatted at 2025-06-28 07:54:28 on dist-test-slave-v4r1"
I20250628 07:54:28.653025 4824 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1751097264090342-4824-0/minicluster-data/master-0-root
metadata directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1751097264090342-4824-0/minicluster-data/master-0-root
1 data directories: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1751097264090342-4824-0/minicluster-data/master-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:54:28.671597 4824 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250628 07:54:28.672559 4824 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:54:28.705674 4824 rpc_server.cc:307] RPC server started. Bound to: 127.4.182.62:35369
I20250628 07:54:28.705806 5439 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.182.62:35369 every 8 connection(s)
I20250628 07:54:28.709131 5440 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:28.718271 5440 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574: Bootstrap starting.
I20250628 07:54:28.722105 5440 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:28.725644 5440 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574: No bootstrap required, opened a new log
I20250628 07:54:28.727299 5440 raft_consensus.cc:357] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f43b1a0935e549019ab0fc3627b81574" member_type: VOTER }
I20250628 07:54:28.727682 5440 raft_consensus.cc:383] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:28.727854 5440 raft_consensus.cc:738] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f43b1a0935e549019ab0fc3627b81574, State: Initialized, Role: FOLLOWER
I20250628 07:54:28.728276 5440 consensus_queue.cc:260] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f43b1a0935e549019ab0fc3627b81574" member_type: VOTER }
I20250628 07:54:28.728650 5440 raft_consensus.cc:397] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:28.728833 5440 raft_consensus.cc:491] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:28.729032 5440 raft_consensus.cc:3058] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:28.733110 5440 raft_consensus.cc:513] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f43b1a0935e549019ab0fc3627b81574" member_type: VOTER }
I20250628 07:54:28.733568 5440 leader_election.cc:304] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: f43b1a0935e549019ab0fc3627b81574; no voters:
I20250628 07:54:28.734560 5440 leader_election.cc:290] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:28.734843 5443 raft_consensus.cc:2802] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:28.736089 5443 raft_consensus.cc:695] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [term 1 LEADER]: Becoming Leader. State: Replica: f43b1a0935e549019ab0fc3627b81574, State: Running, Role: LEADER
I20250628 07:54:28.736618 5443 consensus_queue.cc:237] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f43b1a0935e549019ab0fc3627b81574" member_type: VOTER }
I20250628 07:54:28.737318 5440 sys_catalog.cc:564] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [sys.catalog]: configured and running, proceeding with master startup.
I20250628 07:54:28.739194 5444 sys_catalog.cc:455] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "f43b1a0935e549019ab0fc3627b81574" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f43b1a0935e549019ab0fc3627b81574" member_type: VOTER } }
I20250628 07:54:28.739272 5445 sys_catalog.cc:455] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f43b1a0935e549019ab0fc3627b81574. Latest consensus state: current_term: 1 leader_uuid: "f43b1a0935e549019ab0fc3627b81574" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f43b1a0935e549019ab0fc3627b81574" member_type: VOTER } }
I20250628 07:54:28.739861 5445 sys_catalog.cc:458] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [sys.catalog]: This master's current role is: LEADER
I20250628 07:54:28.739867 5444 sys_catalog.cc:458] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [sys.catalog]: This master's current role is: LEADER
I20250628 07:54:28.742930 5448 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250628 07:54:28.747321 5448 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250628 07:54:28.751775 4824 internal_mini_cluster.cc:184] Waiting to initialize catalog manager on master 0
I20250628 07:54:28.756229 5448 catalog_manager.cc:1349] Generated new cluster ID: 2c49d3abb6fa40169834b807a6c1c015
I20250628 07:54:28.756438 5448 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250628 07:54:28.775641 5448 catalog_manager.cc:1372] Generated new certificate authority record
I20250628 07:54:28.776825 5448 catalog_manager.cc:1506] Loading token signing keys...
I20250628 07:54:28.791829 5448 catalog_manager.cc:5955] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574: Generated new TSK 0
I20250628 07:54:28.792436 5448 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250628 07:54:28.817847 4824 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:54:28.822942 5461 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:28.824167 5462 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:54:28.825393 4824 server_base.cc:1048] running on GCE node
W20250628 07:54:28.826303 5464 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:54:28.827092 4824 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250628 07:54:28.827272 4824 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250628 07:54:28.827471 4824 hybrid_clock.cc:648] HybridClock initialized: now 1751097268827452 us; error 0 us; skew 500 ppm
I20250628 07:54:28.827950 4824 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:54:28.830152 4824 webserver.cc:469] Webserver started at http://127.4.182.1:46077/ using document root <none> and password file <none>
I20250628 07:54:28.830587 4824 fs_manager.cc:362] Metadata directory not provided
I20250628 07:54:28.830760 4824 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:54:28.831017 4824 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:54:28.832121 4824 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1751097264090342-4824-0/minicluster-data/ts-0-root/instance:
uuid: "56a2074011a0441b86b316a782f216be"
format_stamp: "Formatted at 2025-06-28 07:54:28 on dist-test-slave-v4r1"
I20250628 07:54:28.836215 4824 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.002s sys 0.003s
I20250628 07:54:28.839221 5469 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:28.839905 4824 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
I20250628 07:54:28.840149 4824 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1751097264090342-4824-0/minicluster-data/ts-0-root
uuid: "56a2074011a0441b86b316a782f216be"
format_stamp: "Formatted at 2025-06-28 07:54:28 on dist-test-slave-v4r1"
I20250628 07:54:28.840396 4824 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1751097264090342-4824-0/minicluster-data/ts-0-root
metadata directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1751097264090342-4824-0/minicluster-data/ts-0-root
1 data directories: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser.1751097264090342-4824-0/minicluster-data/ts-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:54:28.861794 4824 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250628 07:54:28.862762 4824 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:54:28.864149 4824 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20250628 07:54:28.866215 4824 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20250628 07:54:28.866384 4824 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:28.866621 4824 ts_tablet_manager.cc:610] Registered 0 tablets
I20250628 07:54:28.866776 4824 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:28.901753 4824 rpc_server.cc:307] RPC server started. Bound to: 127.4.182.1:34653
I20250628 07:54:28.901830 5531 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.182.1:34653 every 8 connection(s)
I20250628 07:54:28.914361 5532 heartbeater.cc:344] Connected to a master server at 127.4.182.62:35369
I20250628 07:54:28.914660 5532 heartbeater.cc:461] Registering TS with master...
I20250628 07:54:28.915251 5532 heartbeater.cc:507] Master 127.4.182.62:35369 requested a full tablet report, sending...
I20250628 07:54:28.916947 5405 ts_manager.cc:194] Registered new tserver with Master: 56a2074011a0441b86b316a782f216be (127.4.182.1:34653)
I20250628 07:54:28.917887 4824 internal_mini_cluster.cc:371] 1 TS(s) registered with all masters after 0.012981683s
I20250628 07:54:28.918560 5405 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.0.1:45732
I20250628 07:54:28.935794 4824 catalog_manager.cc:2241] Servicing CreateTable request from test_user:
name: "test_table"
schema {
columns {
name: "key"
type: INT32
is_key: true
}
columns {
name: "int_val"
type: INT32
}
}
num_replicas: 1
owner: "default"
I20250628 07:54:28.944159 4824 tablet_server.cc:178] TabletServer@127.4.182.1:0 shutting down...
I20250628 07:54:28.967039 4824 ts_tablet_manager.cc:1500] Shutting down tablet manager...
W20250628 07:54:28.970270 5470 connection.cc:441] server connection from 127.0.0.1:42892 torn down before Call kudu.tserver.TabletServerAdminService.CreateTablet from 127.0.0.1:42892 (request call id 0) could send its response
I20250628 07:54:28.984623 4824 tablet_server.cc:195] TabletServer@127.4.182.1:0 shutdown complete.
I20250628 07:54:28.990845 4824 master.cc:561] Master@127.4.182.62:35369 shutting down...
I20250628 07:54:29.029170 4824 raft_consensus.cc:2241] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:29.029659 4824 raft_consensus.cc:2270] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:29.030041 4824 tablet_replica.cc:331] T 00000000000000000000000000000000 P f43b1a0935e549019ab0fc3627b81574: stopping tablet replica
I20250628 07:54:29.047907 4824 master.cc:583] Master@127.4.182.62:35369 shutdown complete.
[ OK ] CatalogManagerRpcAndUserFunctionsTest.TestCreateTableWithUser (433 ms)
[ RUN ] CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc
I20250628 07:54:29.065987 4824 internal_mini_cluster.cc:156] Creating distributed mini masters. Addrs: 127.4.182.62:43923
I20250628 07:54:29.066864 4824 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:54:29.070818 5545 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:29.071837 5546 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:54:29.076290 4824 server_base.cc:1048] running on GCE node
W20250628 07:54:29.076311 5548 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:54:29.077291 4824 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250628 07:54:29.077461 4824 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250628 07:54:29.077606 4824 hybrid_clock.cc:648] HybridClock initialized: now 1751097269077590 us; error 0 us; skew 500 ppm
I20250628 07:54:29.078074 4824 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:54:29.080287 4824 webserver.cc:469] Webserver started at http://127.4.182.62:40283/ using document root <none> and password file <none>
I20250628 07:54:29.080735 4824 fs_manager.cc:362] Metadata directory not provided
I20250628 07:54:29.080907 4824 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:54:29.081136 4824 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:54:29.082113 4824 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1751097264090342-4824-0/minicluster-data/master-0-root/instance:
uuid: "5c240f2969e64ecfbfcf5d143c887763"
format_stamp: "Formatted at 2025-06-28 07:54:29 on dist-test-slave-v4r1"
I20250628 07:54:29.086277 4824 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.006s sys 0.000s
I20250628 07:54:29.089282 5553 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:29.089924 4824 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20250628 07:54:29.090163 4824 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1751097264090342-4824-0/minicluster-data/master-0-root
uuid: "5c240f2969e64ecfbfcf5d143c887763"
format_stamp: "Formatted at 2025-06-28 07:54:29 on dist-test-slave-v4r1"
I20250628 07:54:29.090415 4824 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1751097264090342-4824-0/minicluster-data/master-0-root
metadata directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1751097264090342-4824-0/minicluster-data/master-0-root
1 data directories: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1751097264090342-4824-0/minicluster-data/master-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:54:29.106951 4824 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250628 07:54:29.107964 4824 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:54:29.141476 4824 rpc_server.cc:307] RPC server started. Bound to: 127.4.182.62:43923
I20250628 07:54:29.141552 5604 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.182.62:43923 every 8 connection(s)
I20250628 07:54:29.145002 5605 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:29.154649 5605 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763: Bootstrap starting.
I20250628 07:54:29.158435 5605 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:29.162011 5605 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763: No bootstrap required, opened a new log
I20250628 07:54:29.163780 5605 raft_consensus.cc:357] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5c240f2969e64ecfbfcf5d143c887763" member_type: VOTER }
I20250628 07:54:29.164117 5605 raft_consensus.cc:383] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:29.164283 5605 raft_consensus.cc:738] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5c240f2969e64ecfbfcf5d143c887763, State: Initialized, Role: FOLLOWER
I20250628 07:54:29.164742 5605 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5c240f2969e64ecfbfcf5d143c887763" member_type: VOTER }
I20250628 07:54:29.165092 5605 raft_consensus.cc:397] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:29.165261 5605 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:29.165475 5605 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:29.169494 5605 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5c240f2969e64ecfbfcf5d143c887763" member_type: VOTER }
I20250628 07:54:29.169945 5605 leader_election.cc:304] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 5c240f2969e64ecfbfcf5d143c887763; no voters:
I20250628 07:54:29.170898 5605 leader_election.cc:290] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:29.171162 5608 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:29.172456 5608 raft_consensus.cc:695] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [term 1 LEADER]: Becoming Leader. State: Replica: 5c240f2969e64ecfbfcf5d143c887763, State: Running, Role: LEADER
I20250628 07:54:29.173092 5608 consensus_queue.cc:237] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5c240f2969e64ecfbfcf5d143c887763" member_type: VOTER }
I20250628 07:54:29.173652 5605 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [sys.catalog]: configured and running, proceeding with master startup.
I20250628 07:54:29.175709 5610 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5c240f2969e64ecfbfcf5d143c887763. Latest consensus state: current_term: 1 leader_uuid: "5c240f2969e64ecfbfcf5d143c887763" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5c240f2969e64ecfbfcf5d143c887763" member_type: VOTER } }
I20250628 07:54:29.176311 5610 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [sys.catalog]: This master's current role is: LEADER
I20250628 07:54:29.175709 5609 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "5c240f2969e64ecfbfcf5d143c887763" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5c240f2969e64ecfbfcf5d143c887763" member_type: VOTER } }
I20250628 07:54:29.181509 5609 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [sys.catalog]: This master's current role is: LEADER
I20250628 07:54:29.187855 5613 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250628 07:54:29.192921 5613 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250628 07:54:29.195273 4824 internal_mini_cluster.cc:184] Waiting to initialize catalog manager on master 0
I20250628 07:54:29.200603 5613 catalog_manager.cc:1349] Generated new cluster ID: 0387f084d6b74584b0a04b8a9d9de527
I20250628 07:54:29.200884 5613 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250628 07:54:29.217779 5613 catalog_manager.cc:1372] Generated new certificate authority record
I20250628 07:54:29.218969 5613 catalog_manager.cc:1506] Loading token signing keys...
I20250628 07:54:29.231082 5613 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763: Generated new TSK 0
I20250628 07:54:29.231722 5613 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250628 07:54:29.261593 4824 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:54:29.266806 5626 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:29.268182 5627 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:29.270555 5629 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:54:29.271732 4824 server_base.cc:1048] running on GCE node
I20250628 07:54:29.272557 4824 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250628 07:54:29.272727 4824 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250628 07:54:29.272871 4824 hybrid_clock.cc:648] HybridClock initialized: now 1751097269272854 us; error 0 us; skew 500 ppm
I20250628 07:54:29.273331 4824 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:54:29.275594 4824 webserver.cc:469] Webserver started at http://127.4.182.1:44289/ using document root <none> and password file <none>
I20250628 07:54:29.276043 4824 fs_manager.cc:362] Metadata directory not provided
I20250628 07:54:29.276217 4824 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:54:29.276448 4824 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:54:29.277422 4824 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1751097264090342-4824-0/minicluster-data/ts-0-root/instance:
uuid: "64389869905f44778376ebfbae62b450"
format_stamp: "Formatted at 2025-06-28 07:54:29 on dist-test-slave-v4r1"
I20250628 07:54:29.281580 4824 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.006s sys 0.000s
I20250628 07:54:29.284646 5634 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:29.285359 4824 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
I20250628 07:54:29.285634 4824 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1751097264090342-4824-0/minicluster-data/ts-0-root
uuid: "64389869905f44778376ebfbae62b450"
format_stamp: "Formatted at 2025-06-28 07:54:29 on dist-test-slave-v4r1"
I20250628 07:54:29.285889 4824 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1751097264090342-4824-0/minicluster-data/ts-0-root
metadata directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1751097264090342-4824-0/minicluster-data/ts-0-root
1 data directories: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc.1751097264090342-4824-0/minicluster-data/ts-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:54:29.294734 4824 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250628 07:54:29.295701 4824 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:54:29.297021 4824 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20250628 07:54:29.299115 4824 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20250628 07:54:29.299304 4824 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:29.299582 4824 ts_tablet_manager.cc:610] Registered 0 tablets
I20250628 07:54:29.299734 4824 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:29.336319 4824 rpc_server.cc:307] RPC server started. Bound to: 127.4.182.1:41899
I20250628 07:54:29.336397 5696 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.182.1:41899 every 8 connection(s)
I20250628 07:54:29.350162 5697 heartbeater.cc:344] Connected to a master server at 127.4.182.62:43923
I20250628 07:54:29.350467 5697 heartbeater.cc:461] Registering TS with master...
I20250628 07:54:29.351092 5697 heartbeater.cc:507] Master 127.4.182.62:43923 requested a full tablet report, sending...
I20250628 07:54:29.352808 5570 ts_manager.cc:194] Registered new tserver with Master: 64389869905f44778376ebfbae62b450 (127.4.182.1:41899)
I20250628 07:54:29.353739 4824 internal_mini_cluster.cc:371] 1 TS(s) registered with all masters after 0.01429994s
I20250628 07:54:29.354406 5570 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.0.1:33980
I20250628 07:54:29.374405 5570 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33990:
name: "test_table"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
rows: "\004\001d\000\000\000\004\001\310\000\000\000\004\001,\001\000\000\004\001\220\001\000\000\004\001\364\001\000\000\004\001X\002\000\000\004\001\274\002\000\000\004\001 \003\000\000\004\001\204\003\000\000""\004\001d\000\000\000\004\001\310\000\000\000\004\001,\001\000\000\004\001\220\001\000\000\004\001\364\001\000\000\004\001X\002\000\000\004\001\274\002\000\000\004\001 \003\000\000\004\001\204\003\000\000"
indirect_data: """"
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
I20250628 07:54:29.417614 5662 tablet_service.cc:1468] Processing CreateTablet for tablet ec612aed4d124be6b282e3652b42d27e (DEFAULT_TABLE table=test_table [id=ee8c4bd32e7b4e6e8f1ff2bbefee2104]), partition=RANGE (key) PARTITION VALUES < 100
I20250628 07:54:29.417979 5661 tablet_service.cc:1468] Processing CreateTablet for tablet 5ba8f7c933254c5394c0548e64222d97 (DEFAULT_TABLE table=test_table [id=ee8c4bd32e7b4e6e8f1ff2bbefee2104]), partition=RANGE (key) PARTITION 100 <= VALUES < 200
I20250628 07:54:29.418959 5662 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ec612aed4d124be6b282e3652b42d27e. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:29.419690 5656 tablet_service.cc:1468] Processing CreateTablet for tablet 4f2d5c8b48ae4665bb9f6841b64f089a (DEFAULT_TABLE table=test_table [id=ee8c4bd32e7b4e6e8f1ff2bbefee2104]), partition=RANGE (key) PARTITION 600 <= VALUES < 700
I20250628 07:54:29.420248 5660 tablet_service.cc:1468] Processing CreateTablet for tablet 99961049b52d44c783f645d37e264b17 (DEFAULT_TABLE table=test_table [id=ee8c4bd32e7b4e6e8f1ff2bbefee2104]), partition=RANGE (key) PARTITION 200 <= VALUES < 300
I20250628 07:54:29.420971 5656 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4f2d5c8b48ae4665bb9f6841b64f089a. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:29.420732 5653 tablet_service.cc:1468] Processing CreateTablet for tablet 387e2c76406e498d8f03ff84a11ec477 (DEFAULT_TABLE table=test_table [id=ee8c4bd32e7b4e6e8f1ff2bbefee2104]), partition=RANGE (key) PARTITION 900 <= VALUES
I20250628 07:54:29.421921 5653 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 387e2c76406e498d8f03ff84a11ec477. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:29.421918 5659 tablet_service.cc:1468] Processing CreateTablet for tablet 150bdc4ae42c483a944768936f34ae51 (DEFAULT_TABLE table=test_table [id=ee8c4bd32e7b4e6e8f1ff2bbefee2104]), partition=RANGE (key) PARTITION 300 <= VALUES < 400
I20250628 07:54:29.422717 5657 tablet_service.cc:1468] Processing CreateTablet for tablet 335025643e6d4f3ba16a5611084b21cd (DEFAULT_TABLE table=test_table [id=ee8c4bd32e7b4e6e8f1ff2bbefee2104]), partition=RANGE (key) PARTITION 500 <= VALUES < 600
I20250628 07:54:29.423272 5659 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 150bdc4ae42c483a944768936f34ae51. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:29.423974 5657 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 335025643e6d4f3ba16a5611084b21cd. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:29.424571 5660 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 99961049b52d44c783f645d37e264b17. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:29.424281 5655 tablet_service.cc:1468] Processing CreateTablet for tablet f1b9e1ed18e64f8b8345baf3bed2b55c (DEFAULT_TABLE table=test_table [id=ee8c4bd32e7b4e6e8f1ff2bbefee2104]), partition=RANGE (key) PARTITION 700 <= VALUES < 800
I20250628 07:54:29.425451 5655 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f1b9e1ed18e64f8b8345baf3bed2b55c. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:29.425843 5658 tablet_service.cc:1468] Processing CreateTablet for tablet 109fe9395f5f4154b8b3c928b310fcb3 (DEFAULT_TABLE table=test_table [id=ee8c4bd32e7b4e6e8f1ff2bbefee2104]), partition=RANGE (key) PARTITION 400 <= VALUES < 500
I20250628 07:54:29.426730 5661 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5ba8f7c933254c5394c0548e64222d97. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:29.428995 5658 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 109fe9395f5f4154b8b3c928b310fcb3. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:29.432293 5654 tablet_service.cc:1468] Processing CreateTablet for tablet fc356a4cf2e44c69b19ff1288a0e3a73 (DEFAULT_TABLE table=test_table [id=ee8c4bd32e7b4e6e8f1ff2bbefee2104]), partition=RANGE (key) PARTITION 800 <= VALUES < 900
I20250628 07:54:29.433477 5654 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fc356a4cf2e44c69b19ff1288a0e3a73. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:29.463840 5709 tablet_bootstrap.cc:492] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450: Bootstrap starting.
I20250628 07:54:29.468465 5709 tablet_bootstrap.cc:654] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:29.472033 5709 tablet_bootstrap.cc:492] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450: No bootstrap required, opened a new log
I20250628 07:54:29.472362 5709 ts_tablet_manager.cc:1397] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450: Time spent bootstrapping tablet: real 0.009s user 0.007s sys 0.000s
I20250628 07:54:29.474102 5709 raft_consensus.cc:357] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.474488 5709 raft_consensus.cc:383] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:29.474741 5709 raft_consensus.cc:738] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 64389869905f44778376ebfbae62b450, State: Initialized, Role: FOLLOWER
I20250628 07:54:29.475234 5709 consensus_queue.cc:260] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.475778 5709 raft_consensus.cc:397] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:29.476016 5709 raft_consensus.cc:491] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:29.476269 5709 raft_consensus.cc:3058] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:29.480461 5709 raft_consensus.cc:513] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.481002 5709 leader_election.cc:304] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 64389869905f44778376ebfbae62b450; no voters:
I20250628 07:54:29.482046 5709 leader_election.cc:290] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:29.482326 5711 raft_consensus.cc:2802] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:29.483642 5711 raft_consensus.cc:695] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Becoming Leader. State: Replica: 64389869905f44778376ebfbae62b450, State: Running, Role: LEADER
I20250628 07:54:29.483805 5697 heartbeater.cc:499] Master 127.4.182.62:43923 was elected leader, sending a full tablet report...
I20250628 07:54:29.484380 5709 ts_tablet_manager.cc:1428] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450: Time spent starting tablet: real 0.012s user 0.002s sys 0.010s
I20250628 07:54:29.484376 5711 consensus_queue.cc:237] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.485121 5709 tablet_bootstrap.cc:492] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450: Bootstrap starting.
I20250628 07:54:29.490744 5709 tablet_bootstrap.cc:654] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:29.494925 5709 tablet_bootstrap.cc:492] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450: No bootstrap required, opened a new log
I20250628 07:54:29.495337 5709 ts_tablet_manager.cc:1397] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.008s
I20250628 07:54:29.494906 5570 catalog_manager.cc:5582] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450 reported cstate change: term changed from 0 to 1, leader changed from <none> to 64389869905f44778376ebfbae62b450 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "64389869905f44778376ebfbae62b450" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:29.497117 5709 raft_consensus.cc:357] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.497561 5709 raft_consensus.cc:383] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:29.497849 5709 raft_consensus.cc:738] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 64389869905f44778376ebfbae62b450, State: Initialized, Role: FOLLOWER
I20250628 07:54:29.498472 5709 consensus_queue.cc:260] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.499068 5709 raft_consensus.cc:397] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:29.499354 5709 raft_consensus.cc:491] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:29.499706 5709 raft_consensus.cc:3058] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:29.505700 5709 raft_consensus.cc:513] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.506443 5709 leader_election.cc:304] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 64389869905f44778376ebfbae62b450; no voters:
I20250628 07:54:29.507076 5709 leader_election.cc:290] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:29.507184 5711 raft_consensus.cc:2802] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:29.507669 5711 raft_consensus.cc:695] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Becoming Leader. State: Replica: 64389869905f44778376ebfbae62b450, State: Running, Role: LEADER
I20250628 07:54:29.508350 5711 consensus_queue.cc:237] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.508965 5709 ts_tablet_manager.cc:1428] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450: Time spent starting tablet: real 0.013s user 0.010s sys 0.002s
I20250628 07:54:29.509624 5709 tablet_bootstrap.cc:492] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450: Bootstrap starting.
I20250628 07:54:29.515642 5709 tablet_bootstrap.cc:654] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:29.515159 5570 catalog_manager.cc:5582] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450 reported cstate change: term changed from 0 to 1, leader changed from <none> to 64389869905f44778376ebfbae62b450 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "64389869905f44778376ebfbae62b450" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:29.520036 5709 tablet_bootstrap.cc:492] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450: No bootstrap required, opened a new log
I20250628 07:54:29.520401 5709 ts_tablet_manager.cc:1397] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450: Time spent bootstrapping tablet: real 0.011s user 0.005s sys 0.004s
I20250628 07:54:29.522423 5709 raft_consensus.cc:357] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.522949 5709 raft_consensus.cc:383] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:29.523211 5709 raft_consensus.cc:738] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 64389869905f44778376ebfbae62b450, State: Initialized, Role: FOLLOWER
I20250628 07:54:29.523790 5709 consensus_queue.cc:260] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.524406 5709 raft_consensus.cc:397] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:29.524703 5709 raft_consensus.cc:491] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:29.525032 5709 raft_consensus.cc:3058] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:29.530570 5709 raft_consensus.cc:513] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.531096 5709 leader_election.cc:304] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 64389869905f44778376ebfbae62b450; no voters:
I20250628 07:54:29.531538 5709 leader_election.cc:290] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:29.531673 5711 raft_consensus.cc:2802] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:29.532181 5711 raft_consensus.cc:695] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Becoming Leader. State: Replica: 64389869905f44778376ebfbae62b450, State: Running, Role: LEADER
I20250628 07:54:29.532949 5709 ts_tablet_manager.cc:1428] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450: Time spent starting tablet: real 0.012s user 0.012s sys 0.000s
I20250628 07:54:29.532804 5711 consensus_queue.cc:237] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.533628 5709 tablet_bootstrap.cc:492] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450: Bootstrap starting.
I20250628 07:54:29.538838 5709 tablet_bootstrap.cc:654] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:29.539055 5570 catalog_manager.cc:5582] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450 reported cstate change: term changed from 0 to 1, leader changed from <none> to 64389869905f44778376ebfbae62b450 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "64389869905f44778376ebfbae62b450" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:29.543051 5709 tablet_bootstrap.cc:492] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450: No bootstrap required, opened a new log
I20250628 07:54:29.543416 5709 ts_tablet_manager.cc:1397] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450: Time spent bootstrapping tablet: real 0.010s user 0.008s sys 0.000s
I20250628 07:54:29.545301 5709 raft_consensus.cc:357] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.545830 5709 raft_consensus.cc:383] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:29.546103 5709 raft_consensus.cc:738] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 64389869905f44778376ebfbae62b450, State: Initialized, Role: FOLLOWER
I20250628 07:54:29.546698 5709 consensus_queue.cc:260] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.547156 5709 raft_consensus.cc:397] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:29.547372 5709 raft_consensus.cc:491] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:29.547641 5709 raft_consensus.cc:3058] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:29.553436 5709 raft_consensus.cc:513] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.553975 5709 leader_election.cc:304] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 64389869905f44778376ebfbae62b450; no voters:
I20250628 07:54:29.554404 5709 leader_election.cc:290] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:29.554533 5711 raft_consensus.cc:2802] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:29.554988 5711 raft_consensus.cc:695] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Becoming Leader. State: Replica: 64389869905f44778376ebfbae62b450, State: Running, Role: LEADER
I20250628 07:54:29.555626 5711 consensus_queue.cc:237] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.555949 5709 ts_tablet_manager.cc:1428] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450: Time spent starting tablet: real 0.012s user 0.012s sys 0.000s
I20250628 07:54:29.556814 5709 tablet_bootstrap.cc:492] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450: Bootstrap starting.
I20250628 07:54:29.561589 5709 tablet_bootstrap.cc:654] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:29.561484 5570 catalog_manager.cc:5582] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450 reported cstate change: term changed from 0 to 1, leader changed from <none> to 64389869905f44778376ebfbae62b450 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "64389869905f44778376ebfbae62b450" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:29.565642 5709 tablet_bootstrap.cc:492] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450: No bootstrap required, opened a new log
I20250628 07:54:29.566022 5709 ts_tablet_manager.cc:1397] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450: Time spent bootstrapping tablet: real 0.009s user 0.005s sys 0.003s
I20250628 07:54:29.567970 5709 raft_consensus.cc:357] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.568471 5709 raft_consensus.cc:383] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:29.568732 5709 raft_consensus.cc:738] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 64389869905f44778376ebfbae62b450, State: Initialized, Role: FOLLOWER
I20250628 07:54:29.569351 5709 consensus_queue.cc:260] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.569968 5709 raft_consensus.cc:397] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:29.570205 5709 raft_consensus.cc:491] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:29.570451 5709 raft_consensus.cc:3058] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:29.574875 5709 raft_consensus.cc:513] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.575362 5709 leader_election.cc:304] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 64389869905f44778376ebfbae62b450; no voters:
I20250628 07:54:29.575841 5709 leader_election.cc:290] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:29.575969 5711 raft_consensus.cc:2802] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:29.576414 5711 raft_consensus.cc:695] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Becoming Leader. State: Replica: 64389869905f44778376ebfbae62b450, State: Running, Role: LEADER
I20250628 07:54:29.577224 5709 ts_tablet_manager.cc:1428] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450: Time spent starting tablet: real 0.011s user 0.007s sys 0.004s
I20250628 07:54:29.577050 5711 consensus_queue.cc:237] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.577895 5709 tablet_bootstrap.cc:492] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450: Bootstrap starting.
I20250628 07:54:29.582341 5570 catalog_manager.cc:5582] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450 reported cstate change: term changed from 0 to 1, leader changed from <none> to 64389869905f44778376ebfbae62b450 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "64389869905f44778376ebfbae62b450" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:29.583002 5709 tablet_bootstrap.cc:654] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:29.586834 5709 tablet_bootstrap.cc:492] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450: No bootstrap required, opened a new log
I20250628 07:54:29.587178 5709 ts_tablet_manager.cc:1397] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450: Time spent bootstrapping tablet: real 0.009s user 0.004s sys 0.004s
I20250628 07:54:29.588958 5709 raft_consensus.cc:357] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.589372 5709 raft_consensus.cc:383] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:29.589587 5709 raft_consensus.cc:738] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 64389869905f44778376ebfbae62b450, State: Initialized, Role: FOLLOWER
I20250628 07:54:29.590134 5709 consensus_queue.cc:260] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.590523 5709 raft_consensus.cc:397] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:29.590714 5709 raft_consensus.cc:491] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:29.590929 5709 raft_consensus.cc:3058] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:29.595129 5709 raft_consensus.cc:513] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.595661 5709 leader_election.cc:304] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 64389869905f44778376ebfbae62b450; no voters:
I20250628 07:54:29.596066 5709 leader_election.cc:290] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:29.596194 5711 raft_consensus.cc:2802] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:29.596645 5711 raft_consensus.cc:695] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Becoming Leader. State: Replica: 64389869905f44778376ebfbae62b450, State: Running, Role: LEADER
I20250628 07:54:29.597318 5709 ts_tablet_manager.cc:1428] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450: Time spent starting tablet: real 0.010s user 0.008s sys 0.000s
I20250628 07:54:29.597292 5711 consensus_queue.cc:237] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.597946 5709 tablet_bootstrap.cc:492] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450: Bootstrap starting.
I20250628 07:54:29.602303 5709 tablet_bootstrap.cc:654] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:29.603034 5570 catalog_manager.cc:5582] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450 reported cstate change: term changed from 0 to 1, leader changed from <none> to 64389869905f44778376ebfbae62b450 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "64389869905f44778376ebfbae62b450" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:29.605782 5709 tablet_bootstrap.cc:492] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450: No bootstrap required, opened a new log
I20250628 07:54:29.606122 5709 ts_tablet_manager.cc:1397] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450: Time spent bootstrapping tablet: real 0.008s user 0.006s sys 0.000s
I20250628 07:54:29.607857 5709 raft_consensus.cc:357] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.608283 5709 raft_consensus.cc:383] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:29.608487 5709 raft_consensus.cc:738] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 64389869905f44778376ebfbae62b450, State: Initialized, Role: FOLLOWER
I20250628 07:54:29.608909 5709 consensus_queue.cc:260] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.609395 5709 raft_consensus.cc:397] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:29.609665 5709 raft_consensus.cc:491] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:29.609901 5709 raft_consensus.cc:3058] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:29.614444 5709 raft_consensus.cc:513] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.614915 5709 leader_election.cc:304] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 64389869905f44778376ebfbae62b450; no voters:
I20250628 07:54:29.615341 5709 leader_election.cc:290] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:29.615499 5711 raft_consensus.cc:2802] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:29.615933 5711 raft_consensus.cc:695] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Becoming Leader. State: Replica: 64389869905f44778376ebfbae62b450, State: Running, Role: LEADER
I20250628 07:54:29.616590 5711 consensus_queue.cc:237] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.617008 5709 ts_tablet_manager.cc:1428] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450: Time spent starting tablet: real 0.011s user 0.011s sys 0.000s
I20250628 07:54:29.617733 5709 tablet_bootstrap.cc:492] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450: Bootstrap starting.
I20250628 07:54:29.621701 5709 tablet_bootstrap.cc:654] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:29.622709 5570 catalog_manager.cc:5582] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450 reported cstate change: term changed from 0 to 1, leader changed from <none> to 64389869905f44778376ebfbae62b450 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "64389869905f44778376ebfbae62b450" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:29.625425 5709 tablet_bootstrap.cc:492] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450: No bootstrap required, opened a new log
I20250628 07:54:29.625759 5709 ts_tablet_manager.cc:1397] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.003s
I20250628 07:54:29.627607 5709 raft_consensus.cc:357] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.627981 5709 raft_consensus.cc:383] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:29.628165 5709 raft_consensus.cc:738] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 64389869905f44778376ebfbae62b450, State: Initialized, Role: FOLLOWER
I20250628 07:54:29.628664 5709 consensus_queue.cc:260] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.629175 5709 raft_consensus.cc:397] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:29.629377 5709 raft_consensus.cc:491] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:29.629607 5709 raft_consensus.cc:3058] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:29.634383 5709 raft_consensus.cc:513] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.634876 5709 leader_election.cc:304] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 64389869905f44778376ebfbae62b450; no voters:
I20250628 07:54:29.635332 5709 leader_election.cc:290] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:29.635471 5711 raft_consensus.cc:2802] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:29.635954 5711 raft_consensus.cc:695] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Becoming Leader. State: Replica: 64389869905f44778376ebfbae62b450, State: Running, Role: LEADER
I20250628 07:54:29.636746 5709 ts_tablet_manager.cc:1428] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450: Time spent starting tablet: real 0.011s user 0.011s sys 0.000s
I20250628 07:54:29.636564 5711 consensus_queue.cc:237] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.637327 5709 tablet_bootstrap.cc:492] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450: Bootstrap starting.
I20250628 07:54:29.642256 5709 tablet_bootstrap.cc:654] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:29.642321 5570 catalog_manager.cc:5582] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450 reported cstate change: term changed from 0 to 1, leader changed from <none> to 64389869905f44778376ebfbae62b450 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "64389869905f44778376ebfbae62b450" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:29.647655 5709 tablet_bootstrap.cc:492] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450: No bootstrap required, opened a new log
I20250628 07:54:29.648032 5709 ts_tablet_manager.cc:1397] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450: Time spent bootstrapping tablet: real 0.011s user 0.010s sys 0.000s
I20250628 07:54:29.649698 5709 raft_consensus.cc:357] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.650082 5709 raft_consensus.cc:383] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:29.650310 5709 raft_consensus.cc:738] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 64389869905f44778376ebfbae62b450, State: Initialized, Role: FOLLOWER
I20250628 07:54:29.650731 5709 consensus_queue.cc:260] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.651098 5709 raft_consensus.cc:397] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:29.651281 5709 raft_consensus.cc:491] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:29.651546 5709 raft_consensus.cc:3058] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:29.657040 5709 raft_consensus.cc:513] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.657590 5709 leader_election.cc:304] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 64389869905f44778376ebfbae62b450; no voters:
I20250628 07:54:29.657984 5709 leader_election.cc:290] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:29.658106 5711 raft_consensus.cc:2802] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:29.658524 5711 raft_consensus.cc:695] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Becoming Leader. State: Replica: 64389869905f44778376ebfbae62b450, State: Running, Role: LEADER
I20250628 07:54:29.659317 5709 ts_tablet_manager.cc:1428] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450: Time spent starting tablet: real 0.011s user 0.011s sys 0.000s
I20250628 07:54:29.659111 5711 consensus_queue.cc:237] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.659957 5709 tablet_bootstrap.cc:492] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450: Bootstrap starting.
I20250628 07:54:29.664434 5709 tablet_bootstrap.cc:654] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:29.666905 5570 catalog_manager.cc:5582] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450 reported cstate change: term changed from 0 to 1, leader changed from <none> to 64389869905f44778376ebfbae62b450 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "64389869905f44778376ebfbae62b450" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:29.668243 5709 tablet_bootstrap.cc:492] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450: No bootstrap required, opened a new log
I20250628 07:54:29.668736 5709 ts_tablet_manager.cc:1397] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450: Time spent bootstrapping tablet: real 0.009s user 0.007s sys 0.000s
I20250628 07:54:29.670336 5709 raft_consensus.cc:357] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.670692 5709 raft_consensus.cc:383] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:29.670936 5709 raft_consensus.cc:738] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 64389869905f44778376ebfbae62b450, State: Initialized, Role: FOLLOWER
I20250628 07:54:29.671501 5709 consensus_queue.cc:260] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.672008 5709 raft_consensus.cc:397] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:29.672214 5709 raft_consensus.cc:491] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:29.672441 5709 raft_consensus.cc:3058] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:29.677493 5709 raft_consensus.cc:513] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.677968 5709 leader_election.cc:304] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 64389869905f44778376ebfbae62b450; no voters:
I20250628 07:54:29.678336 5709 leader_election.cc:290] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:29.678488 5711 raft_consensus.cc:2802] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:29.678964 5711 raft_consensus.cc:695] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Becoming Leader. State: Replica: 64389869905f44778376ebfbae62b450, State: Running, Role: LEADER
I20250628 07:54:29.679639 5711 consensus_queue.cc:237] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } }
I20250628 07:54:29.679891 5709 ts_tablet_manager.cc:1428] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450: Time spent starting tablet: real 0.011s user 0.009s sys 0.003s
I20250628 07:54:29.685323 5570 catalog_manager.cc:5582] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450 reported cstate change: term changed from 0 to 1, leader changed from <none> to 64389869905f44778376ebfbae62b450 (127.4.182.1). New cstate: current_term: 1 leader_uuid: "64389869905f44778376ebfbae62b450" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "64389869905f44778376ebfbae62b450" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 41899 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:29.712693 4824 catalog_manager.cc:3502] Servicing AlterTable request from internal request:
table { table_name: "test_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "int_val2" type: INT32 is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } }
I20250628 07:54:29.740151 5737 tablet.cc:1722] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:29.745206 5737 tablet.cc:1620] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450: MemRowSet was empty: no flush needed.
I20250628 07:54:29.759116 4824 tablet_server.cc:178] TabletServer@127.4.182.1:0 shutting down...
I20250628 07:54:29.797885 5737 tablet.cc:1722] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:29.801442 5744 tablet.cc:1722] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:29.803342 5744 tablet.cc:1620] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450: MemRowSet was empty: no flush needed.
I20250628 07:54:29.809927 5737 tablet.cc:1620] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450: MemRowSet was empty: no flush needed.
I20250628 07:54:29.810242 5738 tablet.cc:1722] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:29.810915 5743 tablet.cc:1722] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:29.816730 5743 tablet.cc:1620] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450: MemRowSet was empty: no flush needed.
I20250628 07:54:29.825696 5744 tablet.cc:1722] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:29.826985 5737 tablet.cc:1722] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:29.828148 5744 tablet.cc:1620] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450: MemRowSet was empty: no flush needed.
I20250628 07:54:29.828703 5737 tablet.cc:1620] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450: MemRowSet was empty: no flush needed.
I20250628 07:54:29.833875 5743 tablet.cc:1722] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:29.836001 5743 tablet.cc:1620] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450: MemRowSet was empty: no flush needed.
I20250628 07:54:29.839694 5738 tablet.cc:1620] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450: MemRowSet was empty: no flush needed.
I20250628 07:54:29.840204 5744 tablet.cc:1722] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:29.841957 5744 tablet.cc:1620] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450: MemRowSet was empty: no flush needed.
I20250628 07:54:29.847858 5743 tablet.cc:1722] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:29.849874 5743 tablet.cc:1620] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450: MemRowSet was empty: no flush needed.
I20250628 07:54:29.850935 4824 ts_tablet_manager.cc:1500] Shutting down tablet manager...
I20250628 07:54:29.852047 4824 tablet_replica.cc:331] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450: stopping tablet replica
I20250628 07:54:29.852593 4824 raft_consensus.cc:2241] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:29.852999 4824 raft_consensus.cc:2270] T 5ba8f7c933254c5394c0548e64222d97 P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:29.855500 4824 tablet_replica.cc:331] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450: stopping tablet replica
I20250628 07:54:29.855996 4824 raft_consensus.cc:2241] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:29.856367 4824 raft_consensus.cc:2270] T 4f2d5c8b48ae4665bb9f6841b64f089a P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:29.858404 4824 tablet_replica.cc:331] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450: stopping tablet replica
I20250628 07:54:29.858883 4824 raft_consensus.cc:2241] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:29.859244 4824 raft_consensus.cc:2270] T 109fe9395f5f4154b8b3c928b310fcb3 P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:29.861238 4824 tablet_replica.cc:331] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450: stopping tablet replica
I20250628 07:54:29.861718 4824 raft_consensus.cc:2241] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:29.862123 4824 raft_consensus.cc:2270] T fc356a4cf2e44c69b19ff1288a0e3a73 P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:29.864342 4824 tablet_replica.cc:331] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450: stopping tablet replica
I20250628 07:54:29.864815 4824 raft_consensus.cc:2241] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:29.865175 4824 raft_consensus.cc:2270] T ec612aed4d124be6b282e3652b42d27e P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:29.866998 4824 tablet_replica.cc:331] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450: stopping tablet replica
I20250628 07:54:29.867481 4824 raft_consensus.cc:2241] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:29.867836 4824 raft_consensus.cc:2270] T 335025643e6d4f3ba16a5611084b21cd P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:29.869722 4824 tablet_replica.cc:331] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450: stopping tablet replica
I20250628 07:54:29.870180 4824 raft_consensus.cc:2241] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:29.870529 4824 raft_consensus.cc:2270] T f1b9e1ed18e64f8b8345baf3bed2b55c P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:29.872809 4824 tablet_replica.cc:331] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450: stopping tablet replica
I20250628 07:54:29.873409 4824 raft_consensus.cc:2241] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:29.873751 4824 raft_consensus.cc:2270] T 150bdc4ae42c483a944768936f34ae51 P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:29.875519 4824 tablet_replica.cc:331] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450: stopping tablet replica
I20250628 07:54:29.875922 4824 raft_consensus.cc:2241] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:29.876298 4824 raft_consensus.cc:2270] T 99961049b52d44c783f645d37e264b17 P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:29.878121 4824 tablet_replica.cc:331] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450: stopping tablet replica
I20250628 07:54:29.878517 4824 raft_consensus.cc:2241] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:29.879019 4824 raft_consensus.cc:2270] T 387e2c76406e498d8f03ff84a11ec477 P 64389869905f44778376ebfbae62b450 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:29.905012 4824 tablet_server.cc:195] TabletServer@127.4.182.1:0 shutdown complete.
I20250628 07:54:29.917779 4824 master.cc:561] Master@127.4.182.62:43923 shutting down...
I20250628 07:54:29.931823 4824 raft_consensus.cc:2241] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:29.932263 4824 raft_consensus.cc:2270] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:29.932502 4824 tablet_replica.cc:331] T 00000000000000000000000000000000 P 5c240f2969e64ecfbfcf5d143c887763: stopping tablet replica
I20250628 07:54:29.948932 4824 master.cc:583] Master@127.4.182.62:43923 shutdown complete.
[ OK ] CatalogManagerRpcAndUserFunctionsTest.TestAlterTableRpc (903 ms)
[ RUN ] CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser
I20250628 07:54:29.969580 4824 internal_mini_cluster.cc:156] Creating distributed mini masters. Addrs: 127.4.182.62:46145
I20250628 07:54:29.970551 4824 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:54:29.975481 5747 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:29.975627 5748 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:29.976624 5750 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:54:29.978111 4824 server_base.cc:1048] running on GCE node
I20250628 07:54:29.979081 4824 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250628 07:54:29.979244 4824 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250628 07:54:29.979372 4824 hybrid_clock.cc:648] HybridClock initialized: now 1751097269979359 us; error 0 us; skew 500 ppm
I20250628 07:54:29.979969 4824 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:54:29.982111 4824 webserver.cc:469] Webserver started at http://127.4.182.62:45401/ using document root <none> and password file <none>
I20250628 07:54:29.982532 4824 fs_manager.cc:362] Metadata directory not provided
I20250628 07:54:29.982676 4824 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:54:29.982874 4824 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:54:29.983934 4824 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1751097264090342-4824-0/minicluster-data/master-0-root/instance:
uuid: "186ba0a7aae64fd293bd115e37327686"
format_stamp: "Formatted at 2025-06-28 07:54:29 on dist-test-slave-v4r1"
I20250628 07:54:29.988066 4824 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.004s sys 0.000s
I20250628 07:54:29.991010 5755 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:29.991726 4824 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
I20250628 07:54:29.991974 4824 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1751097264090342-4824-0/minicluster-data/master-0-root
uuid: "186ba0a7aae64fd293bd115e37327686"
format_stamp: "Formatted at 2025-06-28 07:54:29 on dist-test-slave-v4r1"
I20250628 07:54:29.992244 4824 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1751097264090342-4824-0/minicluster-data/master-0-root
metadata directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1751097264090342-4824-0/minicluster-data/master-0-root
1 data directories: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1751097264090342-4824-0/minicluster-data/master-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:54:30.008399 4824 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250628 07:54:30.009380 4824 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:54:30.042160 4824 rpc_server.cc:307] RPC server started. Bound to: 127.4.182.62:46145
I20250628 07:54:30.042241 5806 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.182.62:46145 every 8 connection(s)
I20250628 07:54:30.045724 5807 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:30.055402 5807 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686: Bootstrap starting.
I20250628 07:54:30.059262 5807 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:30.062927 5807 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686: No bootstrap required, opened a new log
I20250628 07:54:30.064744 5807 raft_consensus.cc:357] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "186ba0a7aae64fd293bd115e37327686" member_type: VOTER }
I20250628 07:54:30.065122 5807 raft_consensus.cc:383] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:30.065335 5807 raft_consensus.cc:738] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 186ba0a7aae64fd293bd115e37327686, State: Initialized, Role: FOLLOWER
I20250628 07:54:30.065819 5807 consensus_queue.cc:260] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "186ba0a7aae64fd293bd115e37327686" member_type: VOTER }
I20250628 07:54:30.066215 5807 raft_consensus.cc:397] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:30.066419 5807 raft_consensus.cc:491] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:30.066653 5807 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:30.070823 5807 raft_consensus.cc:513] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "186ba0a7aae64fd293bd115e37327686" member_type: VOTER }
I20250628 07:54:30.071317 5807 leader_election.cc:304] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 186ba0a7aae64fd293bd115e37327686; no voters:
I20250628 07:54:30.072356 5807 leader_election.cc:290] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:30.072618 5810 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:30.073887 5810 raft_consensus.cc:695] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [term 1 LEADER]: Becoming Leader. State: Replica: 186ba0a7aae64fd293bd115e37327686, State: Running, Role: LEADER
I20250628 07:54:30.074473 5810 consensus_queue.cc:237] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "186ba0a7aae64fd293bd115e37327686" member_type: VOTER }
I20250628 07:54:30.075136 5807 sys_catalog.cc:564] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [sys.catalog]: configured and running, proceeding with master startup.
I20250628 07:54:30.077178 5811 sys_catalog.cc:455] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "186ba0a7aae64fd293bd115e37327686" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "186ba0a7aae64fd293bd115e37327686" member_type: VOTER } }
I20250628 07:54:30.077256 5812 sys_catalog.cc:455] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 186ba0a7aae64fd293bd115e37327686. Latest consensus state: current_term: 1 leader_uuid: "186ba0a7aae64fd293bd115e37327686" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "186ba0a7aae64fd293bd115e37327686" member_type: VOTER } }
I20250628 07:54:30.077876 5812 sys_catalog.cc:458] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [sys.catalog]: This master's current role is: LEADER
I20250628 07:54:30.077905 5811 sys_catalog.cc:458] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [sys.catalog]: This master's current role is: LEADER
I20250628 07:54:30.084379 5815 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
I20250628 07:54:30.088900 5815 catalog_manager.cc:1486] Initializing Kudu cluster ID...
I20250628 07:54:30.089887 4824 internal_mini_cluster.cc:184] Waiting to initialize catalog manager on master 0
I20250628 07:54:30.096477 5815 catalog_manager.cc:1349] Generated new cluster ID: 3f1dba32218e4657a20e1d5a1f7bcdde
I20250628 07:54:30.096693 5815 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
I20250628 07:54:30.109681 5815 catalog_manager.cc:1372] Generated new certificate authority record
I20250628 07:54:30.110795 5815 catalog_manager.cc:1506] Loading token signing keys...
I20250628 07:54:30.125069 5815 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686: Generated new TSK 0
I20250628 07:54:30.125689 5815 catalog_manager.cc:1516] Initializing in-progress tserver states...
I20250628 07:54:30.155941 4824 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20250628 07:54:30.160882 5828 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20250628 07:54:30.161971 5829 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:54:30.163949 4824 server_base.cc:1048] running on GCE node
W20250628 07:54:30.164359 5831 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20250628 07:54:30.165066 4824 hybrid_clock.cc:584] initializing the hybrid clock with 'system_unsync' time source
W20250628 07:54:30.165263 4824 system_unsync_time.cc:38] NTP support is disabled. Clock error bounds will not be accurate. This configuration is not suitable for distributed clusters.
I20250628 07:54:30.165411 4824 hybrid_clock.cc:648] HybridClock initialized: now 1751097270165395 us; error 0 us; skew 500 ppm
I20250628 07:54:30.165875 4824 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20250628 07:54:30.168228 4824 webserver.cc:469] Webserver started at http://127.4.182.1:42593/ using document root <none> and password file <none>
I20250628 07:54:30.168676 4824 fs_manager.cc:362] Metadata directory not provided
I20250628 07:54:30.168831 4824 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20250628 07:54:30.169075 4824 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
I20250628 07:54:30.170059 4824 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1751097264090342-4824-0/minicluster-data/ts-0-root/instance:
uuid: "b432318b9d784874abacf741a607342b"
format_stamp: "Formatted at 2025-06-28 07:54:30 on dist-test-slave-v4r1"
I20250628 07:54:30.174214 4824 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
I20250628 07:54:30.177301 5836 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:30.177974 4824 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20250628 07:54:30.178208 4824 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1751097264090342-4824-0/minicluster-data/ts-0-root
uuid: "b432318b9d784874abacf741a607342b"
format_stamp: "Formatted at 2025-06-28 07:54:30 on dist-test-slave-v4r1"
I20250628 07:54:30.178439 4824 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1751097264090342-4824-0/minicluster-data/ts-0-root
metadata directory: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1751097264090342-4824-0/minicluster-data/ts-0-root
1 data directories: /tmp/dist-test-taskJ_VzeE/test-tmp/catalog_manager-test.0.CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser.1751097264090342-4824-0/minicluster-data/ts-0-root/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20250628 07:54:30.204682 4824 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20250628 07:54:30.205601 4824 kserver.cc:163] Server-wide thread pool size limit: 3276
I20250628 07:54:30.206919 4824 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20250628 07:54:30.209031 4824 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20250628 07:54:30.209218 4824 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:30.209430 4824 ts_tablet_manager.cc:610] Registered 0 tablets
I20250628 07:54:30.209575 4824 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20250628 07:54:30.244293 4824 rpc_server.cc:307] RPC server started. Bound to: 127.4.182.1:38721
I20250628 07:54:30.244372 5898 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.182.1:38721 every 8 connection(s)
I20250628 07:54:30.256616 5899 heartbeater.cc:344] Connected to a master server at 127.4.182.62:46145
I20250628 07:54:30.256937 5899 heartbeater.cc:461] Registering TS with master...
I20250628 07:54:30.257588 5899 heartbeater.cc:507] Master 127.4.182.62:46145 requested a full tablet report, sending...
I20250628 07:54:30.259276 5772 ts_manager.cc:194] Registered new tserver with Master: b432318b9d784874abacf741a607342b (127.4.182.1:38721)
I20250628 07:54:30.259742 4824 internal_mini_cluster.cc:371] 1 TS(s) registered with all masters after 0.012343521s
I20250628 07:54:30.261029 5772 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.0.1:52500
I20250628 07:54:30.279665 5772 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:52516:
name: "test_table"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
rows: "\004\001d\000\000\000\004\001\310\000\000\000\004\001,\001\000\000\004\001\220\001\000\000\004\001\364\001\000\000\004\001X\002\000\000\004\001\274\002\000\000\004\001 \003\000\000\004\001\204\003\000\000""\004\001d\000\000\000\004\001\310\000\000\000\004\001,\001\000\000\004\001\220\001\000\000\004\001\364\001\000\000\004\001X\002\000\000\004\001\274\002\000\000\004\001 \003\000\000\004\001\204\003\000\000"
indirect_data: """"
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
I20250628 07:54:30.324343 5864 tablet_service.cc:1468] Processing CreateTablet for tablet 2b0b2d989359459384057b6dff3774fb (DEFAULT_TABLE table=test_table [id=0a7fc060f61a414c9a5076f1f539d209]), partition=RANGE (key) PARTITION VALUES < 100
I20250628 07:54:30.324542 5863 tablet_service.cc:1468] Processing CreateTablet for tablet 4eed1597698f455fa3063183469e4a8d (DEFAULT_TABLE table=test_table [id=0a7fc060f61a414c9a5076f1f539d209]), partition=RANGE (key) PARTITION 100 <= VALUES < 200
I20250628 07:54:30.324949 5862 tablet_service.cc:1468] Processing CreateTablet for tablet 75ed8b2968964c25a93e87c36669693b (DEFAULT_TABLE table=test_table [id=0a7fc060f61a414c9a5076f1f539d209]), partition=RANGE (key) PARTITION 200 <= VALUES < 300
I20250628 07:54:30.325608 5864 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2b0b2d989359459384057b6dff3774fb. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:30.326249 5862 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 75ed8b2968964c25a93e87c36669693b. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:30.326977 5863 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4eed1597698f455fa3063183469e4a8d. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:30.327215 5855 tablet_service.cc:1468] Processing CreateTablet for tablet 93a328949c244318bf159e2809364706 (DEFAULT_TABLE table=test_table [id=0a7fc060f61a414c9a5076f1f539d209]), partition=RANGE (key) PARTITION 900 <= VALUES
I20250628 07:54:30.328420 5855 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 93a328949c244318bf159e2809364706. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:30.328078 5856 tablet_service.cc:1468] Processing CreateTablet for tablet c2e65581e3ac41b9bf13b227ace0a214 (DEFAULT_TABLE table=test_table [id=0a7fc060f61a414c9a5076f1f539d209]), partition=RANGE (key) PARTITION 800 <= VALUES < 900
I20250628 07:54:30.327215 5859 tablet_service.cc:1468] Processing CreateTablet for tablet 48d59f36eb4649979abe40b36ee99a2d (DEFAULT_TABLE table=test_table [id=0a7fc060f61a414c9a5076f1f539d209]), partition=RANGE (key) PARTITION 500 <= VALUES < 600
I20250628 07:54:30.329293 5856 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c2e65581e3ac41b9bf13b227ace0a214. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:30.330179 5857 tablet_service.cc:1468] Processing CreateTablet for tablet 0cc05509e7e64fa3a689b7e588f606cf (DEFAULT_TABLE table=test_table [id=0a7fc060f61a414c9a5076f1f539d209]), partition=RANGE (key) PARTITION 700 <= VALUES < 800
I20250628 07:54:30.331403 5857 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0cc05509e7e64fa3a689b7e588f606cf. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:30.332015 5859 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 48d59f36eb4649979abe40b36ee99a2d. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:30.333144 5858 tablet_service.cc:1468] Processing CreateTablet for tablet db2e41adf4bc4bdd91382ada3d6a1469 (DEFAULT_TABLE table=test_table [id=0a7fc060f61a414c9a5076f1f539d209]), partition=RANGE (key) PARTITION 600 <= VALUES < 700
I20250628 07:54:30.334640 5861 tablet_service.cc:1468] Processing CreateTablet for tablet 3119994d67e04de0960f8650aef742dd (DEFAULT_TABLE table=test_table [id=0a7fc060f61a414c9a5076f1f539d209]), partition=RANGE (key) PARTITION 300 <= VALUES < 400
I20250628 07:54:30.336572 5860 tablet_service.cc:1468] Processing CreateTablet for tablet 10e6dbbe5f4e49f4a280dfb6915fc2e7 (DEFAULT_TABLE table=test_table [id=0a7fc060f61a414c9a5076f1f539d209]), partition=RANGE (key) PARTITION 400 <= VALUES < 500
I20250628 07:54:30.341346 5861 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3119994d67e04de0960f8650aef742dd. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:30.343098 5858 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet db2e41adf4bc4bdd91382ada3d6a1469. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:30.343932 5860 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 10e6dbbe5f4e49f4a280dfb6915fc2e7. 1 dirs total, 0 dirs full, 0 dirs failed
I20250628 07:54:30.366057 5911 tablet_bootstrap.cc:492] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b: Bootstrap starting.
I20250628 07:54:30.371944 5911 tablet_bootstrap.cc:654] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:30.376921 5911 tablet_bootstrap.cc:492] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b: No bootstrap required, opened a new log
I20250628 07:54:30.377363 5911 ts_tablet_manager.cc:1397] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b: Time spent bootstrapping tablet: real 0.012s user 0.010s sys 0.000s
I20250628 07:54:30.379781 5911 raft_consensus.cc:357] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.380332 5911 raft_consensus.cc:383] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:30.380602 5911 raft_consensus.cc:738] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b432318b9d784874abacf741a607342b, State: Initialized, Role: FOLLOWER
I20250628 07:54:30.381356 5911 consensus_queue.cc:260] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.381973 5911 raft_consensus.cc:397] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:30.382290 5911 raft_consensus.cc:491] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:30.382612 5911 raft_consensus.cc:3058] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:30.386984 5911 raft_consensus.cc:513] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.387503 5911 leader_election.cc:304] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: b432318b9d784874abacf741a607342b; no voters:
I20250628 07:54:30.388621 5911 leader_election.cc:290] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:30.388919 5913 raft_consensus.cc:2802] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:30.390352 5913 raft_consensus.cc:695] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b [term 1 LEADER]: Becoming Leader. State: Replica: b432318b9d784874abacf741a607342b, State: Running, Role: LEADER
I20250628 07:54:30.390434 5899 heartbeater.cc:499] Master 127.4.182.62:46145 was elected leader, sending a full tablet report...
I20250628 07:54:30.391131 5911 ts_tablet_manager.cc:1428] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b: Time spent starting tablet: real 0.013s user 0.013s sys 0.000s
I20250628 07:54:30.391086 5913 consensus_queue.cc:237] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.391950 5911 tablet_bootstrap.cc:492] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b: Bootstrap starting.
I20250628 07:54:30.397538 5911 tablet_bootstrap.cc:654] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:30.401336 5911 tablet_bootstrap.cc:492] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b: No bootstrap required, opened a new log
I20250628 07:54:30.401682 5911 ts_tablet_manager.cc:1397] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b: Time spent bootstrapping tablet: real 0.010s user 0.000s sys 0.008s
I20250628 07:54:30.401955 5772 catalog_manager.cc:5582] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b reported cstate change: term changed from 0 to 1, leader changed from <none> to b432318b9d784874abacf741a607342b (127.4.182.1). New cstate: current_term: 1 leader_uuid: "b432318b9d784874abacf741a607342b" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:30.403448 5911 raft_consensus.cc:357] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.403829 5911 raft_consensus.cc:383] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:30.404075 5911 raft_consensus.cc:738] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b432318b9d784874abacf741a607342b, State: Initialized, Role: FOLLOWER
I20250628 07:54:30.404554 5911 consensus_queue.cc:260] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.404990 5911 raft_consensus.cc:397] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:30.405201 5911 raft_consensus.cc:491] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:30.405443 5911 raft_consensus.cc:3058] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:30.410408 5911 raft_consensus.cc:513] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.410940 5911 leader_election.cc:304] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: b432318b9d784874abacf741a607342b; no voters:
I20250628 07:54:30.411535 5911 leader_election.cc:290] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:30.411684 5913 raft_consensus.cc:2802] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:30.412114 5913 raft_consensus.cc:695] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b [term 1 LEADER]: Becoming Leader. State: Replica: b432318b9d784874abacf741a607342b, State: Running, Role: LEADER
I20250628 07:54:30.412781 5913 consensus_queue.cc:237] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.413594 5911 ts_tablet_manager.cc:1428] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b: Time spent starting tablet: real 0.012s user 0.012s sys 0.000s
I20250628 07:54:30.414498 5911 tablet_bootstrap.cc:492] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b: Bootstrap starting.
I20250628 07:54:30.418876 5911 tablet_bootstrap.cc:654] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:30.419337 5771 catalog_manager.cc:5582] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b reported cstate change: term changed from 0 to 1, leader changed from <none> to b432318b9d784874abacf741a607342b (127.4.182.1). New cstate: current_term: 1 leader_uuid: "b432318b9d784874abacf741a607342b" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:30.422909 5911 tablet_bootstrap.cc:492] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b: No bootstrap required, opened a new log
I20250628 07:54:30.423333 5911 ts_tablet_manager.cc:1397] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b: Time spent bootstrapping tablet: real 0.009s user 0.007s sys 0.000s
I20250628 07:54:30.425311 5911 raft_consensus.cc:357] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.425817 5911 raft_consensus.cc:383] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:30.426076 5911 raft_consensus.cc:738] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b432318b9d784874abacf741a607342b, State: Initialized, Role: FOLLOWER
I20250628 07:54:30.426556 5911 consensus_queue.cc:260] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.427018 5911 raft_consensus.cc:397] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:30.427276 5911 raft_consensus.cc:491] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:30.427618 5911 raft_consensus.cc:3058] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:30.433480 5911 raft_consensus.cc:513] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.434131 5911 leader_election.cc:304] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: b432318b9d784874abacf741a607342b; no voters:
I20250628 07:54:30.434703 5911 leader_election.cc:290] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:30.434859 5913 raft_consensus.cc:2802] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:30.435299 5913 raft_consensus.cc:695] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b [term 1 LEADER]: Becoming Leader. State: Replica: b432318b9d784874abacf741a607342b, State: Running, Role: LEADER
I20250628 07:54:30.436081 5913 consensus_queue.cc:237] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.436600 5911 ts_tablet_manager.cc:1428] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b: Time spent starting tablet: real 0.013s user 0.008s sys 0.003s
I20250628 07:54:30.437407 5911 tablet_bootstrap.cc:492] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b: Bootstrap starting.
I20250628 07:54:30.441790 5772 catalog_manager.cc:5582] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b reported cstate change: term changed from 0 to 1, leader changed from <none> to b432318b9d784874abacf741a607342b (127.4.182.1). New cstate: current_term: 1 leader_uuid: "b432318b9d784874abacf741a607342b" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:30.442147 5911 tablet_bootstrap.cc:654] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:30.446147 5911 tablet_bootstrap.cc:492] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b: No bootstrap required, opened a new log
I20250628 07:54:30.446483 5911 ts_tablet_manager.cc:1397] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b: Time spent bootstrapping tablet: real 0.009s user 0.007s sys 0.000s
I20250628 07:54:30.448508 5911 raft_consensus.cc:357] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.449038 5911 raft_consensus.cc:383] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:30.449254 5911 raft_consensus.cc:738] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b432318b9d784874abacf741a607342b, State: Initialized, Role: FOLLOWER
I20250628 07:54:30.449798 5911 consensus_queue.cc:260] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.450253 5911 raft_consensus.cc:397] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:30.450471 5911 raft_consensus.cc:491] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:30.450719 5911 raft_consensus.cc:3058] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:30.455519 5911 raft_consensus.cc:513] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.456066 5911 leader_election.cc:304] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: b432318b9d784874abacf741a607342b; no voters:
I20250628 07:54:30.456524 5911 leader_election.cc:290] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:30.456670 5913 raft_consensus.cc:2802] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:30.457130 5913 raft_consensus.cc:695] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b [term 1 LEADER]: Becoming Leader. State: Replica: b432318b9d784874abacf741a607342b, State: Running, Role: LEADER
I20250628 07:54:30.457818 5913 consensus_queue.cc:237] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.458344 5911 ts_tablet_manager.cc:1428] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b: Time spent starting tablet: real 0.012s user 0.011s sys 0.000s
I20250628 07:54:30.459167 5911 tablet_bootstrap.cc:492] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b: Bootstrap starting.
I20250628 07:54:30.464296 5911 tablet_bootstrap.cc:654] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:30.463984 5772 catalog_manager.cc:5582] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b reported cstate change: term changed from 0 to 1, leader changed from <none> to b432318b9d784874abacf741a607342b (127.4.182.1). New cstate: current_term: 1 leader_uuid: "b432318b9d784874abacf741a607342b" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:30.468576 5911 tablet_bootstrap.cc:492] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b: No bootstrap required, opened a new log
I20250628 07:54:30.468963 5911 ts_tablet_manager.cc:1397] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b: Time spent bootstrapping tablet: real 0.010s user 0.008s sys 0.000s
I20250628 07:54:30.470918 5911 raft_consensus.cc:357] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.471469 5911 raft_consensus.cc:383] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:30.471676 5911 raft_consensus.cc:738] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b432318b9d784874abacf741a607342b, State: Initialized, Role: FOLLOWER
I20250628 07:54:30.472209 5911 consensus_queue.cc:260] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.472652 5911 raft_consensus.cc:397] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:30.472855 5911 raft_consensus.cc:491] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:30.473125 5911 raft_consensus.cc:3058] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:30.478621 5911 raft_consensus.cc:513] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.479137 5911 leader_election.cc:304] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: b432318b9d784874abacf741a607342b; no voters:
I20250628 07:54:30.479674 5911 leader_election.cc:290] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:30.479809 5913 raft_consensus.cc:2802] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:30.480321 5913 raft_consensus.cc:695] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b [term 1 LEADER]: Becoming Leader. State: Replica: b432318b9d784874abacf741a607342b, State: Running, Role: LEADER
I20250628 07:54:30.481184 5911 ts_tablet_manager.cc:1428] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b: Time spent starting tablet: real 0.012s user 0.013s sys 0.000s
I20250628 07:54:30.481026 5913 consensus_queue.cc:237] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.481839 5911 tablet_bootstrap.cc:492] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b: Bootstrap starting.
I20250628 07:54:30.486727 5911 tablet_bootstrap.cc:654] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:30.487653 5772 catalog_manager.cc:5582] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b reported cstate change: term changed from 0 to 1, leader changed from <none> to b432318b9d784874abacf741a607342b (127.4.182.1). New cstate: current_term: 1 leader_uuid: "b432318b9d784874abacf741a607342b" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:30.491451 5911 tablet_bootstrap.cc:492] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b: No bootstrap required, opened a new log
I20250628 07:54:30.491776 5911 ts_tablet_manager.cc:1397] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b: Time spent bootstrapping tablet: real 0.010s user 0.009s sys 0.000s
I20250628 07:54:30.493389 5911 raft_consensus.cc:357] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.493752 5911 raft_consensus.cc:383] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:30.493963 5911 raft_consensus.cc:738] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b432318b9d784874abacf741a607342b, State: Initialized, Role: FOLLOWER
I20250628 07:54:30.494424 5911 consensus_queue.cc:260] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.494823 5911 raft_consensus.cc:397] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:30.495018 5911 raft_consensus.cc:491] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:30.495239 5911 raft_consensus.cc:3058] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:30.499472 5911 raft_consensus.cc:513] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.499939 5911 leader_election.cc:304] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: b432318b9d784874abacf741a607342b; no voters:
I20250628 07:54:30.500345 5911 leader_election.cc:290] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:30.500486 5913 raft_consensus.cc:2802] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:30.501246 5913 raft_consensus.cc:695] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b [term 1 LEADER]: Becoming Leader. State: Replica: b432318b9d784874abacf741a607342b, State: Running, Role: LEADER
I20250628 07:54:30.502094 5911 ts_tablet_manager.cc:1428] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b: Time spent starting tablet: real 0.010s user 0.004s sys 0.004s
I20250628 07:54:30.501899 5913 consensus_queue.cc:237] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.502871 5911 tablet_bootstrap.cc:492] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b: Bootstrap starting.
I20250628 07:54:30.506765 5911 tablet_bootstrap.cc:654] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:30.508543 5772 catalog_manager.cc:5582] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b reported cstate change: term changed from 0 to 1, leader changed from <none> to b432318b9d784874abacf741a607342b (127.4.182.1). New cstate: current_term: 1 leader_uuid: "b432318b9d784874abacf741a607342b" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:30.510816 5911 tablet_bootstrap.cc:492] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b: No bootstrap required, opened a new log
I20250628 07:54:30.511119 5911 ts_tablet_manager.cc:1397] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b: Time spent bootstrapping tablet: real 0.008s user 0.007s sys 0.000s
I20250628 07:54:30.512579 5911 raft_consensus.cc:357] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.512931 5911 raft_consensus.cc:383] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:30.513134 5911 raft_consensus.cc:738] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b432318b9d784874abacf741a607342b, State: Initialized, Role: FOLLOWER
I20250628 07:54:30.513710 5911 consensus_queue.cc:260] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.514114 5911 raft_consensus.cc:397] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:30.514319 5911 raft_consensus.cc:491] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:30.514552 5911 raft_consensus.cc:3058] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:30.520053 5911 raft_consensus.cc:513] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.520669 5911 leader_election.cc:304] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: b432318b9d784874abacf741a607342b; no voters:
I20250628 07:54:30.521211 5911 leader_election.cc:290] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:30.521332 5913 raft_consensus.cc:2802] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:30.521798 5913 raft_consensus.cc:695] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b [term 1 LEADER]: Becoming Leader. State: Replica: b432318b9d784874abacf741a607342b, State: Running, Role: LEADER
I20250628 07:54:30.522394 5913 consensus_queue.cc:237] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.522692 5911 ts_tablet_manager.cc:1428] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b: Time spent starting tablet: real 0.011s user 0.010s sys 0.000s
I20250628 07:54:30.523306 5911 tablet_bootstrap.cc:492] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b: Bootstrap starting.
I20250628 07:54:30.527531 5911 tablet_bootstrap.cc:654] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:30.528592 5772 catalog_manager.cc:5582] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b reported cstate change: term changed from 0 to 1, leader changed from <none> to b432318b9d784874abacf741a607342b (127.4.182.1). New cstate: current_term: 1 leader_uuid: "b432318b9d784874abacf741a607342b" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:30.531044 5911 tablet_bootstrap.cc:492] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b: No bootstrap required, opened a new log
I20250628 07:54:30.531338 5911 ts_tablet_manager.cc:1397] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b: Time spent bootstrapping tablet: real 0.008s user 0.007s sys 0.000s
I20250628 07:54:30.532923 5911 raft_consensus.cc:357] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.533352 5911 raft_consensus.cc:383] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:30.533553 5911 raft_consensus.cc:738] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b432318b9d784874abacf741a607342b, State: Initialized, Role: FOLLOWER
I20250628 07:54:30.533989 5911 consensus_queue.cc:260] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.534562 5911 raft_consensus.cc:397] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:30.534780 5911 raft_consensus.cc:491] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:30.535001 5911 raft_consensus.cc:3058] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:30.539269 5911 raft_consensus.cc:513] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.539755 5911 leader_election.cc:304] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: b432318b9d784874abacf741a607342b; no voters:
I20250628 07:54:30.540244 5911 leader_election.cc:290] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:30.540381 5913 raft_consensus.cc:2802] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:30.540848 5913 raft_consensus.cc:695] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b [term 1 LEADER]: Becoming Leader. State: Replica: b432318b9d784874abacf741a607342b, State: Running, Role: LEADER
I20250628 07:54:30.541569 5911 ts_tablet_manager.cc:1428] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b: Time spent starting tablet: real 0.010s user 0.008s sys 0.000s
I20250628 07:54:30.541440 5913 consensus_queue.cc:237] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.542150 5911 tablet_bootstrap.cc:492] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b: Bootstrap starting.
I20250628 07:54:30.546411 5911 tablet_bootstrap.cc:654] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:30.547447 5772 catalog_manager.cc:5582] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b reported cstate change: term changed from 0 to 1, leader changed from <none> to b432318b9d784874abacf741a607342b (127.4.182.1). New cstate: current_term: 1 leader_uuid: "b432318b9d784874abacf741a607342b" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:30.549880 5911 tablet_bootstrap.cc:492] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b: No bootstrap required, opened a new log
I20250628 07:54:30.550194 5911 ts_tablet_manager.cc:1397] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b: Time spent bootstrapping tablet: real 0.008s user 0.006s sys 0.000s
I20250628 07:54:30.551941 5911 raft_consensus.cc:357] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.552337 5911 raft_consensus.cc:383] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:30.552577 5911 raft_consensus.cc:738] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b432318b9d784874abacf741a607342b, State: Initialized, Role: FOLLOWER
I20250628 07:54:30.553128 5911 consensus_queue.cc:260] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.553681 5911 raft_consensus.cc:397] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:30.553952 5911 raft_consensus.cc:491] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:30.554179 5911 raft_consensus.cc:3058] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:30.558799 5911 raft_consensus.cc:513] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.559243 5911 leader_election.cc:304] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: b432318b9d784874abacf741a607342b; no voters:
I20250628 07:54:30.559650 5911 leader_election.cc:290] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:30.559785 5913 raft_consensus.cc:2802] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:30.560330 5913 raft_consensus.cc:695] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b [term 1 LEADER]: Becoming Leader. State: Replica: b432318b9d784874abacf741a607342b, State: Running, Role: LEADER
I20250628 07:54:30.561004 5913 consensus_queue.cc:237] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.561421 5911 ts_tablet_manager.cc:1428] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b: Time spent starting tablet: real 0.011s user 0.011s sys 0.000s
I20250628 07:54:30.562198 5911 tablet_bootstrap.cc:492] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b: Bootstrap starting.
I20250628 07:54:30.566113 5911 tablet_bootstrap.cc:654] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b: Neither blocks nor log segments found. Creating new log.
I20250628 07:54:30.567536 5772 catalog_manager.cc:5582] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b reported cstate change: term changed from 0 to 1, leader changed from <none> to b432318b9d784874abacf741a607342b (127.4.182.1). New cstate: current_term: 1 leader_uuid: "b432318b9d784874abacf741a607342b" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:30.569581 5911 tablet_bootstrap.cc:492] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b: No bootstrap required, opened a new log
I20250628 07:54:30.569900 5911 ts_tablet_manager.cc:1397] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.004s
I20250628 07:54:30.571336 5911 raft_consensus.cc:357] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.571758 5911 raft_consensus.cc:383] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20250628 07:54:30.571954 5911 raft_consensus.cc:738] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b432318b9d784874abacf741a607342b, State: Initialized, Role: FOLLOWER
I20250628 07:54:30.572444 5911 consensus_queue.cc:260] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.573000 5911 raft_consensus.cc:397] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20250628 07:54:30.573271 5911 raft_consensus.cc:491] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20250628 07:54:30.573555 5911 raft_consensus.cc:3058] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b [term 0 FOLLOWER]: Advancing to term 1
I20250628 07:54:30.577831 5911 raft_consensus.cc:513] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.578266 5911 leader_election.cc:304] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: b432318b9d784874abacf741a607342b; no voters:
I20250628 07:54:30.578658 5911 leader_election.cc:290] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b [CANDIDATE]: Term 1 election: Requested vote from peers
I20250628 07:54:30.578806 5913 raft_consensus.cc:2802] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Leader election won for term 1
I20250628 07:54:30.579514 5913 raft_consensus.cc:695] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b [term 1 LEADER]: Becoming Leader. State: Replica: b432318b9d784874abacf741a607342b, State: Running, Role: LEADER
I20250628 07:54:30.580327 5911 ts_tablet_manager.cc:1428] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b: Time spent starting tablet: real 0.010s user 0.011s sys 0.000s
I20250628 07:54:30.580242 5913 consensus_queue.cc:237] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } }
I20250628 07:54:30.585474 5772 catalog_manager.cc:5582] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b reported cstate change: term changed from 0 to 1, leader changed from <none> to b432318b9d784874abacf741a607342b (127.4.182.1). New cstate: current_term: 1 leader_uuid: "b432318b9d784874abacf741a607342b" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b432318b9d784874abacf741a607342b" member_type: VOTER last_known_addr { host: "127.4.182.1" port: 38721 } health_report { overall_health: HEALTHY } } }
I20250628 07:54:30.634912 4824 catalog_manager.cc:3511] Servicing AlterTable request from test_user:
table { table_name: "test_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "int_val2" type: INT32 is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } }
I20250628 07:54:30.666411 4824 tablet_server.cc:178] TabletServer@127.4.182.1:0 shutting down...
I20250628 07:54:30.667922 5941 tablet.cc:1722] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:30.670624 5947 tablet.cc:1722] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:30.672892 5947 tablet.cc:1620] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b: MemRowSet was empty: no flush needed.
I20250628 07:54:30.676198 5941 tablet.cc:1620] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b: MemRowSet was empty: no flush needed.
I20250628 07:54:30.705541 5941 tablet.cc:1722] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:30.708209 5941 tablet.cc:1620] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b: MemRowSet was empty: no flush needed.
I20250628 07:54:30.722523 5941 tablet.cc:1722] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:30.726212 5941 tablet.cc:1620] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b: MemRowSet was empty: no flush needed.
I20250628 07:54:30.728521 5947 tablet.cc:1722] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:30.729768 5943 tablet.cc:1722] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:30.728813 5938 tablet.cc:1722] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:30.736482 5938 tablet.cc:1620] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b: MemRowSet was empty: no flush needed.
I20250628 07:54:30.736265 5947 tablet.cc:1620] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b: MemRowSet was empty: no flush needed.
I20250628 07:54:30.736716 5943 tablet.cc:1620] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b: MemRowSet was empty: no flush needed.
I20250628 07:54:30.757972 5941 tablet.cc:1722] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:30.758059 5938 tablet.cc:1722] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:30.759819 5941 tablet.cc:1620] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b: MemRowSet was empty: no flush needed.
I20250628 07:54:30.759883 5938 tablet.cc:1620] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b: MemRowSet was empty: no flush needed.
I20250628 07:54:30.760317 5947 tablet.cc:1722] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b: Alter schema from (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
PRIMARY KEY (key)
) version 0 to (
10:key INT32 NOT NULL,
11:int_val INT32 NOT NULL,
12:int_val2 INT32 NULLABLE,
PRIMARY KEY (key)
) version 1
I20250628 07:54:30.762105 5947 tablet.cc:1620] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b: MemRowSet was empty: no flush needed.
I20250628 07:54:30.763631 4824 ts_tablet_manager.cc:1500] Shutting down tablet manager...
I20250628 07:54:30.764803 4824 tablet_replica.cc:331] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b: stopping tablet replica
I20250628 07:54:30.765578 4824 raft_consensus.cc:2241] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:30.766072 4824 raft_consensus.cc:2270] T 48d59f36eb4649979abe40b36ee99a2d P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:30.772253 4824 tablet_replica.cc:331] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b: stopping tablet replica
I20250628 07:54:30.772743 4824 raft_consensus.cc:2241] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:30.773097 4824 raft_consensus.cc:2270] T 93a328949c244318bf159e2809364706 P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:30.775712 4824 tablet_replica.cc:331] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b: stopping tablet replica
I20250628 07:54:30.776196 4824 raft_consensus.cc:2241] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:30.776554 4824 raft_consensus.cc:2270] T db2e41adf4bc4bdd91382ada3d6a1469 P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:30.778275 4824 tablet_replica.cc:331] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b: stopping tablet replica
I20250628 07:54:30.778749 4824 raft_consensus.cc:2241] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:30.779116 4824 raft_consensus.cc:2270] T 10e6dbbe5f4e49f4a280dfb6915fc2e7 P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:30.781077 4824 tablet_replica.cc:331] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b: stopping tablet replica
I20250628 07:54:30.781529 4824 raft_consensus.cc:2241] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:30.781883 4824 raft_consensus.cc:2270] T 0cc05509e7e64fa3a689b7e588f606cf P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:30.783699 4824 tablet_replica.cc:331] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b: stopping tablet replica
I20250628 07:54:30.784170 4824 raft_consensus.cc:2241] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:30.784528 4824 raft_consensus.cc:2270] T 4eed1597698f455fa3063183469e4a8d P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:30.786276 4824 tablet_replica.cc:331] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b: stopping tablet replica
I20250628 07:54:30.786713 4824 raft_consensus.cc:2241] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:30.787070 4824 raft_consensus.cc:2270] T 2b0b2d989359459384057b6dff3774fb P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:30.789001 4824 tablet_replica.cc:331] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b: stopping tablet replica
I20250628 07:54:30.789443 4824 raft_consensus.cc:2241] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:30.789788 4824 raft_consensus.cc:2270] T 75ed8b2968964c25a93e87c36669693b P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:30.791577 4824 tablet_replica.cc:331] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b: stopping tablet replica
I20250628 07:54:30.792009 4824 raft_consensus.cc:2241] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:30.792351 4824 raft_consensus.cc:2270] T 3119994d67e04de0960f8650aef742dd P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:30.794102 4824 tablet_replica.cc:331] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b: stopping tablet replica
I20250628 07:54:30.794499 4824 raft_consensus.cc:2241] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:30.794855 4824 raft_consensus.cc:2270] T c2e65581e3ac41b9bf13b227ace0a214 P b432318b9d784874abacf741a607342b [term 1 FOLLOWER]: Raft consensus is shut down!
==================
WARNING: ThreadSanitizer: data race (pid=4824)
Write of size 8 at 0x7b4400112048 by main thread:
#0 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::__destruct_at_end(kudu::MemTracker**) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:429:12 (libkudu_util.so+0x3a7aa0)
#1 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libkudu_util.so+0x3a79b4)
#2 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libkudu_util.so+0x3a77ab)
#3 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libkudu_util.so+0x39fa61)
#4 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x39f613)
#5 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x3a68d5)
#6 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libtablet_test_util.so+0x35485)
#7 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libtablet_test_util.so+0x35429)
#8 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#9 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41cc24)
#10 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cfa05)
#11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cfc19)
#12 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1000f7)
#13 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x1000b9)
#14 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100079)
#15 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0xfffda)
#16 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc329)
#17 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc300)
#18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc2c0)
#19 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b09a6)
#20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b08c4)
#21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b067b)
#22 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x288d61)
#23 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b3cce)
#24 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#25 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#26 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#27 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#28 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#29 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#30 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#31 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#32 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#33 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#34 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#35 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#36 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#37 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#38 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#39 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#40 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#41 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#42 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#43 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#44 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#45 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#46 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Previous read of size 8 at 0x7b4400112048 by thread T75 (mutexes: write M197167946966685248):
#0 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::end() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:1536:30 (libkudu_util.so+0x3a097a)
#1 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:284:22 (libkudu_util.so+0x39e056)
#2 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d5f0)
#3 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416652)
#4 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415de3)
#5 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419ff1)
#6 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419fa9)
#7 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419f39)
#8 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419f01)
#9 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41912d)
#10 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#11 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#12 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#13 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#14 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#15 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#16 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#17 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#18 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#19 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#20 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e4ed2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d72d6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41ded0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d8ae)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d1717)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1d5e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b3bc7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Location is heap block of size 264 at 0x7b4400111fc0 allocated by thread T158:
#0 operator new(unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:64 (catalog_manager-test+0x33f397)
#1 std::__1::__libcpp_allocate(unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:253:10 (libkudu_util.so+0x29b1e6)
#2 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::allocate(unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1789:34 (libkudu_util.so+0x3a61b5)
#3 std::__1::enable_if<!(is_array<long&>::value), std::__1::shared_ptr<long&> >::type std::__1::make_shared<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler&&...) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4290:45 (libkudu_util.so+0x3a5f91)
#4 std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/make_shared.h:61:12 (libkudu_util.so+0x39f8c5)
#5 kudu::MemTracker::CreateTracker(long, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:54:34 (libkudu_util.so+0x39d8ee)
#6 kudu::tablet::OpTracker::StartMemoryTracking(std::__1::shared_ptr<kudu::MemTracker> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:311:20 (libtablet.so+0x41e0ab)
#7 kudu::tablet::TabletReplica::Start(kudu::consensus::ConsensusBootstrapInfo const&, std::__1::shared_ptr<kudu::tablet::Tablet>, kudu::clock::Clock*, std::__1::shared_ptr<kudu::rpc::Messenger>, scoped_refptr<kudu::rpc::ResultTracker>, scoped_refptr<kudu::log::Log>, kudu::ThreadPool*, kudu::DnsResolver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:277:19 (libtablet.so+0x3d0a66)
#8 kudu::tserver::TSTabletManager::OpenTablet(scoped_refptr<kudu::tablet::TabletReplica> const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&, std::__1::atomic<int>*, std::__1::atomic<int>*, kudu::Timer*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1430:18 (libtserver.so+0x2b94a0)
#9 kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:703:3 (libtserver.so+0x2cd411)
#10 decltype(std::__1::forward<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&>(fp)()) std::__1::__invoke<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&>(kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x2cd3b9)
#11 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&>(kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x2cd349)
#12 std::__1::__function::__alloc_func<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11, std::__1::allocator<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x2cd311)
#13 std::__1::__function::__func<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11, std::__1::allocator<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_11>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x2cc5ed)
#14 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#15 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#16 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#17 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#18 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#19 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#20 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#21 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#22 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#23 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#24 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
Mutex M197167946966685248 is already destroyed.
Thread T75 'apply [worker]-' (tid=5947, running) created by thread T95 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c39a5)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642:15 (libkudu_util.so+0x442eca)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:146:12 (libmaster.so+0x2d49b9)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45f7cb)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45da6f)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45fd32)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x415180)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x415719)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x4179aa)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417938)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x4178b8)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417873)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x416a8c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3e9ff6)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4848)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa040)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x181723)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b014)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138534)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1384e3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138458)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138413)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x1376fc)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x161226)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x1491eb)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x1339bd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x137395)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137329)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1372b9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137281)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13657d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
Thread T158 (tid=5911, finished) created by thread T194 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c39a5)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642:15 (libkudu_util.so+0x442eca)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:146:12 (libmaster.so+0x2d49b9)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45f7cb)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45da6f)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45fd32)
#6 kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:703:3 (libtserver.so+0x2b6692)
#7 kudu::tserver::TabletServiceAdminImpl::CreateTablet(kudu::tserver::CreateTabletRequestPB const*, kudu::tserver::CreateTabletResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1475:34 (libtserver.so+0x266c99)
#8 kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/tserver/tserver_admin.service.cc:189:13 (libtserver_admin_proto.so+0x67d34)
#9 decltype(std::__1::forward<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver_admin_proto.so+0x67cc2)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver_admin_proto.so+0x67bf1)
#11 std::__1::__function::__alloc_func<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver_admin_proto.so+0x67b6c)
#12 std::__1::__function::__func<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver_admin_proto.so+0x66e22)
#13 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x1f2dcc)
#14 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x1f2206)
#15 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x1f1baf)
#16 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:229:15 (libkrpc.so+0x1f4ec3)
#17 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:92:5 (libkrpc.so+0x1f61e1)
#18 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x1f6199)
#19 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x1f6129)
#20 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x1f60f1)
#21 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x1f53ed)
#22 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#23 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#24 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:429:12 in std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::__destruct_at_end(kudu::MemTracker**)
==================
==================
WARNING: ThreadSanitizer: data race (pid=4824)
Write of size 8 at 0x7b0800098780 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x33fc69)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34bb99)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34bb39)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29b982)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29b919)
#5 std::__1::allocator<kudu::MemTracker*>::deallocate(kudu::MemTracker**, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a7b7a)
#6 std::__1::allocator_traits<std::__1::allocator<kudu::MemTracker*> >::deallocate(std::__1::allocator<kudu::MemTracker*>&, kudu::MemTracker**, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1533:14 (libkudu_util.so+0x3a79f9)
#7 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:466:9 (libkudu_util.so+0x3a77d7)
#8 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libkudu_util.so+0x39fa61)
#9 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x39f613)
#10 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x3a68d5)
#11 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libtablet_test_util.so+0x35485)
#12 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libtablet_test_util.so+0x35429)
#13 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#14 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41cc24)
#15 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cfa05)
#16 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cfc19)
#17 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1000f7)
#18 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x1000b9)
#19 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100079)
#20 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0xfffda)
#21 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc329)
#22 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc300)
#23 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc2c0)
#24 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b09a6)
#25 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b08c4)
#26 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b067b)
#27 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x288d61)
#28 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b3cce)
#29 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#30 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#31 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#32 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#33 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#34 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#35 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#36 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#37 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#38 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#39 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#40 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#41 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#42 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#43 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#44 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#45 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#46 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#47 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#48 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#49 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#50 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#51 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Previous read of size 8 at 0x7b0800098780 by thread T75 (mutexes: write M197167946966685248):
#0 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:5 (libkudu_util.so+0x39e092)
#1 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d5f0)
#2 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416652)
#3 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415de3)
#4 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419ff1)
#5 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419fa9)
#6 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419f39)
#7 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419f01)
#8 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41912d)
#9 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#10 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#11 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#12 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#13 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#14 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#15 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#16 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#17 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#18 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#19 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e4ed2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d72d6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41ded0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d8ae)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d1717)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1d5e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b3bc7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Mutex M197167946966685248 is already destroyed.
Thread T75 'apply [worker]-' (tid=5947, running) created by thread T95 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c39a5)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642:15 (libkudu_util.so+0x442eca)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:146:12 (libmaster.so+0x2d49b9)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45f7cb)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45da6f)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45fd32)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x415180)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x415719)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x4179aa)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417938)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x4178b8)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417873)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x416a8c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3e9ff6)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4848)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa040)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x181723)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b014)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138534)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1384e3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138458)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138413)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x1376fc)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x161226)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x1491eb)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x1339bd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x137395)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137329)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1372b9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137281)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13657d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 in operator delete(void*)
==================
==================
WARNING: ThreadSanitizer: data race (pid=4824)
Write of size 8 at 0x7b44000cfd70 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x33fc69)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34bb99)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34bb39)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29b982)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29b919)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a6f9d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a6940)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x39f61b)
#11 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x3a68d5)
#12 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libtablet_test_util.so+0x35485)
#13 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libtablet_test_util.so+0x35429)
#14 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#15 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41cc24)
#16 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cfa05)
#17 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cfc19)
#18 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1000f7)
#19 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x1000b9)
#20 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100079)
#21 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0xfffda)
#22 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc329)
#23 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc300)
#24 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc2c0)
#25 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b09a6)
#26 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b08c4)
#27 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b067b)
#28 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x288d61)
#29 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b3cce)
#30 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#31 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#32 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#33 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#34 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#35 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#36 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#37 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#38 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#39 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#40 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#41 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#42 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#43 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#44 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#45 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#46 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#47 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#48 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#49 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#50 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#51 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#52 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Previous atomic write of size 8 at 0x7b44000cfd70 by thread T75 (mutexes: write M197167946966685248):
#0 __tsan_atomic64_fetch_add /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:620 (catalog_manager-test+0x312f45)
#1 long std::__1::__cxx_atomic_fetch_add<long>(std::__1::__cxx_atomic_base_impl<long>*, long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1042:12 (libkudu_util.so+0x2a5fd3)
#2 std::__1::__atomic_base<long, true>::fetch_add(long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1706:17 (libkudu_util.so+0x2a5f19)
#3 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:30 (libkudu_util.so+0x3a0a32)
#4 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x39e0a1)
#5 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d5f0)
#6 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416652)
#7 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415de3)
#8 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419ff1)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419fa9)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419f39)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419f01)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41912d)
#13 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#14 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#15 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#16 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#17 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#18 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#19 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#20 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#21 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#22 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#23 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e4ed2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d72d6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41ded0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d8ae)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d1717)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1d5e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b3bc7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Mutex M197167946966685248 is already destroyed.
Thread T75 'apply [worker]-' (tid=5947, running) created by thread T95 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c39a5)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642:15 (libkudu_util.so+0x442eca)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:146:12 (libmaster.so+0x2d49b9)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45f7cb)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45da6f)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45fd32)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x415180)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x415719)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x4179aa)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417938)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x4178b8)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417873)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x416a8c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3e9ff6)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4848)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa040)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x181723)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b014)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138534)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1384e3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138458)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138413)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x1376fc)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x161226)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x1491eb)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x1339bd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x137395)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137329)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1372b9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137281)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13657d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 in operator delete(void*)
==================
==================
WARNING: ThreadSanitizer: data race (pid=4824)
Write of size 8 at 0x7b44000cfd78 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x33fc69)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34bb99)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34bb39)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29b982)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29b919)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a6f9d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a6940)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x39f61b)
#11 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x3a68d5)
#12 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libtablet_test_util.so+0x35485)
#13 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libtablet_test_util.so+0x35429)
#14 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#15 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41cc24)
#16 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cfa05)
#17 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cfc19)
#18 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1000f7)
#19 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x1000b9)
#20 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100079)
#21 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0xfffda)
#22 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc329)
#23 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc300)
#24 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc2c0)
#25 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b09a6)
#26 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b08c4)
#27 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b067b)
#28 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x288d61)
#29 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b3cce)
#30 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#31 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#32 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#33 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#34 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#35 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#36 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#37 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#38 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#39 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#40 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#41 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#42 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#43 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#44 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#45 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#46 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#47 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#48 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#49 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#50 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#51 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#52 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Previous atomic write of size 8 at 0x7b44000cfd78 by thread T75 (mutexes: write M197167946966685248):
#0 __tsan_atomic64_compare_exchange_val /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:850 (catalog_manager-test+0x31a018)
#1 bool std::__1::__cxx_atomic_compare_exchange_weak<long>(std::__1::__cxx_atomic_base_impl<long>*, long*, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1031:12 (liblog.so+0x8bad4)
#2 std::__1::__atomic_base<long, false>::compare_exchange_weak(long&, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1622:17 (liblog.so+0x8b8f9)
#3 void kudu::AtomicStoreMax<long>(std::__1::atomic<long>&, long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/atomic-utils.h:33:13 (liblog.so+0x7e850)
#4 kudu::HighWaterMark::UpdateMax(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:80:5 (libkudu_util.so+0x3a5bd4)
#5 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:5 (libkudu_util.so+0x3a0a3d)
#6 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x39e0a1)
#7 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d5f0)
#8 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416652)
#9 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415de3)
#10 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419ff1)
#11 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419fa9)
#12 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419f39)
#13 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419f01)
#14 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41912d)
#15 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#16 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#17 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#18 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#19 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#21 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#22 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#23 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#24 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#25 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e4ed2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d72d6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41ded0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d8ae)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d1717)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1d5e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b3bc7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Mutex M197167946966685248 is already destroyed.
Thread T75 'apply [worker]-' (tid=5947, running) created by thread T95 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c39a5)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642:15 (libkudu_util.so+0x442eca)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:146:12 (libmaster.so+0x2d49b9)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45f7cb)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45da6f)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45fd32)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x415180)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x415719)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x4179aa)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417938)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x4178b8)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417873)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x416a8c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3e9ff6)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4848)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa040)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x181723)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b014)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138534)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1384e3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138458)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138413)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x1376fc)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x161226)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x1491eb)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x1339bd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x137395)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137329)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1372b9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137281)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13657d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 in operator delete(void*)
==================
==================
WARNING: ThreadSanitizer: data race (pid=4824)
Write of size 8 at 0x7b4400112030 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x33fc69)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34bb99)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34bb39)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29b982)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29b919)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a6f9d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a6940)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41cc24)
#11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cfa05)
#12 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cfc19)
#13 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1000f7)
#14 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x1000b9)
#15 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100079)
#16 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0xfffda)
#17 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc329)
#18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc300)
#19 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc2c0)
#20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b09a6)
#21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b08c4)
#22 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b067b)
#23 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x288d61)
#24 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b3cce)
#25 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#26 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#27 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#28 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#29 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#30 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#31 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#32 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#33 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#34 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#35 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#36 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#37 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#38 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#39 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#40 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#41 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#42 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#43 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#44 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#45 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#46 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#47 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Previous atomic write of size 8 at 0x7b4400112030 by thread T75 (mutexes: write M197167946966685248):
#0 __tsan_atomic64_fetch_add /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:620 (catalog_manager-test+0x312f45)
#1 long std::__1::__cxx_atomic_fetch_add<long>(std::__1::__cxx_atomic_base_impl<long>*, long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1042:12 (libkudu_util.so+0x2a5fd3)
#2 std::__1::__atomic_base<long, true>::fetch_add(long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1706:17 (libkudu_util.so+0x2a5f19)
#3 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:30 (libkudu_util.so+0x3a0a32)
#4 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x39e0a1)
#5 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d5f0)
#6 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416652)
#7 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415de3)
#8 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419ff1)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419fa9)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419f39)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419f01)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41912d)
#13 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#14 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#15 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#16 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#17 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#18 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#19 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#20 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#21 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#22 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#23 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e4ed2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d72d6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41ded0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d8ae)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d1717)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1d5e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b3bc7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Mutex M197167946966685248 is already destroyed.
Thread T75 'apply [worker]-' (tid=5947, running) created by thread T95 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c39a5)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642:15 (libkudu_util.so+0x442eca)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:146:12 (libmaster.so+0x2d49b9)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45f7cb)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45da6f)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45fd32)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x415180)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x415719)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x4179aa)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417938)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x4178b8)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417873)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x416a8c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3e9ff6)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4848)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa040)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x181723)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b014)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138534)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1384e3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138458)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138413)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x1376fc)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x161226)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x1491eb)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x1339bd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x137395)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137329)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1372b9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137281)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13657d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 in operator delete(void*)
==================
==================
WARNING: ThreadSanitizer: data race (pid=4824)
Write of size 8 at 0x7b4400112038 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x33fc69)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34bb99)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34bb39)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29b982)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29b919)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a6f9d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a6940)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41cc24)
#11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cfa05)
#12 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cfc19)
#13 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1000f7)
#14 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x1000b9)
#15 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100079)
#16 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0xfffda)
#17 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc329)
#18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc300)
#19 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc2c0)
#20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b09a6)
#21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b08c4)
#22 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b067b)
#23 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x288d61)
#24 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b3cce)
#25 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#26 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#27 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#28 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#29 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#30 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#31 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#32 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#33 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#34 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#35 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#36 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#37 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#38 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#39 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#40 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#41 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#42 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#43 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#44 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#45 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#46 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#47 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Previous atomic write of size 8 at 0x7b4400112038 by thread T75 (mutexes: write M197167946966685248):
#0 __tsan_atomic64_compare_exchange_val /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:850 (catalog_manager-test+0x31a018)
#1 bool std::__1::__cxx_atomic_compare_exchange_weak<long>(std::__1::__cxx_atomic_base_impl<long>*, long*, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1031:12 (liblog.so+0x8bad4)
#2 std::__1::__atomic_base<long, false>::compare_exchange_weak(long&, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1622:17 (liblog.so+0x8b8f9)
#3 void kudu::AtomicStoreMax<long>(std::__1::atomic<long>&, long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/atomic-utils.h:33:13 (liblog.so+0x7e850)
#4 kudu::HighWaterMark::UpdateMax(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:80:5 (libkudu_util.so+0x3a5bd4)
#5 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:5 (libkudu_util.so+0x3a0a3d)
#6 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x39e0a1)
#7 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d5f0)
#8 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416652)
#9 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415de3)
#10 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419ff1)
#11 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419fa9)
#12 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419f39)
#13 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419f01)
#14 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41912d)
#15 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#16 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#17 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#18 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#19 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#21 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#22 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#23 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#24 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#25 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e4ed2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d72d6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41ded0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d8ae)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d1717)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1d5e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b3bc7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Mutex M197167946966685248 is already destroyed.
Thread T75 'apply [worker]-' (tid=5947, running) created by thread T95 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c39a5)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642:15 (libkudu_util.so+0x442eca)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:146:12 (libmaster.so+0x2d49b9)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45f7cb)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45da6f)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45fd32)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x415180)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x415719)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x4179aa)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417938)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x4178b8)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417873)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x416a8c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3e9ff6)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4848)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa040)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x181723)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b014)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138534)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1384e3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138458)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138413)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x1376fc)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x161226)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x1491eb)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x1339bd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x137395)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137329)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1372b9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137281)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13657d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 in operator delete(void*)
==================
==================
WARNING: ThreadSanitizer: data race (pid=4824)
Write of size 8 at 0x7b4400112040 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (catalog_manager-test+0x33fc69)
#1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (catalog_manager-test+0x34bb99)
#2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (catalog_manager-test+0x34bb39)
#3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x29b982)
#4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x29b919)
#5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x3a6f9d)
#6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x3a6940)
#7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
#8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libtablet_test_util.so+0x35435)
#9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libtablet_test_util.so+0x31c18)
#10 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x41cc24)
#11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x3cfa05)
#12 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x3cfc19)
#13 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libitest_util.so+0x1000f7)
#14 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libitest_util.so+0x1000b9)
#15 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libitest_util.so+0x100079)
#16 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libitest_util.so+0xfffda)
#17 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3bc329)
#18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3bc300)
#19 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3bc2c0)
#20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b09a6)
#21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b08c4)
#22 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b067b)
#23 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x288d61)
#24 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1572:1 (libtserver.so+0x2b3cce)
#25 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#26 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#27 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#28 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#29 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#30 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#31 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#32 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#33 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#34 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#35 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#36 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#37 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#38 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#39 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#40 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#41 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#42 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#43 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#44 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#45 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#46 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#47 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Previous read of size 8 at 0x7b4400112040 by thread T75 (mutexes: write M197167946966685248):
#0 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::begin() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:1520:30 (libkudu_util.so+0x3a0939)
#1 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:284:22 (libkudu_util.so+0x39e04a)
#2 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x41d5f0)
#3 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:606:16 (libtablet.so+0x416652)
#4 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:577:5 (libtablet.so+0x415de3)
#5 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:65 (libtablet.so+0x419ff1)
#6 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x419fa9)
#7 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x419f39)
#8 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x419f01)
#9 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41912d)
#10 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#11 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#12 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#13 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#14 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#15 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#16 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#17 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#18 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#19 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#20 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:363 (catalog_manager-test+0x2e4ed2)
#1 base::SleepForNanoseconds(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/sysinfo.cc:96:10 (libgutil.so+0xb5f0a)
#2 kudu::SleepFor(kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:264:3 (libkudu_util.so+0x3d72d6)
#3 kudu::tablet::OpTracker::WaitForAllToFinish(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:298:5 (libtablet.so+0x41ded0)
#4 kudu::tablet::OpTracker::WaitForAllToFinish() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:260:3 (libtablet.so+0x41d8ae)
#5 kudu::tablet::TabletReplica::Stop() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:350:17 (libtablet.so+0x3d1717)
#6 kudu::tablet::TabletReplica::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:380:3 (libtablet.so+0x3d1d5e)
#7 kudu::tserver::TSTabletManager::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1545:14 (libtserver.so+0x2b3bc7)
#8 kudu::tserver::TabletServer::ShutdownImpl() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:189:22 (libtserver.so+0x255aee)
#9 kudu::tserver::TabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.h:67:5 (libtserver.so+0x25b2b9)
#10 kudu::tserver::MiniTabletServer::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/mini_tablet_server.cc:124:14 (libtserver.so+0x312eb2)
#11 kudu::cluster::InternalMiniCluster::ShutdownNodes(kudu::cluster::ClusterNodes) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:242:22 (libmini_cluster.so+0xd78c7)
#12 kudu::cluster::MiniCluster::Shutdown() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/mini_cluster.h:84:5 (libitest_util.so+0xe3814)
#13 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:98:3 (libmini_cluster.so+0xd62c3)
#14 kudu::cluster::InternalMiniCluster::~InternalMiniCluster() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/mini-cluster/internal_mini_cluster.cc:97:45 (libmini_cluster.so+0xd6359)
#15 std::__1::default_delete<kudu::cluster::InternalMiniCluster>::operator()(kudu::cluster::InternalMiniCluster*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (catalog_manager-test+0x34e837)
#16 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::reset(kudu::cluster::InternalMiniCluster*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (catalog_manager-test+0x34e79d)
#17 std::__1::unique_ptr<kudu::cluster::InternalMiniCluster, std::__1::default_delete<kudu::cluster::InternalMiniCluster> >::~unique_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:2471:19 (catalog_manager-test+0x34e72b)
#18 kudu::master::CatalogManagerRpcAndUserFunctionsTest::~CatalogManagerRpcAndUserFunctionsTest() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:277:7 (catalog_manager-test+0x34890d)
#19 kudu::master::CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test::~CatalogManagerRpcAndUserFunctionsTest_TestAlterTableWithUser_Test() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager-test.cc:384:1 (catalog_manager-test+0x348969)
#20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:318:24 (libgtest.so.1.12.1+0x65467)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64e2f)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64e2f)
#23 testing::TestInfo::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2859:5 (libgtest.so.1.12.1+0x43dc2)
#24 testing::TestSuite::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44d24)
#25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x59814)
#26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65cef)
#27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65cef)
#28 testing::UnitTest::Run() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58dcc)
#29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (catalog_manager-test+0x35d23b)
#30 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/test_main.cc:109:10 (catalog_manager-test+0x35c13c)
Mutex M197167946966685248 is already destroyed.
Thread T75 'apply [worker]-' (tid=5947, running) created by thread T95 at:
#0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (catalog_manager-test+0x2c39a5)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642:15 (libkudu_util.so+0x442eca)
#2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:146:12 (libmaster.so+0x2d49b9)
#3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x45f7cb)
#4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x45da6f)
#5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x45fd32)
#6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:524:23 (libtablet.so+0x415180)
#7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:474:5 (libtablet.so+0x415719)
#8 kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:186:21 (libtablet.so+0x4179aa)
#9 decltype(std::__1::forward<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x417938)
#10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&>(kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x4178b8)
#11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x417873)
#12 std::__1::__function::__func<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0, std::__1::allocator<kudu::tablet::OpDriver::Init(std::__1::unique_ptr<kudu::tablet::Op, std::__1::default_delete<kudu::tablet::Op> >, kudu::consensus::DriverType)::$_0>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x416a8c)
#13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x3e9ff6)
#14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x3e4848)
#15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3315:3 (libconsensus.so+0x1aa040)
#16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:195:12 (libconsensus.so+0x181723)
#17 kudu::consensus::RaftConsensus::NotifyCommitIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:914:3 (libconsensus.so+0x19b014)
#18 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x138534)
#19 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&>(fp)(std::__1::forward<kudu::consensus::PeerMessageQueueObserver*>(fp0))) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x1384e3)
#20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)&, kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x138458)
#21 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x138413)
#22 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*), std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const::'lambda'(kudu::consensus::PeerMessageQueueObserver*)>, void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x1376fc)
#23 std::__1::__function::__value_func<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libconsensus.so+0x161226)
#24 std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)>::operator()(kudu::consensus::PeerMessageQueueObserver*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libconsensus.so+0x1491eb)
#25 kudu::consensus::PeerMessageQueue::NotifyObserversTask(std::__1::function<void (kudu::consensus::PeerMessageQueueObserver*)> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1564:5 (libconsensus.so+0x1339bd)
#26 kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/consensus_queue.cc:1493:3 (libconsensus.so+0x137395)
#27 decltype(std::__1::forward<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(fp)()) std::__1::__invoke<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus.so+0x137329)
#28 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&>(kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus.so+0x1372b9)
#29 std::__1::__function::__alloc_func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus.so+0x137281)
#30 std::__1::__function::__func<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3, std::__1::allocator<kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange(long)::$_3>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus.so+0x13657d)
#31 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#32 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#33 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x460594)
#34 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x4628e1)
#35 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x462899)
#36 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x462829)
#37 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x4627f1)
#38 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x461aed)
#39 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5c124)
#40 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5bf59)
#41 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:698:3 (libkudu_util.so+0x443666)
SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 in operator delete(void*)
==================
I20250628 07:54:31.167255 4824 tablet_server.cc:195] TabletServer@127.4.182.1:0 shutdown complete.
I20250628 07:54:31.179616 4824 master.cc:561] Master@127.4.182.62:46145 shutting down...
I20250628 07:54:31.192390 4824 raft_consensus.cc:2241] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [term 1 LEADER]: Raft consensus shutting down.
I20250628 07:54:31.192900 4824 raft_consensus.cc:2270] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686 [term 1 FOLLOWER]: Raft consensus is shut down!
I20250628 07:54:31.193187 4824 tablet_replica.cc:331] T 00000000000000000000000000000000 P 186ba0a7aae64fd293bd115e37327686: stopping tablet replica
I20250628 07:54:31.209364 4824 master.cc:583] Master@127.4.182.62:46145 shutdown complete.
[ OK ] CatalogManagerRpcAndUserFunctionsTest.TestAlterTableWithUser (1260 ms)
[----------] 6 tests from CatalogManagerRpcAndUserFunctionsTest (6911 ms total)
[----------] Global test environment tear-down
[==========] 10 tests from 3 test suites ran. (6978 ms total)
[ PASSED ] 10 tests.
I20250628 07:54:31.234033 4824 logging.cc:424] LogThrottler /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/catalog_manager.cc:4671: suppressed but not reported on 26 messages since previous log ~3 seconds ago
I20250628 07:54:31.234272 4824 logging.cc:424] LogThrottler /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/proxy.cc:239: suppressed but not reported on 70 messages since previous log ~4 seconds ago
ThreadSanitizer: reported 7 warnings