[yugabyte#9934] [docdb] Don't update rocksdb_dir on Remote Bootstrap

Summary:
Data Race Issue on Remote Bootstrap with rocksdb_dir:

```
[ts-4] WARNING: ThreadSanitizer: data race (pid=8370)
[ts-4]   Write of size 8 at 0x7b50002606a8 by thread T46 (mutexes: write M257685187220342284):
[ts-4]     #0 std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >::assign(char const*, unsigned long) <null> (libc++.so.1+0xd5fa5)
[ts-4]     #1 std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >::operator=(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) <null> (libc++.so.1+0xd5e8a)
[ts-4]     #2 yb::tablet::KvStoreInfo::LoadFromPB(yb::tablet::KvStoreInfoPB const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) /nfusr/centos-gcp-cloud/jenkins-worker-r2ttyq/jenkins/jenkins-github-yugabyte-db-centos-master-clang7-tsan-339/build/tsan-clang7-dynamic-ninja/../../src/yb/tablet/tablet_metadata.cc:201:15 (libtablet.so+0x36795c)
[ts-4]     #3 yb::tablet::RaftGroupMetadata::LoadFromSuperBlock(yb::tablet::RaftGroupReplicaSuperBlockPB const&) /nfusr/centos-gcp-cloud/jenkins-worker-r2ttyq/jenkins/jenkins-github-yugabyte-db-centos-master-clang7-tsan-339/build/tsan-clang7-dynamic-ninja/../../src/yb/tablet/tablet_metadata.cc:525:5 (libtablet.so+0x36b36b)
[ts-4]     #4 yb::tablet::RaftGroupMetadata::ReplaceSuperBlock(yb::tablet::RaftGroupReplicaSuperBlockPB const&) /nfusr/centos-gcp-cloud/jenkins-worker-r2ttyq/jenkins/jenkins-github-yugabyte-db-centos-master-clang7-tsan-339/build/tsan-clang7-dynamic-ninja/../../src/yb/tablet/tablet_metadata.cc:586:3 (libtablet.so+0x36c078)
[ts-4]     #5 yb::tserver::RemoteBootstrapClient::Finish() /nfusr/centos-gcp-cloud/jenkins-worker-r2ttyq/jenkins/jenkins-github-yugabyte-db-centos-master-clang7-tsan-339/build/tsan-clang7-dynamic-ninja/../../src/yb/tserver/remote_bootstrap_client.cc:421:3 (libtserver.so+0x1d1699)
[ts-4]     #6 yb::tserver::TSTabletManager::StartRemoteBootstrap(yb::consensus::StartRemoteBootstrapRequestPB const&) /nfusr/centos-gcp-cloud/jenkins-worker-r2ttyq/jenkins/jenkins-github-yugabyte-db-centos-master-clang7-tsan-339/build/tsan-clang7-dynamic-ninja/../../src/yb/tserver/ts_tablet_manager.cc:1099:3 (libtserver.so+0x267088)
[ts-4]     #7 yb::tserver::ConsensusServiceImpl::StartRemoteBootstrap(yb::consensus::StartRemoteBootstrapRequestPB const*, yb::consensus::StartRemoteBootstrapResponsePB*, yb::rpc::RpcContext) /nfusr/centos-gcp-cloud/jenkins-worker-r2ttyq/jenkins/jenkins-github-yugabyte-db-centos-master-clang7-tsan-339/build/tsan-clang7-dynamic-ninja/../../src/yb/tserver/tablet_service.cc:2767:31 (libtserver.so+0x21844b)
```

```
[ts-4]   Previous read of size 8 at 0x7b50002606a8 by thread T21:
[ts-4]     #0 std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >::__get_long_size() const /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210813185027-9a29e26965-centos7-x86_64-clang7/installed/tsan/libcxx/include/c++/v1/string:1468:34 (libtablet.so+0x36d6a2)
[ts-4]     #1 std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >::size() const /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210813185027-9a29e26965-centos7-x86_64-clang7/installed/tsan/libcxx/include/c++/v1/string:941 (libtablet.so+0x36d6a2)
[ts-4]     #2 std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >::empty() const /opt/yb-build/thirdparty/yugabyte-db-thirdparty-v20210813185027-9a29e26965-centos7-x86_64-clang7/installed/tsan/libcxx/include/c++/v1/string:957 (libtablet.so+0x36d6a2)
[ts-4]     #3 yb::tablet::RaftGroupMetadata::data_root_dir() const /nfusr/centos-gcp-cloud/jenkins-worker-r2ttyq/jenkins/jenkins-github-yugabyte-db-centos-master-clang7-tsan-339/build/tsan-clang7-dynamic-ninja/../../src/yb/tablet/tablet_metadata.cc:773 (libtablet.so+0x36d6a2)
[ts-4]     #4 yb::tserver::TSTabletManager::CreateReportedTabletPB(std::__1::shared_ptr<yb::tablet::TabletPeer> const&, yb::master::ReportedTabletPB*) /nfusr/centos-gcp-cloud/jenkins-worker-r2ttyq/jenkins/jenkins-github-yugabyte-db-centos-master-clang7-tsan-339/build/tsan-clang7-dynamic-ninja/../../src/yb/tserver/ts_tablet_manager.cc:1840:68 (libtserver.so+0x26c6a2)
[ts-4]     #5 yb::tserver::TSTabletManager::GenerateTabletReport(yb::master::TabletReportPB*, bool) /nfusr/centos-gcp-cloud/jenkins-worker-r2ttyq/jenkins/jenkins-github-yugabyte-db-centos-master-clang7-tsan-339/build/tsan-clang7-dynamic-ninja/../../src/yb/tserver/ts_tablet_manager.cc:1925:5 (libtserver.so+0x26d072)
[ts-4]     #6 yb::tserver::Heartbeater::Thread::TryHeartbeat() /nfusr/centos-gcp-cloud/jenkins-worker-r2ttyq/jenkins/jenkins-github-yugabyte-db-centos-master-clang7-tsan-339/build/tsan-clang7-dynamic-ninja/../../src/yb/tserver/heartbeater.cc:371:32 (libtserver.so+0x1a2331)
[ts-4]     #7 yb::tserver::Heartbeater::Thread::DoHeartbeat() /nfusr/centos-gcp-cloud/jenkins-worker-r2ttyq/jenkins/jenkins-github-yugabyte-db-centos-master-clang7-tsan-339/build/tsan-clang7-dynamic-ninja/../../src/yb/tserver/heartbeater.cc:530:19 (libtserver.so+0x1a3678)
```

Test Plan:
ybd tsan --gtest_filter LoadBalancerMultiTableTest.GlobalLeaderBalancing
ybd tsan --gtest_filter LoadBalancerMultiTableTest.GlobalLoadBalancing

Reviewers: sergei

Reviewed By: sergei

Subscribers: ybase, bogdan

Differential Revision: https://phabricator.dev.yugabyte.com/D12906

Author: mikhpolitov

src/yb/tablet/tablet_metadata.cc

@@ -196,9 +196,13 @@ Status KvStoreInfo::LoadTablesFromPB(
   return Status::OK();
 }
 
-Status KvStoreInfo::LoadFromPB(const KvStoreInfoPB& pb, const TableId& primary_table_id) {
+Status KvStoreInfo::LoadFromPB(const KvStoreInfoPB& pb,
+                               const TableId& primary_table_id,
+                               bool local_superblock) {
   kv_store_id = KvStoreId(pb.kv_store_id());
-  rocksdb_dir = pb.rocksdb_dir();
+  if (local_superblock) {
+    rocksdb_dir = pb.rocksdb_dir();
+  }
   lower_bound_key = pb.lower_bound_key();
   upper_bound_key = pb.upper_bound_key();
   has_been_fully_compacted = pb.has_been_fully_compacted();
@@ -489,18 +493,19 @@ Status RaftGroupMetadata::LoadFromDisk() {
 
   RaftGroupReplicaSuperBlockPB superblock;
   RETURN_NOT_OK(ReadSuperBlockFromDisk(&superblock));
-  RETURN_NOT_OK_PREPEND(LoadFromSuperBlock(superblock),
+  RETURN_NOT_OK_PREPEND(LoadFromSuperBlock(superblock, /* local_superblock = */ true),
                         "Failed to load data from superblock protobuf");
   state_ = kInitialized;
   return Status::OK();
 }
 
-Status RaftGroupMetadata::LoadFromSuperBlock(const RaftGroupReplicaSuperBlockPB& superblock) {
+Status RaftGroupMetadata::LoadFromSuperBlock(const RaftGroupReplicaSuperBlockPB& superblock,
+                                             bool local_superblock) {
   if (!superblock.has_kv_store()) {
     // Backward compatibility for tablet=KV-store=raft-group.
     RaftGroupReplicaSuperBlockPB superblock_migrated(superblock);
     RETURN_NOT_OK(MigrateSuperblock(&superblock_migrated));
-    RETURN_NOT_OK(LoadFromSuperBlock(superblock_migrated));
+    RETURN_NOT_OK(LoadFromSuperBlock(superblock_migrated, local_superblock));
     return Flush();
   }
 
@@ -522,7 +527,9 @@ Status RaftGroupMetadata::LoadFromSuperBlock(const RaftGroupReplicaSuperBlockPB&
     primary_table_id_ = superblock.primary_table_id();
     colocated_ = superblock.colocated();
 
-    RETURN_NOT_OK(kv_store_.LoadFromPB(superblock.kv_store(), primary_table_id_));
+    RETURN_NOT_OK(kv_store_.LoadFromPB(superblock.kv_store(),
+                                       primary_table_id_,
+                                       local_superblock));
 
     wal_dir_ = superblock.wal_dir();
     tablet_data_state_ = superblock.tablet_data_state();
@@ -571,7 +578,7 @@ Status RaftGroupMetadata::Flush() {
     std::lock_guard<MutexType> lock(data_mutex_);
     ToSuperBlockUnlocked(&pb);
   }
-  RETURN_NOT_OK(ReplaceSuperBlockUnlocked(pb));
+  RETURN_NOT_OK(SaveToDiskUnlocked(pb));
   TRACE("Metadata flushed");
 
   return Status::OK();
@@ -580,16 +587,16 @@ Status RaftGroupMetadata::Flush() {
 Status RaftGroupMetadata::ReplaceSuperBlock(const RaftGroupReplicaSuperBlockPB &pb) {
   {
     MutexLock l(flush_lock_);
-    RETURN_NOT_OK_PREPEND(ReplaceSuperBlockUnlocked(pb), "Unable to replace superblock");
+    RETURN_NOT_OK_PREPEND(SaveToDiskUnlocked(pb), "Unable to replace superblock");
   }
 
-  RETURN_NOT_OK_PREPEND(LoadFromSuperBlock(pb),
+  RETURN_NOT_OK_PREPEND(LoadFromSuperBlock(pb, /* local_superblock = */ false),
                         "Failed to load data from superblock protobuf");
 
   return Status::OK();
 }
 
-Status RaftGroupMetadata::ReplaceSuperBlockUnlocked(const RaftGroupReplicaSuperBlockPB &pb) {
+Status RaftGroupMetadata::SaveToDiskUnlocked(const RaftGroupReplicaSuperBlockPB &pb) {
   flush_lock_.AssertAcquired();
 
   string path = fs_manager_->GetRaftGroupMetadataPath(raft_group_id_);
@@ -966,7 +973,7 @@ Result<RaftGroupMetadataPtr> RaftGroupMetadata::CreateSubtabletMetadata(
   ToSuperBlock(&superblock);
 
   RaftGroupMetadataPtr metadata(new RaftGroupMetadata(fs_manager_, raft_group_id_));
-  RETURN_NOT_OK(metadata->LoadFromSuperBlock(superblock));
+  RETURN_NOT_OK(metadata->LoadFromSuperBlock(superblock, /* local_superblock = */ true));
   metadata->raft_group_id_ = raft_group_id;
   metadata->wal_dir_ = GetSubRaftGroupWalDir(raft_group_id);
   metadata->kv_store_.lower_bound_key = lower_bound_key;

src/yb/tablet/tablet_metadata.h

@@ -134,7 +134,9 @@ struct KvStoreInfo {
         rocksdb_dir(rocksdb_dir_),
         snapshot_schedules(snapshot_schedules_.begin(), snapshot_schedules_.end()) {}
 
-  CHECKED_STATUS LoadFromPB(const KvStoreInfoPB& pb, const TableId& primary_table_id);
+  CHECKED_STATUS LoadFromPB(const KvStoreInfoPB& pb,
+                            const TableId& primary_table_id,
+                            bool local_superblock);
 
   CHECKED_STATUS LoadTablesFromPB(
       const google::protobuf::RepeatedPtrField<TableInfoPB>& pbs, const TableId& primary_table_id);
@@ -529,13 +531,14 @@ class RaftGroupMetadata : public RefCountedThreadSafe<RaftGroupMetadata> {
   CHECKED_STATUS LoadFromDisk();
 
   // Update state of metadata to that of the given superblock PB.
-  CHECKED_STATUS LoadFromSuperBlock(const RaftGroupReplicaSuperBlockPB& superblock);
+  CHECKED_STATUS LoadFromSuperBlock(const RaftGroupReplicaSuperBlockPB& superblock,
+                                    bool local_superblock);
 
   CHECKED_STATUS ReadSuperBlock(RaftGroupReplicaSuperBlockPB *pb);
 
   // Fully replace superblock.
   // Requires 'flush_lock_'.
-  CHECKED_STATUS ReplaceSuperBlockUnlocked(const RaftGroupReplicaSuperBlockPB &pb);
+  CHECKED_STATUS SaveToDiskUnlocked(const RaftGroupReplicaSuperBlockPB &pb);
 
   // Requires 'data_mutex_'.
   void ToSuperBlockUnlocked(RaftGroupReplicaSuperBlockPB* superblock) const REQUIRES(data_mutex_);

src/yb/tserver/remote_bootstrap_client.cc

@@ -409,7 +409,7 @@ Status RemoteBootstrapClient::Finish() {
   CHECK(started_);
 
   CHECK(downloaded_wal_);
-  CHECK(downloaded_rocksdb_files_) << "files not downloaded";;
+  CHECK(downloaded_rocksdb_files_) << "files not downloaded";
 
   RETURN_NOT_OK(WriteConsensusMetadata());