add offline check for ops

Author: yamingk

conanfile.py

@@ -46,7 +46,8 @@ def build_requirements(self):
         self.test_requires("gtest/1.14.0")
 
     def requirements(self):
-        self.requires("homestore/[~6.20]@oss/master", transitive_headers=True)
+        #self.requires("homestore/[~6.20]@oss/master", transitive_headers=True)
+        self.requires("homestore/6.20.0", transitive_headers=True)
         self.requires("iomgr/[^11.3]@oss/master", transitive_headers=True)
         self.requires("sisl/[^12.2]@oss/master", transitive_headers=True)
         self.requires("lz4/1.9.4", override=True)

src/include/homeblks/volume_mgr.hpp

@@ -11,7 +11,7 @@
 namespace homeblocks {
 
 ENUM(VolumeError, uint16_t, UNKNOWN = 1, INVALID_ARG, TIMEOUT, UNKNOWN_VOLUME, UNSUPPORTED_OP, CRC_MISMATCH,
-     NO_SPACE_LEFT, DRIVE_WRITE_ERROR, INTERNAL_ERROR, INDEX_ERROR);
+     NO_SPACE_LEFT, DRIVE_WRITE_ERROR, INTERNAL_ERROR, INDEX_ERROR, VOLUME_OFFLINE);
 
 using lba_t = uint64_t;
 using lba_count_t = uint32_t;

src/lib/homeblks_impl.cpp

@@ -373,6 +373,9 @@ void HomeBlocksImpl::on_hb_meta_blk_found(sisl::byte_view const& buf, void* cook
         // if it is a gracefuln shutdown, this flag should be set again in shutdown routine;
         sb_->clear_flag(SB_FLAGS_GRACEFUL_SHUTDOWN);
         LOGI("System was shutdown gracefully");
+    } else if (sb_->test_flag(SB_FLAGS_RESTRICTED)) {
+        is_restricted_.store(true);
+        LOGI("System is booted into restricted mode, reason: {}", sb_->flag);
     } else {
         LOGI("System experienced sudden crash since last boot");
     }
@@ -476,9 +479,12 @@ void HomeBlocksImpl::exit_fc(VolumePtr& vol) { vol->state_change(vol_state::ONLI
 
 void HomeBlocksImpl::fault_containment(const VolumePtr vol, const std::string& reason) {
     if (vol == nullptr) {
-        // TODO: Put entire HB into offline;
-        assert(0);
-        LOGE("Not implemented yet:  putting HB into fault containment, reason: {}", reason);
+        // Put entire HB into offline;
+        std::scoped_lock lg(sb_lock_);
+        sb_->set_flag(SB_FLAGS_RESTRICTED);
+        is_restricted_.store(true);
+        sb_.write();
+        LOGE("Putting HB into restricted mode, reason: {}", reason);
         return;
     }
 

src/lib/homeblks_impl.hpp

@@ -73,13 +73,16 @@ class HomeBlocksImpl : public HomeBlocks, public VolumeManager, public std::enab
     std::unordered_map< std::string, shared< VolumeIndexTable > > idx_tbl_map_;
 
     bool recovery_done_{false};
+    std::mutex sb_lock_;
     superblk< homeblks_sb_t > sb_;
     peer_id_t our_uuid_;
     shared< VolumeChunkSelector > chunk_selector_;
     std::unique_ptr< sisl::IDReserver > ordinal_reserver_;
 
     sisl::atomic_counter< uint64_t > outstanding_reqs_{0};
     bool shutdown_started_{false};
+    std::atomic< bool > is_restricted_{false}; // avoid taking lock in IO path;
+
     folly::Promise< folly::Unit > shutdown_promise_;
     iomgr::io_fiber_t reaper_fiber_;
     iomgr::timer_handle_t vol_gc_timer_hdl_{iomgr::null_timer_handle};
@@ -151,6 +154,7 @@ class HomeBlocksImpl : public HomeBlocks, public VolumeManager, public std::enab
     void fault_containment(const VolumePtr vol, const std::string& reason = "");
     bool fc_on() const;
     void exit_fc(VolumePtr& vol);
+    bool is_restricted() const { return is_restricted_.load(); }
 
 public:
     // public static APIs;

src/lib/volume/volume.hpp

@@ -167,6 +167,7 @@ class Volume : public std::enable_shared_from_this< Volume > {
 
     bool is_destroying() const { return m_state_.load() == vol_state::DESTROYING; }
     bool is_destroy_started() const { return destroy_started_.load(); }
+    bool is_offline() const { return m_state_.load() == vol_state::OFFLINE; }
 
     //
     // This API will be called to set the volume state and persist to disk;
@@ -226,7 +227,7 @@ class Volume : public std::enable_shared_from_this< Volume > {
     sisl::atomic_counter< uint64_t > outstanding_reqs_{0}; // number of outstanding requests
     std::atomic< bool > destroy_started_{
         false}; // indicates if volume destroy has started, avoid destroy to be executed more than once.
-    std::atomic< vol_state > m_state_; // in-memory sb state;
+    std::atomic< vol_state > m_state_; // in-memory sb state, avoid taking lock in IO path;
 };
 
 struct vol_repl_ctx : public homestore::repl_req_ctx {

src/lib/volume_mgr.cpp

@@ -75,6 +75,11 @@ shared< hs_index_table_t > HomeBlocksImpl::recover_index_table(homestore::superb
 // cause crash;
 //
 VolumeManager::NullAsyncResult HomeBlocksImpl::create_volume(VolumeInfo&& vol_info) {
+    if (is_restricted()) {
+        LOGE("Can't serve volume create, System is in restricted mode.");
+        return folly::makeUnexpected(VolumeError::UNSUPPORTED_OP);
+    }
+
     inc_ref();
     auto id = vol_info.id;
 
@@ -115,6 +120,20 @@ VolumeManager::NullAsyncResult HomeBlocksImpl::create_volume(VolumeInfo&& vol_in
 // vol in destroying state already indicates an outstanding volume which consumed in no_outstanding_vols() API;
 //
 VolumeManager::NullAsyncResult HomeBlocksImpl::remove_volume(const volume_id_t& id) {
+    if (is_restricted()) {
+        LOGE("Can't serve volume remove, System is in restricted mode.");
+        return folly::makeUnexpected(VolumeError::UNSUPPORTED_OP);
+    }
+
+    auto vol = lookup_volume(id);
+    if (vol == nullptr) {
+        LOGE("Volume with id {} not found, cannot remove", boost::uuids::to_string(id));
+        return folly::makeUnexpected(VolumeError::INVALID_ARG);
+    } else if (vol->is_offline()) {
+        LOGE("Volume {} is offline, cannot remove", vol->id_str());
+        return folly::makeUnexpected(VolumeError::VOLUME_OFFLINE);
+    }
+
     LOGINFO("remove_volume with input id: {}", boost::uuids::to_string(id));
     iomanager.run_on_forget(iomgr::reactor_regex::random_worker, [this, id]() {
         // 1. get the volume ptr from the map;
@@ -208,6 +227,14 @@ void HomeBlocksImpl::get_volume_ids(std::vector< volume_id_t >& vol_ids) const {
 }
 
 VolumeManager::NullAsyncResult HomeBlocksImpl::write(const VolumePtr& vol, const vol_interface_req_ptr& req) {
+    if (is_restricted()) {
+        LOGE("Can't serve write, System is in restricted mode.");
+        return folly::makeUnexpected(VolumeError::UNSUPPORTED_OP);
+    } else if (vol->is_offline()) {
+        LOGE("Can't serve write, Volume {} is offline.", vol->id_str());
+        return folly::makeUnexpected(VolumeError::VOLUME_OFFLINE);
+    }
+
     if (vol->is_destroying() || is_shutting_down()) {
         LOGE(
             "Can't serve write, Volume {} is_destroying: {} is either in destroying state or System is shutting down. ",
@@ -226,6 +253,14 @@ VolumeManager::NullAsyncResult HomeBlocksImpl::write(const VolumePtr& vol, const
 }
 
 VolumeManager::NullAsyncResult HomeBlocksImpl::read(const VolumePtr& vol, const vol_interface_req_ptr& req) {
+    if (is_restricted()) {
+        LOGE("Can't serve read, System is in restricted mode.");
+        return folly::makeUnexpected(VolumeError::UNSUPPORTED_OP);
+    } else if (vol->is_offline()) {
+        LOGE("Can't serve read, Volume {} is offline.", vol->id_str());
+        return folly::makeUnexpected(VolumeError::VOLUME_OFFLINE);
+    }
+
     if (vol->is_destroying() || is_shutting_down()) {
         LOGE("Can't serve read, Volume {} is_destroying: {} is either in destroying state or System is shutting down. ",
              vol->id_str(), vol->is_destroying());
@@ -245,6 +280,14 @@ VolumeManager::NullAsyncResult HomeBlocksImpl::read(const VolumePtr& vol, const
 VolumeManager::NullAsyncResult HomeBlocksImpl::unmap(const VolumePtr& vol, const vol_interface_req_ptr& req) {
     LOGWARN("Unmap to vol: {} not implemented", vol->id_str());
 
+    if (is_restricted()) {
+        LOGE("Can't serve unmap, System is in restricted mode.");
+        return folly::makeUnexpected(VolumeError::UNSUPPORTED_OP);
+    } else if (vol->is_offline()) {
+        LOGE("Can't serve unmap, Volume {} is offline.", vol->id_str());
+        return folly::makeUnexpected(VolumeError::VOLUME_OFFLINE);
+    }
+
     if (vol->is_destroying() || is_shutting_down()) {
         LOGE(
             "Can't serve unmap, Volume {} is_destroying: {} is either in destroying state or System is shutting down. ",