diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index cabcba988f631..efc5e1ed62b8d 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -177,7 +177,7 @@ if(WITH_RADOSGW_DBSTORE) list(APPEND librgw_common_srcs rgw_sal_dbstore.cc) endif() if(WITH_RADOSGW_MOTR) - list(APPEND librgw_common_srcs rgw_sal_motr.cc) + list(APPEND librgw_common_srcs rgw_sal_motr.cc motr/gc/gc.cc) endif() if(WITH_JAEGER) list(APPEND librgw_common_srcs rgw_tracer.cc) diff --git a/src/rgw/motr/gc/gc.cc b/src/rgw/motr/gc/gc.cc new file mode 100644 index 0000000000000..997aff0c14494 --- /dev/null +++ b/src/rgw/motr/gc/gc.cc @@ -0,0 +1,31 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=2 sw=2 expandtab ft=cpp + +/* + * Garbage Collector implementation for the CORTX Motr backend + * + * Copyright (C) 2022 Seagate Technology LLC and/or its Affiliates + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "gc.h" + +void *MotrGC::entry() { + std::unique_lock lk(mtx); + ldpp_dout(dpp, 10) << __func__ << ": Motr GC started" << dendl; + + do { + ldpp_dout(dpp, 10) << __func__ << ": In a Motr GC loop." << dendl; + cv.wait_for(lk, std::chrono::milliseconds(gc_interval * 10)); + } while (! stop_signalled); + + ldpp_dout(dpp, 0) << __func__ << ": Stop signalled called.#" + << stop_signalled << dendl; + return nullptr; +} + diff --git a/src/rgw/motr/gc/gc.h b/src/rgw/motr/gc/gc.h new file mode 100644 index 0000000000000..ae02863f31e09 --- /dev/null +++ b/src/rgw/motr/gc/gc.h @@ -0,0 +1,47 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=2 sw=2 expandtab ft=cpp + +/* + * Garbage Collector Classes for the CORTX Motr backend + * + * Copyright (C) 2022 Seagate Technology LLC and/or its Affiliates + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef __MOTR_GC_H__ +#define __MOTR_GC_H__ + +#include "rgw_sal_motr.h" +#include "common/Thread.h" +#include +#include + +class MotrGC : public Thread { + private: + const DoutPrefixProvider *dpp; + rgw::sal::Store *store; + std::mutex mtx; + std::condition_variable cv; + bool stop_signalled = false; + uint32_t gc_interval = 60*60; // default: 24*60*60 sec + uint32_t gc_obj_min_wait = 60*60; // 60*60sec default + + public: + MotrGC(const DoutPrefixProvider *_dpp, rgw::sal::Store* _store) : + dpp(_dpp), store(_store) {} + + void *entry() override; + + void signal_stop() { + std::lock_guard lk_guard(mtx); + stop_signalled = true; + cv.notify_all(); + } +}; + +#endif diff --git a/src/rgw/rgw_sal.cc b/src/rgw/rgw_sal.cc index 88dae299a8cc8..3eb1e2e05eeb4 100644 --- a/src/rgw/rgw_sal.cc +++ b/src/rgw/rgw_sal.cc @@ -117,7 +117,13 @@ rgw::sal::Store* StoreManager::init_storage_provider(const DoutPrefixProvider* d ldpp_dout(dpp, 0) << "newMotrStore() failed!" << dendl; return store; } - ((rgw::sal::MotrStore *)store)->init_metadata_cache(dpp, cct, use_cache); + + if ((*(rgw::sal::MotrStore*)store).set_use_cache(use_cache) + .set_run_gc_thread(use_gc_thread) + .initialize(cct, dpp) < 0) { + delete store; + return nullptr; + } return store; } diff --git a/src/rgw/rgw_sal_motr.cc b/src/rgw/rgw_sal_motr.cc index 77007643ba3a2..c9dc62c6dea3e 100644 --- a/src/rgw/rgw_sal_motr.cc +++ b/src/rgw/rgw_sal_motr.cc @@ -1454,12 +1454,58 @@ int MotrBucket::abort_multiparts(const DoutPrefixProvider *dpp, CephContext *cct return 0; } -void MotrStore::finalize(void) -{ +void MotrStore::finalize(void) { + // stop gc worker threads + stop_gc(); // close connection with motr m0_client_fini(this->instance, true); } +MotrStore& MotrStore::set_run_gc_thread(bool _use_gc_thread) { + use_gc_thread = _use_gc_thread; + return *this; +} + +MotrStore& MotrStore::set_use_cache(bool _use_cache) { + use_cache = _use_cache; + return *this; +} + +int MotrStore::initialize(CephContext *cct, const DoutPrefixProvider *dpp) { + int rc = 0; + if (use_cache) { + int rc = init_metadata_cache(dpp, cct); + if (rc != 0) { + ldpp_dout(dpp, 0) << __func__ << ": Metadata cache init failed " << + "with rc = " << rc << dendl; + return rc; + } + } + + if (use_gc_thread) { + int rc = create_gc(dpp); + if (rc != 0) + ldpp_dout(dpp, 0) << __func__ << ": Metadata cache init failed " << + "with rc = " << rc << dendl; + } + return rc; +} + +int MotrStore::create_gc(const DoutPrefixProvider *dpp) { + int ret = 0; + // [TODO] Create multiple GC threads as per config + gc_worker = std::make_unique(dpp, this); + gc_worker->create("motr_gc"); + return ret; +} + +void MotrStore::stop_gc() { + if (gc_worker) { + gc_worker->signal_stop(); + gc_worker->join(); + } +} + uint64_t MotrStore::get_new_req_id() { uint64_t req_id = ceph::util::generate_random_number(); @@ -5362,7 +5408,7 @@ std::string MotrStore::get_cluster_id(const DoutPrefixProvider* dpp, optional_y } int MotrStore::init_metadata_cache(const DoutPrefixProvider *dpp, - CephContext *cct, bool use_cache) + CephContext *cct) { this->obj_meta_cache = new MotrMetaCache(dpp, cct); this->get_obj_meta_cache()->set_enabled(use_cache); diff --git a/src/rgw/rgw_sal_motr.h b/src/rgw/rgw_sal_motr.h index f79cc983b9a67..42f82d2e7d028 100644 --- a/src/rgw/rgw_sal_motr.h +++ b/src/rgw/rgw_sal_motr.h @@ -32,8 +32,11 @@ extern "C" { #include "rgw_role.h" #include "rgw_multi.h" #include "rgw_putobj_processor.h" +#include "motr/gc/gc.h" typedef void (*progress_cb)(off_t, void*); +class MotrGC; + namespace rgw::sal { class MotrStore; @@ -982,6 +985,12 @@ class MotrStore : public Store { MotrMetaCache* user_cache; MotrMetaCache* bucket_inst_cache; + // [TODO] Create vector of gc_workers + // size is input from `rgw_gc_max_concurrent_io` + std::unique_ptr gc_worker; + bool use_gc_thread; + bool use_cache; + public: CephContext *cctx; struct m0_client *instance; @@ -1097,7 +1106,12 @@ class MotrStore : public Store { uint64_t olh_epoch, const std::string& unique_tag) override; + virtual int initialize(CephContext *cct, const DoutPrefixProvider *dpp); virtual void finalize(void) override; + int create_gc(const DoutPrefixProvider *dpp); + void stop_gc(); + MotrStore& set_run_gc_thread(bool _use_gc_thread); + MotrStore& set_use_cache(bool _use_cache); virtual CephContext *ctx(void) override { return cctx; @@ -1133,7 +1147,7 @@ class MotrStore : public Store { int delete_access_key(const DoutPrefixProvider *dpp, optional_yield y, std::string access_key); int store_email_info(const DoutPrefixProvider *dpp, optional_yield y, MotrEmailInfo& email_info); - int init_metadata_cache(const DoutPrefixProvider *dpp, CephContext *cct, bool use_cache); + int init_metadata_cache(const DoutPrefixProvider *dpp, CephContext *cct); MotrMetaCache* get_obj_meta_cache() {return obj_meta_cache;} MotrMetaCache* get_user_cache() {return user_cache;} MotrMetaCache* get_bucket_inst_cache() {return bucket_inst_cache;}