From 0a5050803e2b2a699b3be9dd23972db04b466dfc Mon Sep 17 00:00:00 2001 From: "Sumedh A. Kulkarni" Date: Wed, 20 Jul 2022 03:03:34 -0600 Subject: [PATCH 1/5] rgw_sal_motr, motr_gc: [CORTX-33148] Add MotrGC class For Garbage Collection, cortx-rgw needs a MotrGC thread which works concurrently to delete all the stale objects. Added the MotrGC class and the interface to configure and start GC workers. Signed-off-by: Sumedh A. Kulkarni --- src/rgw/CMakeLists.txt | 2 +- src/rgw/motr/gc/gc.cc | 31 ++++++++++++++++++++++++ src/rgw/motr/gc/gc.h | 47 +++++++++++++++++++++++++++++++++++++ src/rgw/rgw_sal.cc | 8 ++++++- src/rgw/rgw_sal_motr.cc | 52 ++++++++++++++++++++++++++++++++++++++--- src/rgw/rgw_sal_motr.h | 16 ++++++++++++- 6 files changed, 150 insertions(+), 6 deletions(-) create mode 100644 src/rgw/motr/gc/gc.cc create mode 100644 src/rgw/motr/gc/gc.h diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index cabcba988f631..efc5e1ed62b8d 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -177,7 +177,7 @@ if(WITH_RADOSGW_DBSTORE) list(APPEND librgw_common_srcs rgw_sal_dbstore.cc) endif() if(WITH_RADOSGW_MOTR) - list(APPEND librgw_common_srcs rgw_sal_motr.cc) + list(APPEND librgw_common_srcs rgw_sal_motr.cc motr/gc/gc.cc) endif() if(WITH_JAEGER) list(APPEND librgw_common_srcs rgw_tracer.cc) diff --git a/src/rgw/motr/gc/gc.cc b/src/rgw/motr/gc/gc.cc new file mode 100644 index 0000000000000..997aff0c14494 --- /dev/null +++ b/src/rgw/motr/gc/gc.cc @@ -0,0 +1,31 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=2 sw=2 expandtab ft=cpp + +/* + * Garbage Collector implementation for the CORTX Motr backend + * + * Copyright (C) 2022 Seagate Technology LLC and/or its Affiliates + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "gc.h" + +void *MotrGC::entry() { + std::unique_lock lk(mtx); + ldpp_dout(dpp, 10) << __func__ << ": Motr GC started" << dendl; + + do { + ldpp_dout(dpp, 10) << __func__ << ": In a Motr GC loop." << dendl; + cv.wait_for(lk, std::chrono::milliseconds(gc_interval * 10)); + } while (! stop_signalled); + + ldpp_dout(dpp, 0) << __func__ << ": Stop signalled called.#" + << stop_signalled << dendl; + return nullptr; +} + diff --git a/src/rgw/motr/gc/gc.h b/src/rgw/motr/gc/gc.h new file mode 100644 index 0000000000000..ae02863f31e09 --- /dev/null +++ b/src/rgw/motr/gc/gc.h @@ -0,0 +1,47 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=2 sw=2 expandtab ft=cpp + +/* + * Garbage Collector Classes for the CORTX Motr backend + * + * Copyright (C) 2022 Seagate Technology LLC and/or its Affiliates + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef __MOTR_GC_H__ +#define __MOTR_GC_H__ + +#include "rgw_sal_motr.h" +#include "common/Thread.h" +#include +#include + +class MotrGC : public Thread { + private: + const DoutPrefixProvider *dpp; + rgw::sal::Store *store; + std::mutex mtx; + std::condition_variable cv; + bool stop_signalled = false; + uint32_t gc_interval = 60*60; // default: 24*60*60 sec + uint32_t gc_obj_min_wait = 60*60; // 60*60sec default + + public: + MotrGC(const DoutPrefixProvider *_dpp, rgw::sal::Store* _store) : + dpp(_dpp), store(_store) {} + + void *entry() override; + + void signal_stop() { + std::lock_guard lk_guard(mtx); + stop_signalled = true; + cv.notify_all(); + } +}; + +#endif diff --git a/src/rgw/rgw_sal.cc b/src/rgw/rgw_sal.cc index 88dae299a8cc8..3eb1e2e05eeb4 100644 --- a/src/rgw/rgw_sal.cc +++ b/src/rgw/rgw_sal.cc @@ -117,7 +117,13 @@ rgw::sal::Store* StoreManager::init_storage_provider(const DoutPrefixProvider* d ldpp_dout(dpp, 0) << "newMotrStore() failed!" << dendl; return store; } - ((rgw::sal::MotrStore *)store)->init_metadata_cache(dpp, cct, use_cache); + + if ((*(rgw::sal::MotrStore*)store).set_use_cache(use_cache) + .set_run_gc_thread(use_gc_thread) + .initialize(cct, dpp) < 0) { + delete store; + return nullptr; + } return store; } diff --git a/src/rgw/rgw_sal_motr.cc b/src/rgw/rgw_sal_motr.cc index 77007643ba3a2..c9dc62c6dea3e 100644 --- a/src/rgw/rgw_sal_motr.cc +++ b/src/rgw/rgw_sal_motr.cc @@ -1454,12 +1454,58 @@ int MotrBucket::abort_multiparts(const DoutPrefixProvider *dpp, CephContext *cct return 0; } -void MotrStore::finalize(void) -{ +void MotrStore::finalize(void) { + // stop gc worker threads + stop_gc(); // close connection with motr m0_client_fini(this->instance, true); } +MotrStore& MotrStore::set_run_gc_thread(bool _use_gc_thread) { + use_gc_thread = _use_gc_thread; + return *this; +} + +MotrStore& MotrStore::set_use_cache(bool _use_cache) { + use_cache = _use_cache; + return *this; +} + +int MotrStore::initialize(CephContext *cct, const DoutPrefixProvider *dpp) { + int rc = 0; + if (use_cache) { + int rc = init_metadata_cache(dpp, cct); + if (rc != 0) { + ldpp_dout(dpp, 0) << __func__ << ": Metadata cache init failed " << + "with rc = " << rc << dendl; + return rc; + } + } + + if (use_gc_thread) { + int rc = create_gc(dpp); + if (rc != 0) + ldpp_dout(dpp, 0) << __func__ << ": Metadata cache init failed " << + "with rc = " << rc << dendl; + } + return rc; +} + +int MotrStore::create_gc(const DoutPrefixProvider *dpp) { + int ret = 0; + // [TODO] Create multiple GC threads as per config + gc_worker = std::make_unique(dpp, this); + gc_worker->create("motr_gc"); + return ret; +} + +void MotrStore::stop_gc() { + if (gc_worker) { + gc_worker->signal_stop(); + gc_worker->join(); + } +} + uint64_t MotrStore::get_new_req_id() { uint64_t req_id = ceph::util::generate_random_number(); @@ -5362,7 +5408,7 @@ std::string MotrStore::get_cluster_id(const DoutPrefixProvider* dpp, optional_y } int MotrStore::init_metadata_cache(const DoutPrefixProvider *dpp, - CephContext *cct, bool use_cache) + CephContext *cct) { this->obj_meta_cache = new MotrMetaCache(dpp, cct); this->get_obj_meta_cache()->set_enabled(use_cache); diff --git a/src/rgw/rgw_sal_motr.h b/src/rgw/rgw_sal_motr.h index f79cc983b9a67..42f82d2e7d028 100644 --- a/src/rgw/rgw_sal_motr.h +++ b/src/rgw/rgw_sal_motr.h @@ -32,8 +32,11 @@ extern "C" { #include "rgw_role.h" #include "rgw_multi.h" #include "rgw_putobj_processor.h" +#include "motr/gc/gc.h" typedef void (*progress_cb)(off_t, void*); +class MotrGC; + namespace rgw::sal { class MotrStore; @@ -982,6 +985,12 @@ class MotrStore : public Store { MotrMetaCache* user_cache; MotrMetaCache* bucket_inst_cache; + // [TODO] Create vector of gc_workers + // size is input from `rgw_gc_max_concurrent_io` + std::unique_ptr gc_worker; + bool use_gc_thread; + bool use_cache; + public: CephContext *cctx; struct m0_client *instance; @@ -1097,7 +1106,12 @@ class MotrStore : public Store { uint64_t olh_epoch, const std::string& unique_tag) override; + virtual int initialize(CephContext *cct, const DoutPrefixProvider *dpp); virtual void finalize(void) override; + int create_gc(const DoutPrefixProvider *dpp); + void stop_gc(); + MotrStore& set_run_gc_thread(bool _use_gc_thread); + MotrStore& set_use_cache(bool _use_cache); virtual CephContext *ctx(void) override { return cctx; @@ -1133,7 +1147,7 @@ class MotrStore : public Store { int delete_access_key(const DoutPrefixProvider *dpp, optional_yield y, std::string access_key); int store_email_info(const DoutPrefixProvider *dpp, optional_yield y, MotrEmailInfo& email_info); - int init_metadata_cache(const DoutPrefixProvider *dpp, CephContext *cct, bool use_cache); + int init_metadata_cache(const DoutPrefixProvider *dpp, CephContext *cct); MotrMetaCache* get_obj_meta_cache() {return obj_meta_cache;} MotrMetaCache* get_user_cache() {return user_cache;} MotrMetaCache* get_bucket_inst_cache() {return bucket_inst_cache;} From 543a44d38b6956c4986ced1494e5e6321db83192 Mon Sep 17 00:00:00 2001 From: "Sumedh A. Kulkarni" Date: Thu, 21 Jul 2022 02:51:24 -0600 Subject: [PATCH 2/5] rgw_sal_motr, motr_gc: [CORTX-33148] Add MotrGC::GCWorker class For supporting concurrent IOs MotrGC need to have multiple GCWorker threads. Introducing GCWorker class and changing the MotrGC interfaces to manage all the worker threads. Signed-off-by: Sumedh A. Kulkarni --- src/rgw/motr/gc/gc.cc | 45 +++++++++++++++++++++++++++++-------- src/rgw/motr/gc/gc.h | 49 ++++++++++++++++++++++++++++------------- src/rgw/rgw_sal_motr.cc | 13 +++++------ src/rgw/rgw_sal_motr.h | 4 +--- 4 files changed, 77 insertions(+), 34 deletions(-) diff --git a/src/rgw/motr/gc/gc.cc b/src/rgw/motr/gc/gc.cc index 997aff0c14494..95d570b1bf732 100644 --- a/src/rgw/motr/gc/gc.cc +++ b/src/rgw/motr/gc/gc.cc @@ -15,17 +15,44 @@ #include "gc.h" -void *MotrGC::entry() { - std::unique_lock lk(mtx); - ldpp_dout(dpp, 10) << __func__ << ": Motr GC started" << dendl; +void *MotrGC::GCWorker::entry() { + // std::unique_lock lk(lock); + // ldpp_dout(dpp, 10) << __func__ << ": Motr GC started" << dendl; - do { - ldpp_dout(dpp, 10) << __func__ << ": In a Motr GC loop." << dendl; - cv.wait_for(lk, std::chrono::milliseconds(gc_interval * 10)); - } while (! stop_signalled); + // do { + // ldpp_dout(dpp, 10) << __func__ << ": In a Motr GC loop." << dendl; + // cv.wait_for(lk, std::chrono::milliseconds(gc_interval * 10)); + // } while (! stop_signalled); - ldpp_dout(dpp, 0) << __func__ << ": Stop signalled called.#" - << stop_signalled << dendl; + // ldpp_dout(dpp, 0) << __func__ << ": Stop signalled called.#" + // << stop_signalled << dendl; return nullptr; } +void MotrGC::initialize(CephContext *_cct, rgw::sal::Store* _store) { + cct = _cct; + store = _store; + // fetch num_max_queue from config + // create all gc queues in motr index store +} + +void MotrGC::start_processor() { + // fetch max_concurrent_io i.e. max_threads to create from config. + // start all the gc_worker threads +} + +void MotrGC::stop_processor() { + // in case of stop signal, + // gracefully shutdown all the gc threads. + down_flag = true; + for (auto& worker : workers) { + worker->stop(); + worker->join(); + } + workers.clear(); +} + +void MotrGC::GCWorker::stop() { + std::lock_guard l{lock}; + cv.notify_all(); +} diff --git a/src/rgw/motr/gc/gc.h b/src/rgw/motr/gc/gc.h index ae02863f31e09..86935c89d889b 100644 --- a/src/rgw/motr/gc/gc.h +++ b/src/rgw/motr/gc/gc.h @@ -20,28 +20,47 @@ #include "common/Thread.h" #include #include +#include -class MotrGC : public Thread { - private: - const DoutPrefixProvider *dpp; +class MotrGC : public DoutPrefixProvider { + private: + CephContext *cct; rgw::sal::Store *store; - std::mutex mtx; - std::condition_variable cv; - bool stop_signalled = false; - uint32_t gc_interval = 60*60; // default: 24*60*60 sec - uint32_t gc_obj_min_wait = 60*60; // 60*60sec default + int max_indices = 0; + std::vector index_names; + std::atomic down_flag = false; public: - MotrGC(const DoutPrefixProvider *_dpp, rgw::sal::Store* _store) : - dpp(_dpp), store(_store) {} + class GCWorker : public Thread { + private: + const DoutPrefixProvider *dpp; + CephContext *cct; + MotrGC *motr_gc; + int worker_id; + uint32_t gc_interval = 60*60; // default: 24*60*60 sec + std::mutex lock; + std::condition_variable cv; + public: + GCWorker(const DoutPrefixProvider* _dpp, CephContext *_cct, + MotrGC *_motr_gc, int _worker_id) + : dpp(_dpp), cct(_cct), motr_gc(_motr_gc), worker_id(_worker_id) {}; - void *entry() override; + void *entry() override; + void stop(); + }; + std::vector> workers; - void signal_stop() { - std::lock_guard lk_guard(mtx); - stop_signalled = true; - cv.notify_all(); + MotrGC() : cct(nullptr), store(nullptr) {} + ~MotrGC() { + stop_processor(); + finalize(); } + + void initialize(CephContext *_cct, rgw::sal::Store* _store); + void finalize(); + + void start_processor(); + void stop_processor(); }; #endif diff --git a/src/rgw/rgw_sal_motr.cc b/src/rgw/rgw_sal_motr.cc index c9dc62c6dea3e..f2c0bc282be5b 100644 --- a/src/rgw/rgw_sal_motr.cc +++ b/src/rgw/rgw_sal_motr.cc @@ -1485,7 +1485,7 @@ int MotrStore::initialize(CephContext *cct, const DoutPrefixProvider *dpp) { if (use_gc_thread) { int rc = create_gc(dpp); if (rc != 0) - ldpp_dout(dpp, 0) << __func__ << ": Metadata cache init failed " << + ldpp_dout(dpp, 0) << __func__ << ": Failed to Create MotrGC " << "with rc = " << rc << dendl; } return rc; @@ -1493,16 +1493,15 @@ int MotrStore::initialize(CephContext *cct, const DoutPrefixProvider *dpp) { int MotrStore::create_gc(const DoutPrefixProvider *dpp) { int ret = 0; - // [TODO] Create multiple GC threads as per config - gc_worker = std::make_unique(dpp, this); - gc_worker->create("motr_gc"); + motr_gc->initialize(cctx, this); + motr_gc->start_processor(); return ret; } void MotrStore::stop_gc() { - if (gc_worker) { - gc_worker->signal_stop(); - gc_worker->join(); + if (motr_gc) { + motr_gc->stop_processor(); + motr_gc->finalize(); } } diff --git a/src/rgw/rgw_sal_motr.h b/src/rgw/rgw_sal_motr.h index 42f82d2e7d028..e0e2f3f801a3a 100644 --- a/src/rgw/rgw_sal_motr.h +++ b/src/rgw/rgw_sal_motr.h @@ -985,9 +985,7 @@ class MotrStore : public Store { MotrMetaCache* user_cache; MotrMetaCache* bucket_inst_cache; - // [TODO] Create vector of gc_workers - // size is input from `rgw_gc_max_concurrent_io` - std::unique_ptr gc_worker; + std::unique_ptr motr_gc; bool use_gc_thread; bool use_cache; From cbc95aedb753185eb337fff517106ebc6294e6c1 Mon Sep 17 00:00:00 2001 From: "Sumedh A. Kulkarni" Date: Fri, 22 Jul 2022 00:55:47 -0600 Subject: [PATCH 3/5] rgw_sal_motr, motr_gc: [CORTX-33148] implement MotrGC::start_processor() Implemented the logic to spawn multiple GCWorker threads on startup, and gracefully exit on shutdown. Signed-off-by: Sumedh A. Kulkarni --- src/rgw/motr/gc/gc.cc | 52 +++++++++++++++++++++++++++++++---------- src/rgw/motr/gc/gc.h | 20 +++++++++++++--- src/rgw/rgw_sal_motr.cc | 7 +++--- src/rgw/rgw_sal_motr.h | 2 +- 4 files changed, 62 insertions(+), 19 deletions(-) diff --git a/src/rgw/motr/gc/gc.cc b/src/rgw/motr/gc/gc.cc index 95d570b1bf732..1ba063f2cd811 100644 --- a/src/rgw/motr/gc/gc.cc +++ b/src/rgw/motr/gc/gc.cc @@ -16,33 +16,49 @@ #include "gc.h" void *MotrGC::GCWorker::entry() { - // std::unique_lock lk(lock); - // ldpp_dout(dpp, 10) << __func__ << ": Motr GC started" << dendl; + std::unique_lock lk(lock); + ldpp_dout(dpp, 10) << __func__ << ": " << gc_thread_prefix + << worker_id << " started." << dendl; - // do { - // ldpp_dout(dpp, 10) << __func__ << ": In a Motr GC loop." << dendl; - // cv.wait_for(lk, std::chrono::milliseconds(gc_interval * 10)); - // } while (! stop_signalled); + do { - // ldpp_dout(dpp, 0) << __func__ << ": Stop signalled called.#" - // << stop_signalled << dendl; + ldpp_dout(dpp, 10) << __func__ << ": " << gc_thread_prefix + << worker_id << " iteration" << dendl; + cv.wait_for(lk, std::chrono::milliseconds(gc_interval * 10)); + + } while (! motr_gc->going_down()); + + ldpp_dout(dpp, 0) << __func__ << ": Stop signalled called for " + << gc_thread_prefix << worker_id << dendl; return nullptr; } -void MotrGC::initialize(CephContext *_cct, rgw::sal::Store* _store) { - cct = _cct; - store = _store; +void MotrGC::initialize() { + ldpp_dout(this, 10) << __func__ << ": In initialize method." << dendl; // fetch num_max_queue from config // create all gc queues in motr index store } +void MotrGC::finalize() { + // undo steps from initialize stage +} + void MotrGC::start_processor() { // fetch max_concurrent_io i.e. max_threads to create from config. // start all the gc_worker threads + auto max_workers = cct->_conf->rgw_gc_max_concurrent_io; + ldpp_dout(this, 10) << __func__ << ": max_workers = " + << max_workers << dendl; + workers.reserve(max_workers); + for (int ix = 0; ix < max_workers; ++ix) { + auto worker = std::make_unique(this /* dpp */, + cct, this, ix); + worker->create((gc_thread_prefix + std::to_string(ix)).c_str()); + workers.push_back(std::move(worker)); + } } void MotrGC::stop_processor() { - // in case of stop signal, // gracefully shutdown all the gc threads. down_flag = true; for (auto& worker : workers) { @@ -56,3 +72,15 @@ void MotrGC::GCWorker::stop() { std::lock_guard l{lock}; cv.notify_all(); } + +bool MotrGC::going_down() { + return down_flag; +} + +unsigned MotrGC::get_subsys() const { + return dout_subsys; +} + +std::ostream& MotrGC::gen_prefix(std::ostream& out) const { + return out << "garbage_collector: "; +} diff --git a/src/rgw/motr/gc/gc.h b/src/rgw/motr/gc/gc.h index 86935c89d889b..e0e9a3486894f 100644 --- a/src/rgw/motr/gc/gc.h +++ b/src/rgw/motr/gc/gc.h @@ -22,6 +22,8 @@ #include #include +static std::string gc_thread_prefix = "gc_thread_"; + class MotrGC : public DoutPrefixProvider { private: CephContext *cct; @@ -43,24 +45,36 @@ class MotrGC : public DoutPrefixProvider { public: GCWorker(const DoutPrefixProvider* _dpp, CephContext *_cct, MotrGC *_motr_gc, int _worker_id) - : dpp(_dpp), cct(_cct), motr_gc(_motr_gc), worker_id(_worker_id) {}; + : dpp(_dpp), + cct(_cct), + motr_gc(_motr_gc), + worker_id(_worker_id) {}; void *entry() override; void stop(); }; std::vector> workers; - MotrGC() : cct(nullptr), store(nullptr) {} + MotrGC(CephContext *_cct, rgw::sal::Store* _store) + : cct(_cct), store(_store) {} + ~MotrGC() { stop_processor(); finalize(); } - void initialize(CephContext *_cct, rgw::sal::Store* _store); + void initialize(); void finalize(); void start_processor(); void stop_processor(); + + bool going_down(); + + // Set Up logging prefix for GC + CephContext *get_cct() const override { return cct; } + unsigned get_subsys() const; + std::ostream& gen_prefix(std::ostream& out) const; }; #endif diff --git a/src/rgw/rgw_sal_motr.cc b/src/rgw/rgw_sal_motr.cc index f2c0bc282be5b..45ea7fb2dec58 100644 --- a/src/rgw/rgw_sal_motr.cc +++ b/src/rgw/rgw_sal_motr.cc @@ -1483,7 +1483,7 @@ int MotrStore::initialize(CephContext *cct, const DoutPrefixProvider *dpp) { } if (use_gc_thread) { - int rc = create_gc(dpp); + int rc = create_gc(); if (rc != 0) ldpp_dout(dpp, 0) << __func__ << ": Failed to Create MotrGC " << "with rc = " << rc << dendl; @@ -1491,9 +1491,10 @@ int MotrStore::initialize(CephContext *cct, const DoutPrefixProvider *dpp) { return rc; } -int MotrStore::create_gc(const DoutPrefixProvider *dpp) { +int MotrStore::create_gc() { int ret = 0; - motr_gc->initialize(cctx, this); + motr_gc = std::make_unique(cctx, this); + motr_gc->initialize(); motr_gc->start_processor(); return ret; } diff --git a/src/rgw/rgw_sal_motr.h b/src/rgw/rgw_sal_motr.h index e0e2f3f801a3a..3fa0c8cc6d456 100644 --- a/src/rgw/rgw_sal_motr.h +++ b/src/rgw/rgw_sal_motr.h @@ -1106,7 +1106,7 @@ class MotrStore : public Store { virtual int initialize(CephContext *cct, const DoutPrefixProvider *dpp); virtual void finalize(void) override; - int create_gc(const DoutPrefixProvider *dpp); + int create_gc(); void stop_gc(); MotrStore& set_run_gc_thread(bool _use_gc_thread); MotrStore& set_use_cache(bool _use_cache); From 76ae806efaf9f77cc36ece879fd754ccc10afbc4 Mon Sep 17 00:00:00 2001 From: "Sumedh A. Kulkarni" Date: Mon, 25 Jul 2022 00:24:58 -0600 Subject: [PATCH 4/5] rgw_sal_motr: [CORTX-33148] fix init_metadata_cache bug We are bound to create MotrMetaCache objects as we are directly using class methods during CRUD operations without checking if metadata caching is enabled or not. Earlier I was trying to not create meta cache objects which was breaking the CRUD, so now reverting it to create those objects again. NOTE: Ideally we should create MetaCache objects only if caching is enabled. Signed-off-by: Sumedh A. Kulkarni --- src/rgw/motr/gc/gc.cc | 2 +- src/rgw/rgw_sal_motr.cc | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/rgw/motr/gc/gc.cc b/src/rgw/motr/gc/gc.cc index 1ba063f2cd811..c5e92e91f5ca1 100644 --- a/src/rgw/motr/gc/gc.cc +++ b/src/rgw/motr/gc/gc.cc @@ -13,7 +13,7 @@ * */ -#include "gc.h" +#include "motr/gc/gc.h" void *MotrGC::GCWorker::entry() { std::unique_lock lk(lock); diff --git a/src/rgw/rgw_sal_motr.cc b/src/rgw/rgw_sal_motr.cc index 45ea7fb2dec58..537bfce064d1e 100644 --- a/src/rgw/rgw_sal_motr.cc +++ b/src/rgw/rgw_sal_motr.cc @@ -1472,17 +1472,16 @@ MotrStore& MotrStore::set_use_cache(bool _use_cache) { } int MotrStore::initialize(CephContext *cct, const DoutPrefixProvider *dpp) { - int rc = 0; - if (use_cache) { - int rc = init_metadata_cache(dpp, cct); - if (rc != 0) { - ldpp_dout(dpp, 0) << __func__ << ": Metadata cache init failed " << - "with rc = " << rc << dendl; - return rc; - } + // Create metadata objects and set enabled=use_cache value + int rc = init_metadata_cache(dpp, cct); + if (rc != 0) { + ldpp_dout(dpp, 0) << __func__ << ": Metadata cache init failed " << + "with rc = " << rc << dendl; + return rc; } if (use_gc_thread) { + // Create MotrGC object and start GCWorker threads int rc = create_gc(); if (rc != 0) ldpp_dout(dpp, 0) << __func__ << ": Failed to Create MotrGC " << From bc6b0e4c23969b2423bbde55742cadd66bea9845 Mon Sep 17 00:00:00 2001 From: "Sumedh A. Kulkarni" Date: Mon, 25 Jul 2022 04:30:21 -0600 Subject: [PATCH 5/5] motr_gc: [CORTX-33148] partially implement MotrGC::initialize() add code to fetch the max gc indices to create. create and append the index name in gc index list. Signed-off-by: Sumedh A. Kulkarni --- src/rgw/motr/gc/gc.cc | 24 +++++++++++++++++------- src/rgw/motr/gc/gc.h | 2 ++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/rgw/motr/gc/gc.cc b/src/rgw/motr/gc/gc.cc index c5e92e91f5ca1..4fb3f9e8ae62f 100644 --- a/src/rgw/motr/gc/gc.cc +++ b/src/rgw/motr/gc/gc.cc @@ -19,7 +19,7 @@ void *MotrGC::GCWorker::entry() { std::unique_lock lk(lock); ldpp_dout(dpp, 10) << __func__ << ": " << gc_thread_prefix << worker_id << " started." << dendl; - + do { ldpp_dout(dpp, 10) << __func__ << ": " << gc_thread_prefix @@ -34,24 +34,34 @@ void *MotrGC::GCWorker::entry() { } void MotrGC::initialize() { - ldpp_dout(this, 10) << __func__ << ": In initialize method." << dendl; - // fetch num_max_queue from config - // create all gc queues in motr index store + // fetch max gc indices from config + auto max_indices = std::min(cct->_conf->rgw_gc_max_objs, + GC_MAX_SHARDS_PRIME); + ldpp_dout(this, 50) << __func__ << ": max_indices = " << max_indices << dendl; + + index_names.reserve(max_indices); + for (int i = 0; i < max_indices; i++) { + // Append index name to the gc index list + index_names.push_back(gc_index_prefix + std::to_string(i)); + + // [To be Implemented] create index in motr dix + } + } void MotrGC::finalize() { - // undo steps from initialize stage + // [To be Implemented] undo steps from initialize stage } void MotrGC::start_processor() { // fetch max_concurrent_io i.e. max_threads to create from config. // start all the gc_worker threads auto max_workers = cct->_conf->rgw_gc_max_concurrent_io; - ldpp_dout(this, 10) << __func__ << ": max_workers = " + ldpp_dout(this, 50) << __func__ << ": max_workers = " << max_workers << dendl; workers.reserve(max_workers); for (int ix = 0; ix < max_workers; ++ix) { - auto worker = std::make_unique(this /* dpp */, + auto worker = std::make_unique(this /* dpp */, cct, this, ix); worker->create((gc_thread_prefix + std::to_string(ix)).c_str()); workers.push_back(std::move(worker)); diff --git a/src/rgw/motr/gc/gc.h b/src/rgw/motr/gc/gc.h index e0e9a3486894f..eecfe0a223c29 100644 --- a/src/rgw/motr/gc/gc.h +++ b/src/rgw/motr/gc/gc.h @@ -22,6 +22,8 @@ #include #include +const int64_t GC_MAX_SHARDS_PRIME = 65521; +static std::string gc_index_prefix = "gc."; static std::string gc_thread_prefix = "gc_thread_"; class MotrGC : public DoutPrefixProvider {