diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index efc5e1ed62b8d..dacb1783f9f2e 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -177,7 +177,7 @@ if(WITH_RADOSGW_DBSTORE) list(APPEND librgw_common_srcs rgw_sal_dbstore.cc) endif() if(WITH_RADOSGW_MOTR) - list(APPEND librgw_common_srcs rgw_sal_motr.cc motr/gc/gc.cc) + list(APPEND librgw_common_srcs rgw_sal_motr.cc motr/gc/gc.cc motr/sync/motr_sync_impl.cc) endif() if(WITH_JAEGER) list(APPEND librgw_common_srcs rgw_tracer.cc) diff --git a/src/rgw/motr/gc/gc.cc b/src/rgw/motr/gc/gc.cc index e564be359d026..e8cfb71ca9560 100644 --- a/src/rgw/motr/gc/gc.cc +++ b/src/rgw/motr/gc/gc.cc @@ -91,7 +91,7 @@ void MotrGC::initialize() { } index_names.reserve(max_indices); ldpp_dout(this, 50) << __func__ << ": max_indices = " << max_indices << dendl; - for (int ind_suf = 0; ind_suf < max_indices; ind_suf++) { + for (uint32_t ind_suf = 0; ind_suf < max_indices; ind_suf++) { std::string iname = gc_index_prefix + "." + std::to_string(ind_suf); int rc = static_cast(store)->create_motr_idx_by_name(iname); if (rc < 0 && rc != -EEXIST){ @@ -167,7 +167,7 @@ int MotrGC::get_locked_gc_index(uint32_t& rand_ind) int rc = -1; uint32_t new_index = 0; // attempt to lock GC starting with passed in index - for (int ind = 0; ind < max_indices; ind++) + for (uint32_t ind = 0; ind < max_indices; ind++) { new_index = (ind + rand_ind) % max_indices; // try locking index diff --git a/src/rgw/motr/sync/motr_sync.h b/src/rgw/motr/sync/motr_sync.h new file mode 100644 index 0000000000000..ed2fa5824ffde --- /dev/null +++ b/src/rgw/motr/sync/motr_sync.h @@ -0,0 +1,99 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=2 sw=2 expandtab ft=cpp + +/* + * Abstract thread/process synchronization for the CORTX Motr backend + * + * Copyright (C) 2022 Seagate Technology LLC and/or its Affiliates + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef __MOTR_SYNCHROZATION__ +#define __MOTR_SYNCHROZATION__ + +#include + +#include "include/types.h" +#include "include/utime.h" +#include "rgw_sal_motr.h" + +// Define MOTR_EXCLUSIVE_RW_LOCK only when system needs exclusive read/write lock +#define MOTR_EXCLUSIVE_RW_LOCK 1 + +class MotrLockProvider; + +struct motr_locker_info_t +{ + std::string cookie; // unique id of caller + utime_t expiration; // expiration: non-zero means epoch of locker expiration + std::string description; // description: locker description, may be empty + + motr_locker_info_t() {} + motr_locker_info_t(const utime_t& _e, std::string& _c, const std::string& _d) + : expiration(_e), cookie(_c), description(_d) {} + void encode(ceph::buffer::list &bl) const { + ENCODE_START(1, 1, bl); + encode(cookie, bl); + encode(expiration, bl); + encode(description, bl); + ENCODE_FINISH(bl); + } + void decode(ceph::buffer::list::const_iterator &bl) { + DECODE_START_LEGACY_COMPAT_LEN(1, 1, 1, bl); + decode(cookie, bl); + decode(expiration, bl); + decode(description, bl); + DECODE_FINISH(bl); + } +}; +WRITE_CLASS_ENCODER(motr_locker_info_t) + +struct motr_lock_info_t { +public: + std::map lockers; + void encode(ceph::buffer::list &bl) const { + ENCODE_START(1, 1, bl); + encode(lockers, bl); + ENCODE_FINISH(bl); + } + void decode(ceph::buffer::list::const_iterator &bl) { + DECODE_START_LEGACY_COMPAT_LEN(1, 1, 1, bl); + decode(lockers, bl); + DECODE_FINISH(bl); + } +}; +WRITE_CLASS_ENCODER(motr_lock_info_t) + +enum class MotrLockType { + NONE = 0, + EXCLUSIVE = 1, + SHARED = 2, + EXCLUSIVE_EPHEMERAL = 3, /* lock object is removed @ unlock */ +}; + +// Abstract interface for Motr named synchronization +class MotrSync { +public: + virtual void initialize(std::shared_ptr lock_provider) = 0; + virtual int lock(const std::string& lock_name, MotrLockType lock_type, + utime_t lock_duration, const std::string& locker_id) = 0; + virtual int unlock(const std::string& lock_name, MotrLockType lock_type, + const std::string& locker_id) = 0; +}; + +// Abstract interface for entity that implements backend for lock objects +class MotrLockProvider { +public: + virtual int read_lock(const std::string& lock_name, + motr_lock_info_t* lock_info) = 0; + virtual int write_lock(const std::string& lock_name, + motr_lock_info_t* lock_info, bool update) = 0; + virtual int remove_lock(const std::string& lock_name, + const std::string& locker_id) = 0; +}; +#endif __MOTR_SYNCHROZATION__ diff --git a/src/rgw/motr/sync/motr_sync_impl.cc b/src/rgw/motr/sync/motr_sync_impl.cc new file mode 100644 index 0000000000000..b0fd036043fd3 --- /dev/null +++ b/src/rgw/motr/sync/motr_sync_impl.cc @@ -0,0 +1,153 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=2 sw=2 expandtab ft=cpp + +/* + * GC thread/process synchronization for the CORTX Motr backend + * + * Copyright (C) 2022 Seagate Technology LLC and/or its Affiliates + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "motr/sync/motr_sync_impl.h" + +void MotrLock::initialize(std::shared_ptr lock_provider) { + _lock_provider = lock_provider; +} + +int MotrLock::lock(const std::string& lock_name, + MotrLockType lock_type, + utime_t lock_duration, + const std::string& locker_id = "") { + if (!_lock_provider || (MotrLockType::EXCLUSIVE != lock_type && + locker_id.empty())) { + return -EINVAL; + } + int rc = 0; + motr_lock_info_t lock_obj; + rc = _lock_provider->read_lock(lock_name, &lock_obj); + if (rc != 0 && rc != -ENOENT) + return rc; + if (rc == 0) { + // lock entry is present. Check if this is a stale/expired lock + utime_t now = ceph_clock_now(); + auto iter = lock_obj.lockers.begin(); + while (iter != lock_obj.lockers.end()) { + struct motr_locker_info_t &info = iter->second; + if (!info.expiration.is_zero() && info.expiration < now) { + // locker has expired; delete it + iter = lock_obj.lockers.erase(iter); + } else { + ++iter; + } + } + // remove the lock if no locker is left + if (lock_obj.lockers.empty()) + _lock_provider->remove_lock(lock_name, locker_id); + } + + if (!lock_obj.lockers.empty() && MotrLockType::EXCLUSIVE == lock_type) { + // lock is not available + return -EBUSY; + } else { + // Try to acquire lock object + struct motr_locker_info_t locker; + locker.cookie = locker_id; + locker.expiration = ceph_clock_now() + lock_duration; + locker.description = ""; + // Update lock entry with current locker and lock the resource + lock_obj.lockers.insert( + std::pair(locker_id, locker)); + rc = _lock_provider->write_lock(lock_name, &lock_obj, false); + if (rc < 0) { + // Failed to acquire lock object; possibly, already acquired + return -EBUSY; + } + } + return rc; +} + +int MotrLock::unlock(const std::string& lock_name, + MotrLockType lock_type, const std::string& locker_id) { + return _lock_provider->remove_lock(lock_name, locker_id); +} + +int MotrKVLockProvider::initialize(const DoutPrefixProvider* dpp, + rgw::sal::MotrStore* _s, + std::string& lock_index_name) { + _dpp = dpp; + _store = _s; + _lock_index = lock_index_name; + if (!_store || lock_index_name.empty()) { + return -EINVAL; + } + return _store->create_motr_idx_by_name(lock_index_name); +} + +int MotrKVLockProvider::read_lock(const std::string& lock_name, + motr_lock_info_t* lock_info) { + if (!_store || _lock_index.empty() || !lock_info) { + return -EINVAL; + } + int rc = 0; + bufferlist bl; + rc = _store->do_idx_op_by_name(_lock_index, M0_IC_GET, lock_name, bl); + if (rc != 0) { + return rc; + } + bufferlist::const_iterator bitr = bl.begin(); + lock_info->decode(bitr); + return 0; +} + +int MotrKVLockProvider::write_lock(const std::string& lock_name, + motr_lock_info_t* lock_info, bool update) { + if (!_store || _lock_index.empty() || !lock_info) { + return -EINVAL; + } + bufferlist bl; + lock_info->encode(bl); + int rc = + _store->do_idx_op_by_name(_lock_index, M0_IC_PUT, lock_name, bl, update); + return rc; +} + +int MotrKVLockProvider::remove_lock(const std::string& lock_name, + const std::string& locker_id = "") { + if (!_store || _lock_index.empty() || lock_name.empty()) { + return -EINVAL; + } + motr_lock_info_t lock_info; + bufferlist bl; + int rc = 0; + bool del_lock_entry = false; + rc = read_lock(lock_name, &lock_info); + if (rc != 0) + return (rc == -ENOENT) ? 0 : rc; + + if (locker_id.empty()) { + // this is exclusive lock + del_lock_entry = true; + } else { + auto it = lock_info.lockers.find(locker_id); + if (it != lock_info.lockers.end()) { + lock_info.lockers.erase(it); + } + if (lock_info.lockers.empty()) { + del_lock_entry = true; + } + } + if (del_lock_entry) { + // all lockers removed; now delete lock object + rc = _store->do_idx_op_by_name(_lock_index, M0_IC_DEL, lock_name, bl); + } else { + // update the lock entry + lock_info.encode(bl); + rc = _store->do_idx_op_by_name(_lock_index, M0_IC_PUT, lock_name, bl); + } + return rc; +} diff --git a/src/rgw/motr/sync/motr_sync_impl.h b/src/rgw/motr/sync/motr_sync_impl.h new file mode 100644 index 0000000000000..2bd774945ca55 --- /dev/null +++ b/src/rgw/motr/sync/motr_sync_impl.h @@ -0,0 +1,118 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=2 sw=2 expandtab ft=cpp + +/* + * GC thread/process synchronization for the CORTX Motr backend + * + * Copyright (C) 2022 Seagate Technology LLC and/or its Affiliates + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef __MOTR_LOCK__ +#define __MOTR_LOCK__ + +/* +Usage: + Approach 1. + 1. Instantiate one of the LockProvider Implementation. + example: + std::unique_ptr lock_provider; + + 2. Initialize the lock provider with global lock index table + example: + lock_provider.initialize(pp, store, "motr.gc.index.locks"); + + The caller can do this during the process/thread + start-up/initialization; if "initialize()" returns < 0, the + caller is expected to stop further initialization if locking is esential + in the system. In multi-process environment, each process can call + "MotrKVLockProvider::initialize" during it's start-up, providing the same + name for the global lock index. + + 3. Instantiate one of the Locker Implmentations. + example: + std::unique_ptr motr_lock; + + 4. Initialize the Locker class with LockProvider Object. + example: + motr_lock.initialize(lock_provider); + + 5. Lock the required resource using Locker object + example: + motr_lock.lock(resource_name, lock_type, + lock_duration, locker_id); + + Any -ve value returned by MotrLock::lock() indicates resource + can't be locked; a value 0 indicates success. + + 6. Unlock the resource using the same Locker Object. + exmaple: + motr_lock.unlock(resource_name, lock_type, locker_id); + + Approach 2. + 1. Follow step (1) & step (2) in Approach 1. + 2. Call create_motr_Lock_instance(), passing it lock provider instance created + in step (1) above. + 3. Call get_lock_instance() wherever lock is needed. + +Note: Presently, locking framework supports EXCLUSIVE lock; it means +caller needs to set MotrLockType::EXCLUSIVE while calling MotrLock::lock(), +and do not pass locker_id. +*/ + +#include "rgw_sal_motr.h" +#include "motr/sync/motr_sync.h" + +class MotrLock : public MotrSync { +private: + std::shared_ptr _lock_provider; + +public: + virtual void initialize(std::shared_ptr lock_provider) override; + virtual int lock(const std::string& lock_name, MotrLockType lock_type, + utime_t lock_duration, const std::string& locker_id) override; + virtual int unlock(const std::string& lock_name, MotrLockType lock_type, + const std::string& locker_id) override; +}; + +class MotrKVLockProvider : public MotrLockProvider { +private: + const DoutPrefixProvider* _dpp; + rgw::sal::MotrStore* _store; + std::string _lock_index; +public: + int initialize(const DoutPrefixProvider* dpp, rgw::sal::MotrStore* _s, + std::string& lock_index_name); + virtual int read_lock(const std::string& lock_name, + motr_lock_info_t* lock_info) override; + virtual int write_lock(const std::string& lock_name, + motr_lock_info_t* lock_info, + bool update = false) override; + virtual int remove_lock(const std::string& lock_name, + const std::string& locker_id) override; +}; + +// Creates a global instance of MotrLock that can be used by caller +// This needs to be called by a single main thread of caller. +std::shared_ptr g_motr_lock; +std::shared_ptr& create_motr_Lock_instance( + std::shared_ptr lock_provider) { + static bool initialize = false; + if (!initialize) { + g_motr_lock = std::make_shared(); + g_motr_lock->initialize(lock_provider); + initialize = true; + return g_motr_lock; + } +} + +std::shared_ptr& get_lock_instance() { + return g_motr_lock; +} + +#endif