From d97d9681766766836f991ec65f64b66b654b966c Mon Sep 17 00:00:00 2001 From: Steve Atherton Date: Mon, 8 Nov 2021 13:04:53 -0800 Subject: [PATCH] Added KeyBackedObjectMap and KeyBackedObjectProperty classes for storing serializable objects in FDB (#5896) * Cleaned up some lambda capture workaround since x=y captures weren't available when these classes were originally written. * Added KeyBackedObjectMap and KeyBackObjectProperty, which work like KeyBackedMap and KeyBackedProperty but use ObjectWriter/Reader for Value serialization so that the type can evolve over time. * Disabled unit tests which shouldn't run as part of random selection. --- fdbclient/KeyBackedTypes.h | 192 +++++++++++++++++++++++++---- fdbserver/BlobManager.actor.cpp | 2 +- fdbserver/VersionedBTree.actor.cpp | 2 +- 3 files changed, 172 insertions(+), 24 deletions(-) diff --git a/fdbclient/KeyBackedTypes.h b/fdbclient/KeyBackedTypes.h index fc73a737494..881415dce18 100644 --- a/fdbclient/KeyBackedTypes.h +++ b/fdbclient/KeyBackedTypes.h @@ -26,7 +26,9 @@ #include "fdbclient/IClientApi.h" #include "fdbclient/ReadYourWrites.h" #include "fdbclient/Subspace.h" +#include "flow/ObjectSerializer.h" #include "flow/genericactors.actor.h" +#include "flow/serialize.h" // Codec is a utility struct to convert a type to and from a Tuple. It is used by the template // classes below like KeyBackedProperty and KeyBackedMap to convert key parts and values @@ -168,14 +170,8 @@ class KeyBackedProperty { Future getOrThrow(Reference tr, Snapshot snapshot = Snapshot::False, Error err = key_not_found()) const { - auto keyCopy = key; - auto backtrace = platform::get_backtrace(); return map(get(tr, snapshot), [=](Optional val) -> T { if (!val.present()) { - TraceEvent(SevInfo, "KeyBackedProperty_KeyNotFound") - .detail("Key", keyCopy) - .detail("Err", err.code()) - .detail("ParentTrace", backtrace.c_str()); throw err; } @@ -184,45 +180,39 @@ class KeyBackedProperty { } Future> get(Database cx, Snapshot snapshot = Snapshot::False) const { - auto& copy = *this; - return runRYWTransaction(cx, [=](Reference tr) { + return runRYWTransaction(cx, [=, self = *this](Reference tr) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); - return copy.get(tr, snapshot); + return self.get(tr, snapshot); }); } Future getD(Database cx, Snapshot snapshot = Snapshot::False, T defaultValue = T()) const { - auto& copy = *this; - return runRYWTransaction(cx, [=](Reference tr) { + return runRYWTransaction(cx, [=, self = *this](Reference tr) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); - return copy.getD(tr, snapshot, defaultValue); + return self.getD(tr, snapshot, defaultValue); }); } Future getOrThrow(Database cx, Snapshot snapshot = Snapshot::False, Error err = key_not_found()) const { - auto& copy = *this; - return runRYWTransaction(cx, [=](Reference tr) { + return runRYWTransaction(cx, [=, self = *this](Reference tr) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); - return copy.getOrThrow(tr, snapshot, err); + return self.getOrThrow(tr, snapshot, err); }); } void set(Reference tr, T const& val) { return tr->set(key, Codec::pack(val).pack()); } Future set(Database cx, T const& val) { - auto _key = key; - Value _val = Codec::pack(val).pack(); - return runRYWTransaction(cx, [_key, _val](Reference tr) { + return runRYWTransaction(cx, [=, self = *this](Reference tr) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); - tr->set(_key, _val); - + self->set(tr, val); return Future(Void()); }); } @@ -262,12 +252,12 @@ class KeyBackedBinaryValue { Key key; }; -// Convenient read/write access to a sorted map of KeyType to ValueType that has key as its prefix +// Convenient read/write access to a sorted map of KeyType to ValueType under prefix // Even though 'this' is not actually mutated, methods that change db keys are not const. template class KeyBackedMap { public: - KeyBackedMap(KeyRef key) : space(key) {} + KeyBackedMap(KeyRef prefix) : space(prefix) {} typedef _KeyType KeyType; typedef _ValueType ValueType; @@ -336,6 +326,164 @@ class KeyBackedMap { Subspace space; }; +// Convenient read/write access to a single value of type T stored at key +// Even though 'this' is not actually mutated, methods that change the db key are not const. +template +class KeyBackedObjectProperty { +public: + KeyBackedObjectProperty(KeyRef key, VersionOptions versionOptions) : key(key), versionOptions(versionOptions) {} + Future> get(Reference tr, Snapshot snapshot = Snapshot::False) const { + + return map(tr->get(key, snapshot), [vo = versionOptions](Optional const& val) -> Optional { + if (val.present()) + return ObjectReader::fromStringRef(val.get(), vo); + return {}; + }); + } + + // Get property's value or defaultValue if it doesn't exist + Future getD(Reference tr, + Snapshot snapshot = Snapshot::False, + T defaultValue = T()) const { + return map(get(tr, snapshot), [=](Optional val) -> T { return val.present() ? val.get() : defaultValue; }); + } + // Get property's value or throw error if it doesn't exist + Future getOrThrow(Reference tr, + Snapshot snapshot = Snapshot::False, + Error err = key_not_found()) const { + return map(get(tr, snapshot), [=](Optional val) -> T { + if (!val.present()) { + throw err; + } + + return val.get(); + }); + } + + Future> get(Database cx, Snapshot snapshot = Snapshot::False) const { + return runRYWTransaction(cx, [=, self = *this](Reference tr) { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + + return self.get(tr, snapshot); + }); + } + + Future getD(Database cx, Snapshot snapshot = Snapshot::False, T defaultValue = T()) const { + return runRYWTransaction(cx, [=, self = *this](Reference tr) { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + + return self.getD(tr, snapshot, defaultValue); + }); + } + + Future getOrThrow(Database cx, Snapshot snapshot = Snapshot::False, Error err = key_not_found()) const { + return runRYWTransaction(cx, [=, self = *this](Reference tr) { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + + return self.getOrThrow(tr, snapshot, err); + }); + } + + void set(Reference tr, T const& val) { + return tr->set(key, ObjectWriter::toValue(val, versionOptions)); + } + + Future set(Database cx, T const& val) { + return runRYWTransaction(cx, [=, self = *this](Reference tr) { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + self.set(tr, val); + return Future(Void()); + }); + } + + void clear(Reference tr) { return tr->clear(key); } + + Key key; + VersionOptions versionOptions; +}; + +// Convenient read/write access to a sorted map of KeyType to ValueType under key prefix +// ValueType is encoded / decoded with ObjectWriter/ObjectReader +// Even though 'this' is not actually mutated, methods that change db keys are not const. +template +class KeyBackedObjectMap { +public: + KeyBackedObjectMap(KeyRef prefix, VersionOptions versionOptions) : space(prefix), versionOptions(versionOptions) {} + + typedef _KeyType KeyType; + typedef _ValueType ValueType; + typedef std::pair PairType; + typedef std::vector PairsType; + + // If end is not present one key past the end of the map is used. + Future getRange(Reference tr, + KeyType const& begin, + Optional const& end, + int limit, + Snapshot snapshot = Snapshot::False, + Reverse reverse = Reverse::False) const { + Key endKey = end.present() ? space.pack(Codec::pack(end.get())) : space.range().end; + return map( + tr->getRange( + KeyRangeRef(space.pack(Codec::pack(begin)), endKey), GetRangeLimits(limit), snapshot, reverse), + [self = *this](RangeResult const& kvs) -> PairsType { + PairsType results; + for (int i = 0; i < kvs.size(); ++i) { + KeyType key = Codec::unpack(self.space.unpack(kvs[i].key)); + ValueType val = ObjectReader::fromStringRef(kvs[i].value, self.versionOptions); + results.push_back(PairType(key, val)); + } + return results; + }); + } + + Future> get(Reference tr, + KeyType const& key, + Snapshot snapshot = Snapshot::False) const { + return map(tr->get(space.pack(Codec::pack(key)), snapshot), + [vo = versionOptions](Optional const& val) -> Optional { + if (val.present()) + return ObjectReader::fromStringRef(val.get(), vo); + return {}; + }); + } + + // Returns a Property that can be get/set that represents key's entry in this this. + KeyBackedObjectProperty getProperty(KeyType const& key) const { + return KeyBackedObjectProperty(space.pack(Codec::pack(key)), + versionOptions); + } + + // Returns the expectedSize of the set key + int set(Reference tr, KeyType const& key, ValueType const& val) { + Key k = space.pack(Codec::pack(key)); + Value v = ObjectWriter::toValue(val, versionOptions); + tr->set(k, v); + return k.expectedSize() + v.expectedSize(); + } + + void erase(Reference tr, KeyType const& key) { + return tr->clear(space.pack(Codec::pack(key))); + } + + void erase(Reference tr, KeyType const& key) { + return tr->clear(space.pack(Codec::pack(key))); + } + + void erase(Reference tr, KeyType const& begin, KeyType const& end) { + return tr->clear(KeyRangeRef(space.pack(Codec::pack(begin)), space.pack(Codec::pack(end)))); + } + + void clear(Reference tr) { return tr->clear(space.range()); } + + Subspace space; + VersionOptions versionOptions; +}; + template class KeyBackedSet { public: diff --git a/fdbserver/BlobManager.actor.cpp b/fdbserver/BlobManager.actor.cpp index 557352f11c7..23d05225c8d 100644 --- a/fdbserver/BlobManager.actor.cpp +++ b/fdbserver/BlobManager.actor.cpp @@ -1169,7 +1169,7 @@ ACTOR Future blobManager(BlobManagerInterface bmInterf, // DB has [A - B) and [C - D). They should show up in knownBlobRanges, and [B - C) should be in removed. // DB has [B - C). It should show up in knownBlobRanges, [B - C) should be in added, and [A - B) and [C - D) should // be in removed. -TEST_CASE("/blobmanager/updateranges") { +TEST_CASE(":/blobmanager/updateranges") { KeyRangeMap knownBlobRanges(false, normalKeys.end); Arena ar; diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 7c79052dd30..05abe0b7951 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -10254,7 +10254,7 @@ ACTOR Future randomRangeScans(IKeyValueStore* kvs, return Void(); } -TEST_CASE("!/redwood/performance/randomRangeScans") { +TEST_CASE(":/redwood/performance/randomRangeScans") { state int prefixLen = 30; state int suffixSize = 12; state int valueSize = 100;