diff --git a/.travis.yml b/.travis.yml index 247d909d17c..e33ac11e6a5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -226,24 +226,12 @@ matrix: - BUILD_TYPE=Debug - NINJA_BUILD=false # misc alternative compilers - - <<: *linux - compiler: gcc-7 - name: gcc-7 - env: - - MATRIX_EVAL="CC=gcc-7 && CXX=g++-7" - - BUILD_TYPE=Debug - <<: *linux compiler: gcc-9 name: gcc-9 env: - MATRIX_EVAL="CC=gcc-9 && CXX=g++-9" - BUILD_TYPE=Debug - - <<: *linux - compiler: clang-7 - name: clang-7 - env: - - MATRIX_EVAL="CC=clang-7 && CXX=clang++-7" - - BUILD_TYPE=Debug - <<: *linux compiler: clang-9 name: clang-9 @@ -274,8 +262,8 @@ matrix: if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_mac/ stage: build os: osx - osx_image: xcode10.3 - name: xcode10, debug + osx_image: xcode11.2 + name: xcode11.2, debug env: # put NIH in non-cache location since it seems to # cause failures when homebrew updates @@ -313,7 +301,7 @@ matrix: - travis_wait ${MAX_TIME_MIN} cmake --build . --parallel --verbose - ./rippled --unittest --quiet --unittest-log --unittest-jobs ${NUM_PROCESSORS} ${TEST_EXTRA_ARGS} - <<: *macos - name: xcode10, release + name: xcode11.2, release before_script: - export BLD_CONFIG=Release - export CMAKE_EXTRA_ARGS="${CMAKE_EXTRA_ARGS} -Dassert=ON" @@ -323,7 +311,7 @@ matrix: - export TEST_EXTRA_ARGS="--unittest-ipv6" - <<: *macos osx_image: xcode11.2 - name: xcode11, debug + name: xcode11.2, debug # windows - &windows if: commit_message !~ /travis_run_/ OR commit_message =~ /travis_run_win/ diff --git a/Builds/CMake/RippledCore.cmake b/Builds/CMake/RippledCore.cmake index bd2b2412f44..82b27ee1896 100644 --- a/Builds/CMake/RippledCore.cmake +++ b/Builds/CMake/RippledCore.cmake @@ -272,19 +272,6 @@ install ( DESTINATION include/ripple/beast/core) install ( FILES - src/ripple/beast/crypto/detail/mac_facade.h - src/ripple/beast/crypto/detail/ripemd_context.h - src/ripple/beast/crypto/detail/sha2_context.h - src/ripple/beast/crypto/ripemd.h - src/ripple/beast/crypto/sha2.h - DESTINATION include/ripple/beast/crypto) -install ( - FILES - src/ripple/beast/cxx17/type_traits.h - DESTINATION include/ripple/beast/cxx17) -install ( - FILES - src/ripple/beast/hash/endian.h src/ripple/beast/hash/hash_append.h src/ripple/beast/hash/meta.h src/ripple/beast/hash/uhash.h @@ -648,7 +635,9 @@ target_sources (rippled PRIVATE src/ripple/shamap/impl/NodeFamily.cpp src/ripple/shamap/impl/SHAMap.cpp src/ripple/shamap/impl/SHAMapDelta.cpp + src/ripple/shamap/impl/SHAMapInnerNode.cpp src/ripple/shamap/impl/SHAMapItem.cpp + src/ripple/shamap/impl/SHAMapLeafNode.cpp src/ripple/shamap/impl/SHAMapNodeID.cpp src/ripple/shamap/impl/SHAMapSync.cpp src/ripple/shamap/impl/SHAMapTreeNode.cpp @@ -895,7 +884,6 @@ target_sources (rippled PRIVATE src/test/protocol/Seed_test.cpp src/test/protocol/SeqProxy_test.cpp src/test/protocol/TER_test.cpp - src/test/protocol/digest_test.cpp src/test/protocol/types_test.cpp #[===============================[ test sources: diff --git a/Builds/CMake/RippledInterface.cmake b/Builds/CMake/RippledInterface.cmake index e0c0c1e5c0f..28a531246fe 100644 --- a/Builds/CMake/RippledInterface.cmake +++ b/Builds/CMake/RippledInterface.cmake @@ -16,9 +16,6 @@ target_compile_definitions (opts BOOST_BEAST_ALLOW_DEPRECATED BOOST_FILESYSTEM_DEPRECATED > - $<$: - USE_BEAST_HASHER - > $<$:BEAST_NO_UNIT_TEST_INLINE=1> $<$:BEAST_DONT_AUTOLINK_TO_WIN32_LIBRARIES=1> $<$:RIPPLE_SINGLE_IO_SERVICE_THREAD=1>) diff --git a/bin/ci/test.sh b/bin/ci/test.sh index 6162f37aa0e..11615d732b7 100755 --- a/bin/ci/test.sh +++ b/bin/ci/test.sh @@ -24,7 +24,6 @@ declare -a manual_tests=( 'ripple.consensus.ByzantineFailureSim' 'ripple.consensus.DistributedValidators' 'ripple.consensus.ScaleFreeSim' - 'ripple.ripple_data.digest' 'ripple.tx.CrossingLimits' 'ripple.tx.FindOversizeCross' 'ripple.tx.Offer_manual' diff --git a/bin/ci/ubuntu/build-and-test.sh b/bin/ci/ubuntu/build-and-test.sh index ea83e768656..1f9da09b884 100755 --- a/bin/ci/ubuntu/build-and-test.sh +++ b/bin/ci/ubuntu/build-and-test.sh @@ -137,7 +137,6 @@ else # descending execution time (longest running tests at top) declare -a manual_tests=( 'ripple.ripple_data.reduce_relay_simulate' - 'ripple.ripple_data.digest' 'ripple.tx.Offer_manual' 'ripple.tx.CrossingLimits' 'ripple.tx.PlumpBook' diff --git a/cfg/rippled-example.cfg b/cfg/rippled-example.cfg index 67baf6ffcd4..264b22f8a57 100644 --- a/cfg/rippled-example.cfg +++ b/cfg/rippled-example.cfg @@ -489,6 +489,23 @@ # single host from consuming all inbound slots. If the value is not # present the server will autoconfigure an appropriate limit. # +# max_unknown_time = +# +# The maximum amount of time, in seconds, that an outbound connection +# is allowed to stay in the "unknown" tracking state. This option can +# take any value between 300 and 1800 seconds, inclusive. If the option +# is not present the server will autoconfigure an appropriate limit. +# +# The current default (which is subject to change) is 600 seconds. +# +# max_diverged_time = +# +# The maximum amount of time, in seconds, that an outbound connection +# is allowed to stay in the "diverged" tracking state. The option can +# take any value between 60 and 900 seconds, inclusive. If the option +# is not present the server will autoconfigure an appropriate limit. +# +# The current default (which is subject to change) is 300 seconds. # # # [transaction_queue] EXPERIMENTAL @@ -590,13 +607,13 @@ # #------------------------------------------------------------------------------- # -# 3. Ripple Protocol +# 3. Protocol # #------------------- # # These settings affect the behavior of the server instance with respect -# to Ripple payment protocol level activities such as validating and -# closing ledgers or adjusting fees in response to server overloads. +# to protocol level activities such as validating and closing ledgers +# adjusting fees in response to server overloads. # # # @@ -617,12 +634,6 @@ # Legal values are: "trusted" and "all". The default is "all". # # -# [node_size] -# -# Tunes the servers based on the expected load and available memory. Legal -# sizes are "tiny", "small", "medium", "large", and "huge". We recommend -# you start at the default and raise the setting if you have extra memory. -# The default is "tiny". # # # @@ -1191,6 +1202,14 @@ # #----------------- # +# [node_size] +# +# Tunes the servers based on the expected load and available memory. Legal +# sizes are "tiny", "small", "medium", "large", and "huge". We recommend +# you start at the default and raise the setting if you have extra memory. +# If no value is specified, the code assumes the proper size is "tiny". The +# default configuration file explicitly specifies "medium" as the size. +# # [signing_support] # # Specifies whether the server will accept "sign" and "sign_for" commands diff --git a/src/ripple/app/consensus/RCLConsensus.cpp b/src/ripple/app/consensus/RCLConsensus.cpp index fd61ae2bf41..d0bb15fcac8 100644 --- a/src/ripple/app/consensus/RCLConsensus.cpp +++ b/src/ripple/app/consensus/RCLConsensus.cpp @@ -315,9 +315,8 @@ RCLConsensus::Adaptor::onClose( Serializer s(2048); tx.first->add(s); initialSet->addItem( - SHAMapItem(tx.first->getTransactionID(), std::move(s)), - true, - false); + SHAMapNodeType::tnTRANSACTION_NM, + SHAMapItem(tx.first->getTransactionID(), std::move(s))); } // Add pseudo-transactions to the set diff --git a/src/ripple/app/consensus/RCLCxPeerPos.h b/src/ripple/app/consensus/RCLCxPeerPos.h index 83456a101f2..9d448aac48c 100644 --- a/src/ripple/app/consensus/RCLCxPeerPos.h +++ b/src/ripple/app/consensus/RCLCxPeerPos.h @@ -113,12 +113,6 @@ class RCLCxPeerPos Slice const& signature, uint256 const& suppress, Proposal&& proposal); - - static char const* - getCountedObjectName() - { - return "RCLCxPeerPos::Data"; - } }; std::shared_ptr data_; diff --git a/src/ripple/app/consensus/RCLCxTx.h b/src/ripple/app/consensus/RCLCxTx.h index 14c61a6023d..45ee55743c2 100644 --- a/src/ripple/app/consensus/RCLCxTx.h +++ b/src/ripple/app/consensus/RCLCxTx.h @@ -91,7 +91,8 @@ class RCLTxSet insert(Tx const& t) { return map_->addItem( - SHAMapItem{t.id(), t.tx_.peekData()}, true, false); + SHAMapNodeType::tnTRANSACTION_NM, + SHAMapItem{t.id(), t.tx_.peekData()}); } /** Remove a transaction from the set. diff --git a/src/ripple/app/ledger/AcceptedLedgerTx.cpp b/src/ripple/app/ledger/AcceptedLedgerTx.cpp index 4c0150d4a8d..c92ebffe04b 100644 --- a/src/ripple/app/ledger/AcceptedLedgerTx.cpp +++ b/src/ripple/app/ledger/AcceptedLedgerTx.cpp @@ -74,7 +74,7 @@ std::string AcceptedLedgerTx::getEscMeta() const { assert(!mRawMeta.empty()); - return sqlEscape(mRawMeta); + return sqlBlobLiteral(mRawMeta); } void diff --git a/src/ripple/app/ledger/AccountStateSF.cpp b/src/ripple/app/ledger/AccountStateSF.cpp index 5d3456a9fcc..1137fae1315 100644 --- a/src/ripple/app/ledger/AccountStateSF.cpp +++ b/src/ripple/app/ledger/AccountStateSF.cpp @@ -27,7 +27,7 @@ AccountStateSF::gotNode( SHAMapHash const& nodeHash, std::uint32_t ledgerSeq, Blob&& nodeData, - SHAMapTreeNode::TNType) const + SHAMapNodeType) const { db_.store( hotACCOUNT_NODE, std::move(nodeData), nodeHash.as_uint256(), ledgerSeq); diff --git a/src/ripple/app/ledger/AccountStateSF.h b/src/ripple/app/ledger/AccountStateSF.h index aa1557f98de..5829880d57a 100644 --- a/src/ripple/app/ledger/AccountStateSF.h +++ b/src/ripple/app/ledger/AccountStateSF.h @@ -42,7 +42,7 @@ class AccountStateSF : public SHAMapSyncFilter SHAMapHash const& nodeHash, std::uint32_t ledgerSeq, Blob&& nodeData, - SHAMapTreeNode::TNType type) const override; + SHAMapNodeType type) const override; boost::optional getNode(SHAMapHash const& nodeHash) const override; diff --git a/src/ripple/app/ledger/ConsensusTransSetSF.cpp b/src/ripple/app/ledger/ConsensusTransSetSF.cpp index 3368751216b..5ae86443886 100644 --- a/src/ripple/app/ledger/ConsensusTransSetSF.cpp +++ b/src/ripple/app/ledger/ConsensusTransSetSF.cpp @@ -41,14 +41,14 @@ ConsensusTransSetSF::gotNode( SHAMapHash const& nodeHash, std::uint32_t, Blob&& nodeData, - SHAMapTreeNode::TNType type) const + SHAMapNodeType type) const { if (fromFilter) return; m_nodeCache.insert(nodeHash, nodeData); - if ((type == SHAMapTreeNode::tnTRANSACTION_NM) && (nodeData.size() > 16)) + if ((type == SHAMapNodeType::tnTRANSACTION_NM) && (nodeData.size() > 16)) { // this is a transaction, and we didn't have it JLOG(j_.debug()) diff --git a/src/ripple/app/ledger/ConsensusTransSetSF.h b/src/ripple/app/ledger/ConsensusTransSetSF.h index c413caffa67..a010a1ba3be 100644 --- a/src/ripple/app/ledger/ConsensusTransSetSF.h +++ b/src/ripple/app/ledger/ConsensusTransSetSF.h @@ -45,7 +45,7 @@ class ConsensusTransSetSF : public SHAMapSyncFilter SHAMapHash const& nodeHash, std::uint32_t ledgerSeq, Blob&& nodeData, - SHAMapTreeNode::TNType type) const override; + SHAMapNodeType type) const override; boost::optional getNode(SHAMapHash const& nodeHash) const override; diff --git a/src/ripple/app/ledger/InboundLedger.h b/src/ripple/app/ledger/InboundLedger.h index 6688d0dc358..4ab66b6a676 100644 --- a/src/ripple/app/ledger/InboundLedger.h +++ b/src/ripple/app/ledger/InboundLedger.h @@ -38,12 +38,6 @@ class InboundLedger final : public PeerSet, public: using clock_type = beast::abstract_clock; - static char const* - getCountedObjectName() - { - return "InboundLedger"; - } - using PeerDataPairType = std::pair, std::shared_ptr>; diff --git a/src/ripple/app/ledger/InboundLedgers.h b/src/ripple/app/ledger/InboundLedgers.h index 71ba74a071c..ae27b03d9ce 100644 --- a/src/ripple/app/ledger/InboundLedgers.h +++ b/src/ripple/app/ledger/InboundLedgers.h @@ -36,7 +36,7 @@ class InboundLedgers public: using clock_type = beast::abstract_clock; - virtual ~InboundLedgers() = 0; + virtual ~InboundLedgers() = default; // VFALCO TODO Should this be called findOrAdd ? // diff --git a/src/ripple/app/ledger/Ledger.cpp b/src/ripple/app/ledger/Ledger.cpp index 524f81c8446..9aa5e212c7e 100644 --- a/src/ripple/app/ledger/Ledger.cpp +++ b/src/ripple/app/ledger/Ledger.cpp @@ -202,7 +202,7 @@ Ledger::Ledger( rawInsert(sle); } - stateMap_->flushDirty(hotACCOUNT_NODE, info_.seq); + stateMap_->flushDirty(hotACCOUNT_NODE); setImmutable(config); } @@ -354,7 +354,7 @@ bool Ledger::addSLE(SLE const& sle) { SHAMapItem item(sle.key(), sle.getSerializer()); - return stateMap_->addItem(std::move(item), false, false); + return stateMap_->addItem(SHAMapNodeType::tnACCOUNT_STATE, std::move(item)); } //------------------------------------------------------------------------------ @@ -499,8 +499,9 @@ Ledger::rawInsert(std::shared_ptr const& sle) { Serializer ss; sle->add(ss); - auto item = std::make_shared(sle->key(), std::move(ss)); - if (!stateMap_->addGiveItem(std::move(item), false, false)) + if (!stateMap_->addGiveItem( + SHAMapNodeType::tnACCOUNT_STATE, + std::make_shared(sle->key(), std::move(ss)))) LogicError("Ledger::rawInsert: key already exists"); } @@ -509,9 +510,9 @@ Ledger::rawReplace(std::shared_ptr const& sle) { Serializer ss; sle->add(ss); - auto item = std::make_shared(sle->key(), std::move(ss)); - - if (!stateMap_->updateGiveItem(std::move(item), false, false)) + if (!stateMap_->updateGiveItem( + SHAMapNodeType::tnACCOUNT_STATE, + std::make_shared(sle->key(), std::move(ss)))) LogicError("Ledger::rawReplace: key not found"); } @@ -527,8 +528,9 @@ Ledger::rawTxInsert( Serializer s(txn->getDataLength() + metaData->getDataLength() + 16); s.addVL(txn->peekData()); s.addVL(metaData->peekData()); - auto item = std::make_shared(key, std::move(s)); - if (!txMap().addGiveItem(std::move(item), true, true)) + if (!txMap().addGiveItem( + SHAMapNodeType::tnTRANSACTION_MD, + std::make_shared(key, std::move(s)))) LogicError("duplicate_tx: " + to_string(key)); } @@ -755,7 +757,7 @@ Ledger::walkLedger(beast::Journal j) const } bool -Ledger::assertSane(beast::Journal ledgerJ) const +Ledger::assertSensible(beast::Journal ledgerJ) const { if (info_.hash.isNonZero() && info_.accountHash.isNonZero() && stateMap_ && txMap_ && (info_.accountHash == stateMap_->getHash().as_uint256()) && @@ -769,7 +771,7 @@ Ledger::assertSane(beast::Journal ledgerJ) const j[jss::accountTreeHash] = to_string(info_.accountHash); j[jss::transTreeHash] = to_string(info_.txHash); - JLOG(ledgerJ.fatal()) << "ledger is not sane" << j; + JLOG(ledgerJ.fatal()) << "ledger is not sensible" << j; assert(false); @@ -1173,13 +1175,13 @@ loadLedgerHelper(std::string const& sqlSuffix, Application& app, bool acquire) uint256 prevHash{}, accountHash{}, transHash{}; if (sLedgerHash) - ledgerHash.SetHexExact(*sLedgerHash); + (void)ledgerHash.parseHex(*sLedgerHash); if (sPrevHash) - prevHash.SetHexExact(*sPrevHash); + (void)prevHash.parseHex(*sPrevHash); if (sAccountHash) - accountHash.SetHexExact(*sAccountHash); + (void)accountHash.parseHex(*sAccountHash); if (sTransHash) - transHash.SetHexExact(*sTransHash); + (void)transHash.parseHex(*sTransHash); using time_point = NetClock::time_point; using duration = NetClock::duration; @@ -1266,7 +1268,7 @@ getHashByIndex(std::uint32_t ledgerIndex, Application& app) std::string sql = "SELECT LedgerHash FROM Ledgers INDEXED BY SeqLedger WHERE LedgerSeq='"; - sql.append(beast::lexicalCastThrow(ledgerIndex)); + sql.append(std::to_string(ledgerIndex)); sql.append("';"); std::string hash; @@ -1284,7 +1286,7 @@ getHashByIndex(std::uint32_t ledgerIndex, Application& app) return ret; } - ret.SetHexExact(hash); + (void)ret.parseHex(hash); return ret; } @@ -1310,10 +1312,7 @@ getHashesByIndex( return false; } - ledgerHash.SetHexExact(*lhO); - parentHash.SetHexExact(*phO); - - return true; + return ledgerHash.parseHex(*lhO) && parentHash.parseHex(*phO); } std::map> @@ -1323,9 +1322,9 @@ getHashesByIndex(std::uint32_t minSeq, std::uint32_t maxSeq, Application& app) std::string sql = "SELECT LedgerSeq,LedgerHash,PrevHash FROM Ledgers WHERE LedgerSeq >= "; - sql.append(beast::lexicalCastThrow(minSeq)); + sql.append(std::to_string(minSeq)); sql.append(" AND LedgerSeq <= "); - sql.append(beast::lexicalCastThrow(maxSeq)); + sql.append(std::to_string(maxSeq)); sql.append(";"); auto db = app.getLedgerDB().checkoutDb(); @@ -1341,9 +1340,9 @@ getHashesByIndex(std::uint32_t minSeq, std::uint32_t maxSeq, Application& app) { std::pair& hashes = ret[rangeCheckedCast(ls)]; - hashes.first.SetHexExact(lh); + (void)hashes.first.parseHex(lh); if (ph) - hashes.second.SetHexExact(*ph); + (void)hashes.second.parseHex(*ph); else hashes.second.zero(); if (!ph) diff --git a/src/ripple/app/ledger/Ledger.h b/src/ripple/app/ledger/Ledger.h index ce67f381142..1e3eb791d6e 100644 --- a/src/ripple/app/ledger/Ledger.h +++ b/src/ripple/app/ledger/Ledger.h @@ -80,12 +80,6 @@ class Ledger final : public std::enable_shared_from_this, public CountedObject { public: - static char const* - getCountedObjectName() - { - return "Ledger"; - } - Ledger(Ledger const&) = delete; Ledger& operator=(Ledger const&) = delete; @@ -322,7 +316,7 @@ class Ledger final : public std::enable_shared_from_this, walkLedger(beast::Journal j) const; bool - assertSane(beast::Journal ledgerJ) const; + assertSensible(beast::Journal ledgerJ) const; void invariants() const; diff --git a/src/ripple/app/ledger/TransactionMaster.h b/src/ripple/app/ledger/TransactionMaster.h index 6c3f13bffa0..a902fadf9d7 100644 --- a/src/ripple/app/ledger/TransactionMaster.h +++ b/src/ripple/app/ledger/TransactionMaster.h @@ -69,7 +69,7 @@ class TransactionMaster std::shared_ptr fetch( std::shared_ptr const& item, - SHAMapTreeNode::TNType type, + SHAMapNodeType type, std::uint32_t uCommitLedger); // return value: true = we had the transaction already diff --git a/src/ripple/app/ledger/TransactionStateSF.cpp b/src/ripple/app/ledger/TransactionStateSF.cpp index 5ac9b44fe9c..e717eef9bf9 100644 --- a/src/ripple/app/ledger/TransactionStateSF.cpp +++ b/src/ripple/app/ledger/TransactionStateSF.cpp @@ -27,10 +27,10 @@ TransactionStateSF::gotNode( SHAMapHash const& nodeHash, std::uint32_t ledgerSeq, Blob&& nodeData, - SHAMapTreeNode::TNType type) const + SHAMapNodeType type) const { - assert(type != SHAMapTreeNode::tnTRANSACTION_NM); + assert(type != SHAMapNodeType::tnTRANSACTION_NM); db_.store( hotTRANSACTION_NODE, std::move(nodeData), diff --git a/src/ripple/app/ledger/TransactionStateSF.h b/src/ripple/app/ledger/TransactionStateSF.h index bfe33dbe51c..c9cae3dc8c9 100644 --- a/src/ripple/app/ledger/TransactionStateSF.h +++ b/src/ripple/app/ledger/TransactionStateSF.h @@ -42,7 +42,7 @@ class TransactionStateSF : public SHAMapSyncFilter SHAMapHash const& nodeHash, std::uint32_t ledgerSeq, Blob&& nodeData, - SHAMapTreeNode::TNType type) const override; + SHAMapNodeType type) const override; boost::optional getNode(SHAMapHash const& nodeHash) const override; diff --git a/src/ripple/app/ledger/impl/BuildLedger.cpp b/src/ripple/app/ledger/impl/BuildLedger.cpp index 97592220eb9..f70b754ab7c 100644 --- a/src/ripple/app/ledger/impl/BuildLedger.cpp +++ b/src/ripple/app/ledger/impl/BuildLedger.cpp @@ -67,10 +67,8 @@ buildLedgerImpl( // Write the final version of all modified SHAMap // nodes to the node store to preserve the new LCL - int const asf = - built->stateMap().flushDirty(hotACCOUNT_NODE, built->info().seq); - int const tmf = - built->txMap().flushDirty(hotTRANSACTION_NODE, built->info().seq); + int const asf = built->stateMap().flushDirty(hotACCOUNT_NODE); + int const tmf = built->txMap().flushDirty(hotTRANSACTION_NODE); JLOG(j.debug()) << "Flushed " << asf << " accounts and " << tmf << " transaction nodes"; } diff --git a/src/ripple/app/ledger/impl/InboundLedger.cpp b/src/ripple/app/ledger/impl/InboundLedger.cpp index 39fe904016c..ffe9e284cea 100644 --- a/src/ripple/app/ledger/impl/InboundLedger.cpp +++ b/src/ripple/app/ledger/impl/InboundLedger.cpp @@ -235,36 +235,42 @@ InboundLedger::~InboundLedger() } } -std::vector -InboundLedger::neededTxHashes(int max, SHAMapSyncFilter* filter) const +static std::vector +neededHashes( + uint256 const& root, + SHAMap& map, + int max, + SHAMapSyncFilter* filter) { std::vector ret; - if (mLedger->info().txHash.isNonZero()) + if (!root.isZero()) { - if (mLedger->txMap().getHash().isZero()) - ret.push_back(mLedger->info().txHash); + if (map.getHash().isZero()) + ret.push_back(root); else - ret = mLedger->txMap().getNeededHashes(max, filter); + { + auto mn = map.getMissingNodes(max, filter); + ret.reserve(mn.size()); + for (auto const& n : mn) + ret.push_back(n.second); + } } return ret; } std::vector -InboundLedger::neededStateHashes(int max, SHAMapSyncFilter* filter) const +InboundLedger::neededTxHashes(int max, SHAMapSyncFilter* filter) const { - std::vector ret; - - if (mLedger->info().accountHash.isNonZero()) - { - if (mLedger->stateMap().getHash().isZero()) - ret.push_back(mLedger->info().accountHash); - else - ret = mLedger->stateMap().getNeededHashes(max, filter); - } + return neededHashes(mLedger->info().txHash, mLedger->txMap(), max, filter); +} - return ret; +std::vector +InboundLedger::neededStateHashes(int max, SHAMapSyncFilter* filter) const +{ + return neededHashes( + mLedger->info().accountHash, mLedger->stateMap(), max, filter); } LedgerInfo @@ -947,14 +953,20 @@ InboundLedger::receiveNode(protocol::TMLedgerData& packet, SHAMapAddNode& san) { for (auto const& node : packet.nodes()) { - SHAMapNodeID const nodeID( - node.nodeid().data(), node.nodeid().size()); - if (nodeID.isRoot()) + auto const nodeID = deserializeSHAMapNodeID(node.nodeid()); + + if (!nodeID) + { + san.incInvalid(); + return; + } + + if (nodeID->isRoot()) san += map.addRootNode( rootHash, makeSlice(node.nodedata()), filter.get()); else san += map.addKnownNode( - nodeID, makeSlice(node.nodedata()), filter.get()); + *nodeID, makeSlice(node.nodedata()), filter.get()); if (!san.isGood()) { diff --git a/src/ripple/app/ledger/impl/InboundLedgers.cpp b/src/ripple/app/ledger/impl/InboundLedgers.cpp index 91bb735086c..fa8cc9d17cf 100644 --- a/src/ripple/app/ledger/impl/InboundLedgers.cpp +++ b/src/ripple/app/ledger/impl/InboundLedgers.cpp @@ -43,10 +43,8 @@ class InboundLedgersImp : public InboundLedgers, public Stoppable beast::Journal const j_; public: - using u256_acq_pair = std::pair>; - // How long before we try again to acquire the same ledger - static const std::chrono::minutes kReacquireInterval; + static constexpr std::chrono::minutes const kReacquireInterval{5}; InboundLedgersImp( Application& app, @@ -166,40 +164,38 @@ class InboundLedgersImp : public InboundLedgers, public Stoppable gotLedgerData( LedgerHash const& hash, std::shared_ptr peer, - std::shared_ptr packet_ptr) override + std::shared_ptr packet) override { - protocol::TMLedgerData& packet = *packet_ptr; - - JLOG(j_.trace()) << "Got data (" << packet.nodes().size() - << ") for acquiring ledger: " << hash; - - auto ledger = find(hash); - - if (!ledger) + if (auto ledger = find(hash)) { - JLOG(j_.trace()) << "Got data for ledger we're no longer acquiring"; + JLOG(j_.trace()) << "Got data (" << packet->nodes().size() + << ") for acquiring ledger: " << hash; - // If it's state node data, stash it because it still might be - // useful. - if (packet.type() == protocol::liAS_NODE) - { + // Stash the data for later processing and see if we need to + // dispatch + if (ledger->gotData(std::weak_ptr(peer), packet)) app_.getJobQueue().addJob( - jtLEDGER_DATA, "gotStaleData", [this, packet_ptr](Job&) { - gotStaleData(packet_ptr); + jtLEDGER_DATA, "processLedgerData", [ledger](Job&) { + ledger->runData(); }); - } - return false; + return true; } - // Stash the data for later processing and see if we need to dispatch - if (ledger->gotData(std::weak_ptr(peer), packet_ptr)) + JLOG(j_.trace()) << "Got data for ledger " << hash + << " which we're no longer acquiring"; + + // If it's state node data, stash it because it still might be + // useful. + if (packet->type() == protocol::liAS_NODE) + { app_.getJobQueue().addJob( - jtLEDGER_DATA, "processLedgerData", [this, hash](Job&) { - doLedgerData(hash); + jtLEDGER_DATA, "gotStaleData", [this, packet](Job&) { + gotStaleData(packet); }); + } - return true; + return false; } void @@ -219,14 +215,6 @@ class InboundLedgersImp : public InboundLedgers, public Stoppable return mRecentFailures.find(h) != mRecentFailures.end(); } - /** Called (indirectly) only by gotLedgerData(). */ - void - doLedgerData(LedgerHash hash) - { - if (auto ledger = find(hash)) - ledger->runData(); - } - /** We got some data for a ledger we are no longer acquiring Since we paid the price to receive it, we might as well stash it in case we need it. @@ -247,17 +235,17 @@ class InboundLedgersImp : public InboundLedgers, public Stoppable if (!node.has_nodeid() || !node.has_nodedata()) return; - auto newNode = SHAMapAbstractNode::makeFromWire( - makeSlice(node.nodedata())); + auto newNode = + SHAMapTreeNode::makeFromWire(makeSlice(node.nodedata())); if (!newNode) return; s.erase(); - newNode->addRaw(s, snfPREFIX); + newNode->serializeWithPrefix(s); app_.getLedgerMaster().addFetchPack( - newNode->getNodeHash().as_uint256(), + newNode->getHash().as_uint256(), std::make_shared(s.begin(), s.end())); } } @@ -296,27 +284,27 @@ class InboundLedgersImp : public InboundLedgers, public Stoppable { Json::Value ret(Json::objectValue); - std::vector acquires; + std::vector>> acqs; + { ScopedLockType sl(mLock); - acquires.reserve(mLedgers.size()); + acqs.reserve(mLedgers.size()); for (auto const& it : mLedgers) { assert(it.second); - acquires.push_back(it); + acqs.push_back(it); } for (auto const& it : mRecentFailures) { if (it.second > 1) - ret[beast::lexicalCastThrow(it.second)] - [jss::failed] = true; + ret[std::to_string(it.second)][jss::failed] = true; else ret[to_string(it.first)][jss::failed] = true; } } - for (auto const& it : acquires) + for (auto const& it : acqs) { // getJson is expensive, so call without the lock std::uint32_t seq = it.second->getSeq(); @@ -420,11 +408,6 @@ class InboundLedgersImp : public InboundLedgers, public Stoppable //------------------------------------------------------------------------------ -decltype(InboundLedgersImp::kReacquireInterval) - InboundLedgersImp::kReacquireInterval{5}; - -InboundLedgers::~InboundLedgers() = default; - std::unique_ptr make_InboundLedgers( Application& app, diff --git a/src/ripple/app/ledger/impl/InboundTransactions.cpp b/src/ripple/app/ledger/impl/InboundTransactions.cpp index b4c2cf734b0..db67f1738b6 100644 --- a/src/ripple/app/ledger/impl/InboundTransactions.cpp +++ b/src/ripple/app/ledger/impl/InboundTransactions.cpp @@ -159,15 +159,21 @@ class InboundTransactionsImp : public InboundTransactions, public Stoppable std::list nodeData; for (auto const& node : packet.nodes()) { - if (!node.has_nodeid() || !node.has_nodedata() || - (node.nodeid().size() != 33)) + if (!node.has_nodeid() || !node.has_nodedata()) { peer->charge(Resource::feeInvalidRequest); return; } - nodeIDs.emplace_back( - node.nodeid().data(), static_cast(node.nodeid().size())); + auto const id = deserializeSHAMapNodeID(node.nodeid()); + + if (!id) + { + peer->charge(Resource::feeBadData); + return; + } + + nodeIDs.emplace_back(*id); nodeData.emplace_back( node.nodedata().begin(), node.nodedata().end()); } diff --git a/src/ripple/app/ledger/impl/LedgerMaster.cpp b/src/ripple/app/ledger/impl/LedgerMaster.cpp index 1ac7bb122a1..3a21c050d28 100644 --- a/src/ripple/app/ledger/impl/LedgerMaster.cpp +++ b/src/ripple/app/ledger/impl/LedgerMaster.cpp @@ -404,8 +404,8 @@ LedgerMaster::addHeldTransaction( } // Validate a ledger's close time and sequence number if we're considering -// jumping to that ledger. This helps defend agains some rare hostile or -// insane majority scenarios. +// jumping to that ledger. This helps defend against some rare hostile or +// diverged majority scenarios. bool LedgerMaster::canBeCurrent(std::shared_ptr const& ledger) { @@ -964,9 +964,9 @@ LedgerMaster::checkAccept(uint256 const& hash, std::uint32_t seq) { if ((seq != 0) && (getValidLedgerIndex() == 0)) { - // Set peers sane early if we can + // Set peers converged early if we can if (valCount >= app_.validators().quorum()) - app_.overlay().checkSanity(seq); + app_.overlay().checkTracking(seq); } // FIXME: We may not want to fetch a ledger with just one @@ -2033,6 +2033,64 @@ LedgerMaster::gotFetchPack(bool progress, std::uint32_t seq) } } +/** Populate a fetch pack with data from the map the recipient wants. + + A recipient may or may not have the map that they are asking for. If + they do, we can optimize the transfer by not including parts of the + map that they are already have. + + @param have The map that the recipient already has (if any). + @param cnt The maximum number of nodes to return. + @param into The protocol object into which we add information. + @param seq The sequence number of the ledger the map is a part of. + @param withLeaves True if leaf nodes should be included. + + @note: The withLeaves parameter is configurable even though the + code, so far, only ever sets the parameter to true. + + The rationale is that for transaction trees, it may make + sense to not include the leaves if the fetch pack is being + constructed for someone attempting to get a recent ledger + for which they already have the transactions. + + However, for historical ledgers, which is the only use we + have for fetch packs right now, it makes sense to include + the transactions because the caller is unlikely to have + them. + */ +static void +populateFetchPack( + SHAMap const& want, + SHAMap const* have, + std::uint32_t cnt, + protocol::TMGetObjectByHash* into, + std::uint32_t seq, + bool withLeaves = true) +{ + assert(cnt != 0); + + Serializer s(1024); + + want.visitDifferences( + have, + [&s, withLeaves, &cnt, into, seq](SHAMapTreeNode const& n) -> bool { + if (!withLeaves && n.isLeaf()) + return true; + + s.erase(); + n.serializeWithPrefix(s); + + auto const& hash = n.getHash().as_uint256(); + + protocol::TMIndexedObject* obj = into->add_objects(); + obj->set_ledgerseq(seq); + obj->set_hash(hash.data(), hash.size()); + obj->set_data(s.getDataPtr(), s.getLength()); + + return --cnt != 0; + }); +} + void LedgerMaster::makeFetchPack( std::weak_ptr const& wPeer, @@ -2058,55 +2116,46 @@ LedgerMaster::makeFetchPack( if (!peer) return; - auto haveLedger = getLedgerByHash(haveLedgerHash); + auto have = getLedgerByHash(haveLedgerHash); - if (!haveLedger) + if (!have) { JLOG(m_journal.info()) - << "Peer requests fetch pack for ledger we don't have: " - << haveLedger; + << "Peer requests fetch pack for ledger we don't have: " << have; peer->charge(Resource::feeRequestNoReply); return; } - if (haveLedger->open()) + if (have->open()) { JLOG(m_journal.warn()) - << "Peer requests fetch pack from open ledger: " << haveLedger; + << "Peer requests fetch pack from open ledger: " << have; peer->charge(Resource::feeInvalidRequest); return; } - if (haveLedger->info().seq < getEarliestFetch()) + if (have->info().seq < getEarliestFetch()) { JLOG(m_journal.debug()) << "Peer requests fetch pack that is too early"; peer->charge(Resource::feeInvalidRequest); return; } - auto wantLedger = getLedgerByHash(haveLedger->info().parentHash); + auto want = getLedgerByHash(have->info().parentHash); - if (!wantLedger) + if (!want) { JLOG(m_journal.info()) << "Peer requests fetch pack for ledger whose predecessor we " - << "don't have: " << haveLedger; + << "don't have: " << have; peer->charge(Resource::feeRequestNoReply); return; } - auto fpAppender = [](protocol::TMGetObjectByHash* reply, - std::uint32_t ledgerSeq, - SHAMapHash const& hash, - const Blob& blob) { - protocol::TMIndexedObject& newObj = *(reply->add_objects()); - newObj.set_ledgerseq(ledgerSeq); - newObj.set_hash(hash.as_uint256().begin(), 256 / 8); - newObj.set_data(&blob[0], blob.size()); - }; - try { + Serializer hdr(128); + protocol::TMGetObjectByHash reply; reply.set_query(false); @@ -2121,56 +2170,49 @@ LedgerMaster::makeFetchPack( // 2. Add the nodes for the AccountStateMap of that ledger. // 3. If there are transactions, add the nodes for the // transactions of the ledger. - // 4. If the FetchPack now contains greater than or equal to - // 256 entries then stop. + // 4. If the FetchPack now contains at least 512 entries then stop. // 5. If not very much time has elapsed, then loop back and repeat // the same process adding the previous ledger to the FetchPack. do { - std::uint32_t lSeq = wantLedger->info().seq; - - protocol::TMIndexedObject& newObj = *reply.add_objects(); - newObj.set_hash(wantLedger->info().hash.data(), 256 / 8); - Serializer s(256); - s.add32(HashPrefix::ledgerMaster); - addRaw(wantLedger->info(), s); - newObj.set_data(s.getDataPtr(), s.getLength()); - newObj.set_ledgerseq(lSeq); - - wantLedger->stateMap().getFetchPack( - &haveLedger->stateMap(), - true, - 16384, - std::bind( - fpAppender, - &reply, - lSeq, - std::placeholders::_1, - std::placeholders::_2)); - - if (wantLedger->info().txHash.isNonZero()) - wantLedger->txMap().getFetchPack( - nullptr, - true, - 512, - std::bind( - fpAppender, - &reply, - lSeq, - std::placeholders::_1, - std::placeholders::_2)); + std::uint32_t lSeq = want->info().seq; + + { + // Serialize the ledger header: + hdr.erase(); + + hdr.add32(HashPrefix::ledgerMaster); + addRaw(want->info(), hdr); + + // Add the data + protocol::TMIndexedObject* obj = reply.add_objects(); + obj->set_hash( + want->info().hash.data(), want->info().hash.size()); + obj->set_data(hdr.getDataPtr(), hdr.getLength()); + obj->set_ledgerseq(lSeq); + } + + populateFetchPack( + want->stateMap(), &have->stateMap(), 16384, &reply, lSeq); + + // We use nullptr here because transaction maps are per ledger + // and so the requestor is unlikely to already have it. + if (want->info().txHash.isNonZero()) + populateFetchPack(want->txMap(), nullptr, 512, &reply, lSeq); if (reply.objects().size() >= 512) break; - // move may save a ref/unref - haveLedger = std::move(wantLedger); - wantLedger = getLedgerByHash(haveLedger->info().parentHash); - } while (wantLedger && UptimeClock::now() <= uptime + 1s); + have = std::move(want); + want = getLedgerByHash(have->info().parentHash); + } while (want && UptimeClock::now() <= uptime + 1s); - JLOG(m_journal.info()) - << "Built fetch pack with " << reply.objects().size() << " nodes"; auto msg = std::make_shared(reply, protocol::mtGET_OBJECTS); + + JLOG(m_journal.info()) + << "Built fetch pack with " << reply.objects().size() << " nodes (" + << msg->getBufferSize() << " bytes)"; + peer->send(msg); } catch (std::exception const&) diff --git a/src/ripple/app/ledger/impl/TransactionAcquire.h b/src/ripple/app/ledger/impl/TransactionAcquire.h index be93a86a68c..c1119c4acec 100644 --- a/src/ripple/app/ledger/impl/TransactionAcquire.h +++ b/src/ripple/app/ledger/impl/TransactionAcquire.h @@ -34,15 +34,8 @@ class TransactionAcquire final public CountedObject { public: - static char const* - getCountedObjectName() - { - return "TransactionAcquire"; - } - using pointer = std::shared_ptr; -public: TransactionAcquire(Application& app, uint256 const& hash); ~TransactionAcquire() = default; diff --git a/src/ripple/app/ledger/impl/TransactionMaster.cpp b/src/ripple/app/ledger/impl/TransactionMaster.cpp index f83b1aa92d4..e01c0d7c3a6 100644 --- a/src/ripple/app/ledger/impl/TransactionMaster.cpp +++ b/src/ripple/app/ledger/impl/TransactionMaster.cpp @@ -108,7 +108,7 @@ TransactionMaster::fetch( std::shared_ptr TransactionMaster::fetch( std::shared_ptr const& item, - SHAMapTreeNode::TNType type, + SHAMapNodeType type, std::uint32_t uCommitLedger) { std::shared_ptr txn; @@ -116,12 +116,12 @@ TransactionMaster::fetch( if (!iTx) { - if (type == SHAMapTreeNode::tnTRANSACTION_NM) + if (type == SHAMapNodeType::tnTRANSACTION_NM) { SerialIter sit(item->slice()); txn = std::make_shared(std::ref(sit)); } - else if (type == SHAMapTreeNode::tnTRANSACTION_MD) + else if (type == SHAMapNodeType::tnTRANSACTION_MD) { auto blob = SerialIter{item->data(), item->size()}.getVL(); txn = std::make_shared( diff --git a/src/ripple/app/main/Application.cpp b/src/ripple/app/main/Application.cpp index 18afa228f5e..5b4531af49e 100644 --- a/src/ripple/app/main/Application.cpp +++ b/src/ripple/app/main/Application.cpp @@ -1878,7 +1878,7 @@ ApplicationImp::loadLedgerFromFile(std::string const& name) uint256 uIndex; - if (!uIndex.SetHex(entry[jss::index].asString())) + if (!uIndex.parseHex(entry[jss::index].asString())) { JLOG(m_journal.fatal()) << "Invalid entry in ledger"; return nullptr; @@ -1906,8 +1906,7 @@ ApplicationImp::loadLedgerFromFile(std::string const& name) } } - loadLedger->stateMap().flushDirty( - hotACCOUNT_NODE, loadLedger->info().seq); + loadLedger->stateMap().flushDirty(hotACCOUNT_NODE); loadLedger->setAccepted( closeTime, closeTimeResolution, !closeTimeEstimated, *config_); @@ -1940,7 +1939,7 @@ ApplicationImp::loadOldLedger( { uint256 hash; - if (hash.SetHex(ledgerID)) + if (hash.parseHex(ledgerID)) { loadLedger = loadByHash(hash, *this); @@ -2043,9 +2042,9 @@ ApplicationImp::loadOldLedger( return false; } - if (!loadLedger->assertSane(journal("Ledger"))) + if (!loadLedger->assertSensible(journal("Ledger"))) { - JLOG(m_journal.fatal()) << "Ledger is not sane."; + JLOG(m_journal.fatal()) << "Ledger is not sensible."; assert(false); return false; } diff --git a/src/ripple/app/misc/AmendmentTable.h b/src/ripple/app/misc/AmendmentTable.h index bcd21f763b5..e0884fecbbf 100644 --- a/src/ripple/app/misc/AmendmentTable.h +++ b/src/ripple/app/misc/AmendmentTable.h @@ -155,10 +155,9 @@ class AmendmentTable amendTx.add(s); initialPosition->addGiveItem( + SHAMapNodeType::tnTRANSACTION_NM, std::make_shared( - amendTx.getTransactionID(), s.peekData()), - true, - false); + amendTx.getTransactionID(), s.peekData())); } } }; diff --git a/src/ripple/app/misc/FeeVoteImpl.cpp b/src/ripple/app/misc/FeeVoteImpl.cpp index e2dc2e40712..ef2ffecc517 100644 --- a/src/ripple/app/misc/FeeVoteImpl.cpp +++ b/src/ripple/app/misc/FeeVoteImpl.cpp @@ -244,9 +244,9 @@ FeeVoteImpl::doVoting( Serializer s; feeTx.add(s); - auto tItem = std::make_shared(txID, s.peekData()); - - if (!initialPosition->addGiveItem(std::move(tItem), true, false)) + if (!initialPosition->addGiveItem( + SHAMapNodeType::tnTRANSACTION_NM, + std::make_shared(txID, s.peekData()))) { JLOG(journal_.warn()) << "Ledger already had fee change"; } diff --git a/src/ripple/app/misc/HashRouter.h b/src/ripple/app/misc/HashRouter.h index 76cf4431ec7..ba2f9436dd8 100644 --- a/src/ripple/app/misc/HashRouter.h +++ b/src/ripple/app/misc/HashRouter.h @@ -61,12 +61,6 @@ class HashRouter class Entry : public CountedObject { public: - static char const* - getCountedObjectName() - { - return "HashRouterEntry"; - } - Entry() { } diff --git a/src/ripple/app/misc/Manifest.h b/src/ripple/app/misc/Manifest.h index 52bebb1ddf0..532cb88d7b8 100644 --- a/src/ripple/app/misc/Manifest.h +++ b/src/ripple/app/misc/Manifest.h @@ -219,6 +219,8 @@ class ManifestCache /** Master public keys stored by current ephemeral public key. */ hash_map signingToMasterKeys_; + std::atomic seq_{0}; + public: explicit ManifestCache( beast::Journal j = beast::Journal(beast::Journal::getNullSink())) @@ -226,6 +228,13 @@ class ManifestCache { } + /** A monotonically increasing number used to detect new manifests. */ + std::uint32_t + sequence() const + { + return seq_.load(); + } + /** Returns master key's current signing key. @param pk Master public key diff --git a/src/ripple/app/misc/NegativeUNLVote.cpp b/src/ripple/app/misc/NegativeUNLVote.cpp index 3e502fc81fa..6b02bc3216e 100644 --- a/src/ripple/app/misc/NegativeUNLVote.cpp +++ b/src/ripple/app/misc/NegativeUNLVote.cpp @@ -119,7 +119,8 @@ NegativeUNLVote::addTx( Serializer s; negUnlTx.add(s); if (!initialSet->addGiveItem( - std::make_shared(txID, s.peekData()), true, false)) + SHAMapNodeType::tnTRANSACTION_NM, + std::make_shared(txID, s.peekData()))) { JLOG(j_.warn()) << "N-UNL: ledger seq=" << seq << ", add ttUNL_MODIFY tx failed"; diff --git a/src/ripple/app/misc/NetworkOPs.cpp b/src/ripple/app/misc/NetworkOPs.cpp index bc4bdd9d82e..3d4ff057e00 100644 --- a/src/ripple/app/misc/NetworkOPs.cpp +++ b/src/ripple/app/misc/NetworkOPs.cpp @@ -2229,8 +2229,7 @@ NetworkOPsImp::transactionsSQL( boost::format("SELECT %s FROM AccountTransactions " "WHERE Account = '%s' %s %s LIMIT %u, %u;") % selection % app_.accountIDCache().toBase58(account) % maxClause % - minClause % beast::lexicalCastThrow(offset) % - beast::lexicalCastThrow(numberOfResults)); + minClause % offset % numberOfResults); else sql = boost::str( boost::format( @@ -2244,8 +2243,7 @@ NetworkOPsImp::transactionsSQL( selection % app_.accountIDCache().toBase58(account) % maxClause % minClause % (descending ? "DESC" : "ASC") % (descending ? "DESC" : "ASC") % (descending ? "DESC" : "ASC") % - beast::lexicalCastThrow(offset) % - beast::lexicalCastThrow(numberOfResults)); + offset % numberOfResults); JLOG(m_journal.trace()) << "txSQL query: " << sql; return sql; } diff --git a/src/ripple/app/misc/SHAMapStoreImp.cpp b/src/ripple/app/misc/SHAMapStoreImp.cpp index 3e96bbc2741..887bb580c25 100644 --- a/src/ripple/app/misc/SHAMapStoreImp.cpp +++ b/src/ripple/app/misc/SHAMapStoreImp.cpp @@ -299,12 +299,10 @@ SHAMapStoreImp::fdRequired() const } bool -SHAMapStoreImp::copyNode( - std::uint64_t& nodeCount, - SHAMapAbstractNode const& node) +SHAMapStoreImp::copyNode(std::uint64_t& nodeCount, SHAMapTreeNode const& node) { // Copy a single record from node to dbRotating_ - dbRotating_->fetchNodeObject(node.getNodeHash().as_uint256()); + dbRotating_->fetchNodeObject(node.getHash().as_uint256()); if (!(++nodeCount % checkHealthInterval_)) { if (health()) diff --git a/src/ripple/app/misc/SHAMapStoreImp.h b/src/ripple/app/misc/SHAMapStoreImp.h index ea75eedb9c5..541d74a38ee 100644 --- a/src/ripple/app/misc/SHAMapStoreImp.h +++ b/src/ripple/app/misc/SHAMapStoreImp.h @@ -194,7 +194,7 @@ class SHAMapStoreImp : public SHAMapStore private: // callback for visitNodes bool - copyNode(std::uint64_t& nodeCount, SHAMapAbstractNode const& node); + copyNode(std::uint64_t& nodeCount, SHAMapTreeNode const& node); void run(); void diff --git a/src/ripple/app/misc/Transaction.h b/src/ripple/app/misc/Transaction.h index 4dbe2f06b87..79da8b22445 100644 --- a/src/ripple/app/misc/Transaction.h +++ b/src/ripple/app/misc/Transaction.h @@ -61,12 +61,6 @@ class Transaction : public std::enable_shared_from_this, public CountedObject { public: - static char const* - getCountedObjectName() - { - return "Transaction"; - } - using pointer = std::shared_ptr; using ref = const pointer&; diff --git a/src/ripple/app/misc/ValidatorList.h b/src/ripple/app/misc/ValidatorList.h index 4a6a010073c..69352652096 100644 --- a/src/ripple/app/misc/ValidatorList.h +++ b/src/ripple/app/misc/ValidatorList.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -139,7 +140,7 @@ class ValidatorList TimeKeeper& timeKeeper_; boost::filesystem::path const dataPath_; beast::Journal const j_; - std::shared_timed_mutex mutable mutex_; + boost::shared_mutex mutable mutex_; std::atomic quorum_; boost::optional minimumQuorum_; @@ -505,7 +506,7 @@ class ValidatorList QuorumKeys getQuorumKeys() const { - std::shared_lock read_lock{mutex_}; + std::shared_lock read_lock{mutex_}; return {quorum_, trustedSigningKeys_}; } diff --git a/src/ripple/app/misc/impl/AmendmentTable.cpp b/src/ripple/app/misc/impl/AmendmentTable.cpp index 2f29a2f5788..68ad92b204b 100644 --- a/src/ripple/app/misc/impl/AmendmentTable.cpp +++ b/src/ripple/app/misc/impl/AmendmentTable.cpp @@ -56,7 +56,7 @@ parseSection(Section const& section) uint256 id; - if (!id.SetHexExact(match[1])) + if (!id.parseHex(match[1])) Throw( "Invalid amendment ID '" + match[1] + "' in [" + section.name() + "]"); diff --git a/src/ripple/app/misc/impl/Manifest.cpp b/src/ripple/app/misc/impl/Manifest.cpp index 319d09b2c18..aa2f99e0198 100644 --- a/src/ripple/app/misc/impl/Manifest.cpp +++ b/src/ripple/app/misc/impl/Manifest.cpp @@ -22,12 +22,12 @@ #include #include #include -#include #include #include #include #include #include +#include #include #include #include @@ -235,7 +235,7 @@ loadValidatorToken(std::vector const& blob) })); for (auto const& line : blob) - tokenStr += beast::rfc2616::trim(line); + tokenStr += boost::algorithm::trim_copy(line); tokenStr = base64_decode(tokenStr); @@ -431,6 +431,9 @@ ManifestCache::applyManifest(Manifest m) iter->second = std::move(m); } + // Something has changed. Keep track of it. + seq_++; + return ManifestDisposition::accepted; } @@ -506,7 +509,7 @@ ManifestCache::load( })); for (auto const& line : configRevocation) - revocationStr += beast::rfc2616::trim(line); + revocationStr += boost::algorithm::trim_copy(line); auto mo = deserializeManifest(base64_decode(revocationStr)); diff --git a/src/ripple/app/misc/impl/ValidatorList.cpp b/src/ripple/app/misc/impl/ValidatorList.cpp index 4742fb733d4..b50238b3caa 100644 --- a/src/ripple/app/misc/impl/ValidatorList.cpp +++ b/src/ripple/app/misc/impl/ValidatorList.cpp @@ -96,7 +96,7 @@ ValidatorList::load( ")?" // end optional comment block ); - std::unique_lock read_lock{mutex_}; + std::unique_lock lock{mutex_}; JLOG(j_.debug()) << "Loading configured trusted validator list publisher keys"; @@ -306,7 +306,7 @@ ValidatorList::applyList( if (version != requiredListVersion) return PublisherListStats{ListDisposition::unsupported_version}; - std::unique_lock lock{mutex_}; + std::unique_lock lock{mutex_}; Json::Value list; PublicKey pubKey; @@ -441,7 +441,7 @@ ValidatorList::loadLists() using namespace boost::filesystem; using namespace boost::system::errc; - std::unique_lock lock{mutex_}; + std::unique_lock lock{mutex_}; std::vector sites; sites.reserve(publisherLists_.size()); @@ -551,7 +551,7 @@ ValidatorList::verify( bool ValidatorList::listed(PublicKey const& identity) const { - std::shared_lock read_lock{mutex_}; + std::shared_lock read_lock{mutex_}; auto const pubKey = validatorManifests_.getMasterKey(identity); return keyListings_.find(pubKey) != keyListings_.end(); @@ -560,7 +560,7 @@ ValidatorList::listed(PublicKey const& identity) const bool ValidatorList::trusted(PublicKey const& identity) const { - std::shared_lock read_lock{mutex_}; + std::shared_lock read_lock{mutex_}; auto const pubKey = validatorManifests_.getMasterKey(identity); return trustedMasterKeys_.find(pubKey) != trustedMasterKeys_.end(); @@ -569,7 +569,7 @@ ValidatorList::trusted(PublicKey const& identity) const boost::optional ValidatorList::getListedKey(PublicKey const& identity) const { - std::shared_lock read_lock{mutex_}; + std::shared_lock read_lock{mutex_}; auto const pubKey = validatorManifests_.getMasterKey(identity); if (keyListings_.find(pubKey) != keyListings_.end()) @@ -580,7 +580,7 @@ ValidatorList::getListedKey(PublicKey const& identity) const boost::optional ValidatorList::getTrustedKey(PublicKey const& identity) const { - std::shared_lock read_lock{mutex_}; + std::shared_lock read_lock{mutex_}; auto const pubKey = validatorManifests_.getMasterKey(identity); if (trustedMasterKeys_.find(pubKey) != trustedMasterKeys_.end()) @@ -591,14 +591,14 @@ ValidatorList::getTrustedKey(PublicKey const& identity) const bool ValidatorList::trustedPublisher(PublicKey const& identity) const { - std::shared_lock read_lock{mutex_}; + std::shared_lock read_lock{mutex_}; return identity.size() && publisherLists_.count(identity); } PublicKey ValidatorList::localPublicKey() const { - std::shared_lock read_lock{mutex_}; + std::shared_lock read_lock{mutex_}; return localPubKey_; } @@ -633,14 +633,14 @@ ValidatorList::removePublisherList(PublicKey const& publisherKey) std::size_t ValidatorList::count() const { - std::shared_lock read_lock{mutex_}; + std::shared_lock read_lock{mutex_}; return publisherLists_.size(); } boost::optional ValidatorList::expires() const { - std::shared_lock read_lock{mutex_}; + std::shared_lock read_lock{mutex_}; boost::optional res{boost::none}; for (auto const& p : publisherLists_) { @@ -660,7 +660,7 @@ ValidatorList::getJson() const { Json::Value res(Json::objectValue); - std::shared_lock read_lock{mutex_}; + std::shared_lock read_lock{mutex_}; res[jss::validation_quorum] = static_cast(quorum()); @@ -764,7 +764,7 @@ void ValidatorList::for_each_listed( std::function func) const { - std::shared_lock read_lock{mutex_}; + std::shared_lock read_lock{mutex_}; for (auto const& v : keyListings_) func(v.first, trusted(v.first)); @@ -780,7 +780,7 @@ ValidatorList::for_each_available(std::function func) const { - std::shared_lock read_lock{mutex_}; + std::shared_lock read_lock{mutex_}; for (auto const& [key, pl] : publisherLists_) { @@ -800,7 +800,7 @@ ValidatorList::for_each_available(std::function ValidatorList::getAvailable(boost::beast::string_view const& pubKey) { - std::shared_lock read_lock{mutex_}; + std::shared_lock read_lock{mutex_}; auto const keyBlob = strViewUnHex(pubKey); @@ -896,7 +896,7 @@ ValidatorList::calculateQuorum( TrustChanges ValidatorList::updateTrusted(hash_set const& seenValidators) { - std::unique_lock lock{mutex_}; + std::unique_lock lock{mutex_}; // Remove any expired published lists for (auto const& list : publisherLists_) @@ -983,14 +983,14 @@ ValidatorList::updateTrusted(hash_set const& seenValidators) hash_set ValidatorList::getTrustedMasterKeys() const { - std::shared_lock lock{mutex_}; + std::shared_lock read_lock{mutex_}; return trustedMasterKeys_; } hash_set ValidatorList::getNegativeUNL() const { - std::shared_lock lock{mutex_}; + std::shared_lock read_lock{mutex_}; return negativeUNL_; } @@ -1008,7 +1008,7 @@ ValidatorList::negativeUNLFilter( // Remove validations that are from validators on the negative UNL. auto ret = std::move(validations); - std::shared_lock lock{mutex_}; + std::shared_lock read_lock{mutex_}; if (!negativeUNL_.empty()) { ret.erase( diff --git a/src/ripple/app/paths/PathRequest.h b/src/ripple/app/paths/PathRequest.h index e0b9f37cdbf..4cba34fbad6 100644 --- a/src/ripple/app/paths/PathRequest.h +++ b/src/ripple/app/paths/PathRequest.h @@ -49,12 +49,6 @@ class PathRequest : public std::enable_shared_from_this, public CountedObject { public: - static char const* - getCountedObjectName() - { - return "PathRequest"; - } - using wptr = std::weak_ptr; using pointer = std::shared_ptr; using ref = const pointer&; @@ -112,8 +106,6 @@ class PathRequest : public std::enable_shared_from_this, private: bool isValid(std::shared_ptr const& crCache); - void - setValid(); std::unique_ptr const& getPathFinder( diff --git a/src/ripple/app/paths/impl/DirectStep.cpp b/src/ripple/app/paths/impl/DirectStep.cpp index 760029a8612..0056f3bab6c 100644 --- a/src/ripple/app/paths/impl/DirectStep.cpp +++ b/src/ripple/app/paths/impl/DirectStep.cpp @@ -337,7 +337,7 @@ DirectIPaymentStep::quality(ReadView const& sb, QualityDirection qDir) const if (!sle) return QUALITY_ONE; - auto const& field = [this, qDir]() -> SF_U32 const& { + auto const& field = [this, qDir]() -> SF_UINT32 const& { if (qDir == QualityDirection::in) { // compute dst quality in diff --git a/src/ripple/app/tx/impl/PayChan.cpp b/src/ripple/app/tx/impl/PayChan.cpp index ebbd0064d7c..654cf90e198 100644 --- a/src/ripple/app/tx/impl/PayChan.cpp +++ b/src/ripple/app/tx/impl/PayChan.cpp @@ -326,7 +326,7 @@ PayChanFund::preflight(PreflightContext const& ctx) TER PayChanFund::doApply() { - Keylet const k(ltPAYCHAN, ctx_.tx[sfPayChannel]); + Keylet const k(ltPAYCHAN, ctx_.tx[sfChannel]); auto const slep = ctx_.view().peek(k); if (!slep) return tecNO_ENTRY; @@ -441,7 +441,7 @@ PayChanClaim::preflight(PreflightContext const& ctx) if (reqBalance > authAmt) return temBAD_AMOUNT; - Keylet const k(ltPAYCHAN, ctx.tx[sfPayChannel]); + Keylet const k(ltPAYCHAN, ctx.tx[sfChannel]); if (!publicKeyType(ctx.tx[sfPublicKey])) return temMALFORMED; @@ -458,7 +458,7 @@ PayChanClaim::preflight(PreflightContext const& ctx) TER PayChanClaim::doApply() { - Keylet const k(ltPAYCHAN, ctx_.tx[sfPayChannel]); + Keylet const k(ltPAYCHAN, ctx_.tx[sfChannel]); auto const slep = ctx_.view().peek(k); if (!slep) return tecNO_TARGET; diff --git a/src/ripple/basics/CountedObject.h b/src/ripple/basics/CountedObject.h index cf57b2a3095..a5bccf89ec5 100644 --- a/src/ripple/basics/CountedObject.h +++ b/src/ripple/basics/CountedObject.h @@ -21,6 +21,7 @@ #define RIPPLE_BASICS_COUNTEDOBJECT_H_INCLUDED #include +#include #include #include #include @@ -45,47 +46,60 @@ class CountedObjects @internal */ - class CounterBase + class Counter { public: - CounterBase() noexcept; + Counter(std::string name) noexcept : name_(std::move(name)), count_(0) + { + // Insert ourselves at the front of the lock-free linked list + CountedObjects& instance = CountedObjects::getInstance(); + Counter* head; + + do + { + head = instance.m_head.load(); + next_ = head; + } while (instance.m_head.exchange(this) != head); + + ++instance.m_count; + } - virtual ~CounterBase() noexcept; + ~Counter() noexcept = default; int increment() noexcept { - return ++m_count; + return ++count_; } int decrement() noexcept { - return --m_count; + return --count_; } int getCount() const noexcept { - return m_count.load(); + return count_.load(); } - CounterBase* + Counter* getNext() const noexcept { - return m_next; + return next_; } - virtual char const* - getName() const = 0; + std::string const& + getName() const noexcept + { + return name_; + } private: - virtual void - checkPureVirtual() const = 0; - - protected: - std::atomic m_count; - CounterBase* m_next; + std::string const name_; + std::atomic count_; + Counter* next_; }; private: @@ -94,7 +108,7 @@ class CountedObjects private: std::atomic m_count; - std::atomic m_head; + std::atomic m_head; }; //------------------------------------------------------------------------------ @@ -109,6 +123,14 @@ class CountedObjects template class CountedObject { +private: + static auto& + getCounter() noexcept + { + static CountedObjects::Counter c{beast::type_name()}; + return c; + } + public: CountedObject() noexcept { @@ -127,35 +149,6 @@ class CountedObject { getCounter().decrement(); } - -private: - class Counter : public CountedObjects::CounterBase - { - public: - Counter() noexcept - { - } - - char const* - getName() const override - { - return Object::getCountedObjectName(); - } - - void - checkPureVirtual() const override - { - } - }; - -private: - static Counter& - getCounter() noexcept - { - static_assert(std::is_nothrow_constructible{}, ""); - static Counter c; - return c; - } }; } // namespace ripple diff --git a/src/ripple/basics/StringUtilities.h b/src/ripple/basics/StringUtilities.h index 9cedbbe175a..759d407361c 100644 --- a/src/ripple/basics/StringUtilities.h +++ b/src/ripple/basics/StringUtilities.h @@ -30,42 +30,19 @@ #include namespace ripple { -inline static std::string -sqlEscape(std::string const& strSrc) -{ - static boost::format f("X'%s'"); - return str(boost::format(f) % strHex(strSrc)); -} - -inline static std::string -sqlEscape(Blob const& vecSrc) -{ - size_t size = vecSrc.size(); - if (size == 0) - return "X''"; +/** Format arbitrary binary data as an SQLite "blob literal". - std::string j(size * 2 + 3, 0); + In SQLite, blob literals must be encoded when used in a query. Per + https://sqlite.org/lang_expr.html#literal_values_constants_ they are + encoded as string literals containing hexadecimal data and preceded + by a single 'X' character. - unsigned char* oPtr = reinterpret_cast(&*j.begin()); - const unsigned char* iPtr = &vecSrc[0]; - - *oPtr++ = 'X'; - *oPtr++ = '\''; - - for (int i = size; i != 0; --i) - { - unsigned char c = *iPtr++; - *oPtr++ = charHex(c >> 4); - *oPtr++ = charHex(c & 15); - } - - *oPtr++ = '\''; - return j; -} - -uint64_t -uintFromHex(std::string const& strSrc); + @param blob An arbitrary blob of binary data + @return The input, encoded as a blob literal. + */ +std::string +sqlBlobLiteral(Blob const& blob); template boost::optional diff --git a/src/ripple/basics/XRPAmount.h b/src/ripple/basics/XRPAmount.h index a9611814d8b..e8f26266e94 100644 --- a/src/ripple/basics/XRPAmount.h +++ b/src/ripple/basics/XRPAmount.h @@ -22,7 +22,6 @@ #include #include -#include #include #include @@ -32,6 +31,7 @@ #include #include +#include namespace ripple { diff --git a/src/ripple/basics/base_uint.h b/src/ripple/basics/base_uint.h index 56c42665840..3ada512b51c 100644 --- a/src/ripple/basics/base_uint.h +++ b/src/ripple/basics/base_uint.h @@ -27,7 +27,6 @@ #include #include -#include #include #include #include @@ -57,8 +56,19 @@ struct is_contiguous_container< } // namespace detail -// This class stores its values internally in big-endian form +/** Integers of any length that is a multiple of 32-bits + @note This class stores its values internally in big-endian + form and that internal representation is part of the + binary protocol of the XRP Ledger and cannot be changed + arbitrarily without causing breakage. + + @tparam Bits The number of bits this integer should have; must + be at least 64 and a multiple of 32. + @tparam Tag An arbitrary type that functions as a tag and allows + the instantiation of "distinct" types that the same + number of bits. + */ template class base_uint { @@ -363,99 +373,53 @@ class base_uint } /** Parse a hex string into a base_uint - The string must contain exactly bytes * 2 hex characters and must not - have any leading or trailing whitespace. - */ - bool - SetHexExact(const char* psz) - { - unsigned char* pOut = begin(); - - for (int i = 0; i < sizeof(data_); ++i) - { - auto hi = charUnHex(*psz++); - if (hi == -1) - return false; - - auto lo = charUnHex(*psz++); - if (lo == -1) - return false; - - *pOut++ = (hi << 4) | lo; - } - - // We've consumed exactly as many bytes as we needed at this point - // so we should be at the end of the string. - return (*psz == 0); - } - - /** Parse a hex string into a base_uint - The input can be: - - shorter than the full hex representation by not including leading - zeroes. - - longer than the full hex representation in which case leading - bytes are discarded. - When finished parsing, the string must be fully consumed with only a - null terminator remaining. + The input must be precisely `2 * bytes` hexadecimal characters + long, with one exception: the value '0'. - When bStrict is false, the parsing is done in non-strict mode, and, if - present, leading whitespace and the 0x prefix will be skipped. - */ - bool - SetHex(const char* psz, bool bStrict = false) + @param sv A null-terminated string of hexadecimal characters + @return true if the input was parsed properly; false otherwise. + */ + [[nodiscard]] bool + parseHex(std::string_view sv) { - // Find beginning. - auto pBegin = reinterpret_cast(psz); - // skip leading spaces - if (!bStrict) - while (isspace(*pBegin)) - pBegin++; - - // skip 0x - if (!bStrict && pBegin[0] == '0' && tolower(pBegin[1]) == 'x') - pBegin += 2; - - // Find end. - auto pEnd = pBegin; - while (charUnHex(*pEnd) != -1) - pEnd++; - - // Take only last digits of over long string. - if ((unsigned int)(pEnd - pBegin) > 2 * size()) - pBegin = pEnd - 2 * size(); + if (sv == "0") + { + zero(); + return true; + } - unsigned char* pOut = end() - ((pEnd - pBegin + 1) / 2); + if (sv.size() != bytes * 2) + return false; - *this = beast::zero; + auto out = data(); - if ((pEnd - pBegin) & 1) - *pOut++ = charUnHex(*pBegin++); + auto in = sv.begin(); - while (pBegin != pEnd) + while (in != sv.end()) { - auto cHigh = charUnHex(*pBegin++); - auto cLow = pBegin == pEnd ? 0 : charUnHex(*pBegin++); + auto const hi = charUnHex(*in++); + auto const lo = charUnHex(*in++); - if (cHigh == -1 || cLow == -1) + if (hi == -1 || lo == -1) return false; - *pOut++ = (cHigh << 4) | cLow; + *out++ = static_cast((hi << 4) + lo); } - return !*pEnd; + return true; } - bool - SetHex(std::string const& str, bool bStrict = false) + [[nodiscard]] bool + parseHex(const char* str) { - return SetHex(str.c_str(), bStrict); + return parseHex(std::string_view{str}); } - bool - SetHexExact(std::string const& str) + [[nodiscard]] bool + parseHex(std::string const& str) { - return SetHexExact(str.c_str()); + return parseHex(std::string_view{str}); } constexpr static std::size_t @@ -603,31 +567,6 @@ to_string(base_uint const& a) return strHex(a.cbegin(), a.cend()); } -// Function templates that return a base_uint given text in hexadecimal. -// Invoke like: -// auto i = from_hex_text("AAAAA"); -template -auto -from_hex_text(char const* text) -> std::enable_if_t< - std::is_same>::value, - T> -{ - T ret; - ret.SetHex(text); - return ret; -} - -template -auto -from_hex_text(std::string const& text) -> std::enable_if_t< - std::is_same>::value, - T> -{ - T ret; - ret.SetHex(text); - return ret; -} - template inline std::ostream& operator<<(std::ostream& out, base_uint const& u) diff --git a/src/ripple/basics/impl/CountedObject.cpp b/src/ripple/basics/impl/CountedObject.cpp index da42bf5b491..5b25091632d 100644 --- a/src/ripple/basics/impl/CountedObject.cpp +++ b/src/ripple/basics/impl/CountedObject.cpp @@ -18,6 +18,7 @@ //============================================================================== #include +#include #include namespace ripple { @@ -41,53 +42,17 @@ CountedObjects::getCounts(int minimumThreshold) const // When other operations are concurrent, the count // might be temporarily less than the actual count. - int const count = m_count.load(); + counts.reserve(m_count.load()); - counts.reserve(count); - - CounterBase* counter = m_head.load(); - - while (counter != nullptr) + for (auto* ctr = m_head.load(); ctr != nullptr; ctr = ctr->getNext()) { - if (counter->getCount() >= minimumThreshold) - { - Entry entry; - - entry.first = counter->getName(); - entry.second = counter->getCount(); - - counts.push_back(entry); - } - - counter = counter->getNext(); + if (ctr->getCount() >= minimumThreshold) + counts.emplace_back(ctr->getName(), ctr->getCount()); } - return counts; -} - -//------------------------------------------------------------------------------ - -CountedObjects::CounterBase::CounterBase() noexcept : m_count(0) -{ - // Insert ourselves at the front of the lock-free linked list - - CountedObjects& instance = CountedObjects::getInstance(); - CounterBase* head; - - do - { - head = instance.m_head.load(); - m_next = head; - } while (instance.m_head.exchange(this) != head); + std::sort(counts.begin(), counts.end()); - ++instance.m_count; -} - -CountedObjects::CounterBase::~CounterBase() noexcept -{ - // VFALCO NOTE If the counters are destroyed before the singleton, - // undefined behavior will result if the singleton's member - // functions are called. + return counts; } } // namespace ripple diff --git a/src/ripple/basics/impl/StringUtilities.cpp b/src/ripple/basics/impl/StringUtilities.cpp index d76bd79b6bd..346e94c7560 100644 --- a/src/ripple/basics/impl/StringUtilities.cpp +++ b/src/ripple/basics/impl/StringUtilities.cpp @@ -26,29 +26,21 @@ #include #include #include -#include namespace ripple { -uint64_t -uintFromHex(std::string const& strSrc) +std::string +sqlBlobLiteral(Blob const& blob) { - uint64_t uValue(0); - - if (strSrc.size() > 16) - Throw("overlong 64-bit value"); - - for (auto c : strSrc) - { - int ret = charUnHex(c); - - if (ret == -1) - Throw("invalid hex digit"); + std::string j; - uValue = (uValue << 4) | ret; - } + j.reserve(blob.size() * 2 + 3); + j.push_back('X'); + j.push_back('\''); + boost::algorithm::hex(blob.begin(), blob.end(), std::back_inserter(j)); + j.push_back('\''); - return uValue; + return j; } bool diff --git a/src/ripple/basics/impl/strHex.cpp b/src/ripple/basics/impl/strHex.cpp index 66aa8949f0b..084493af53a 100644 --- a/src/ripple/basics/impl/strHex.cpp +++ b/src/ripple/basics/impl/strHex.cpp @@ -18,37 +18,30 @@ //============================================================================== #include -#include -#include +#include namespace ripple { int charUnHex(unsigned char c) { - struct HexTab - { - int hex[256]; + static constexpr std::array const xtab = []() { + std::array t{}; - HexTab() - { - std::fill(std::begin(hex), std::end(hex), -1); - for (int i = 0; i < 10; ++i) - hex['0' + i] = i; - for (int i = 0; i < 6; ++i) - { - hex['A' + i] = 10 + i; - hex['a' + i] = 10 + i; - } - } - int - operator[](unsigned char c) const + for (auto& x : t) + x = -1; + + for (int i = 0; i < 10; ++i) + t['0' + i] = i; + + for (int i = 0; i < 6; ++i) { - return hex[c]; + t['A' + i] = 10 + i; + t['a' + i] = 10 + i; } - }; - static HexTab xtab; + return t; + }(); return xtab[c]; } diff --git a/src/ripple/basics/random.h b/src/ripple/basics/random.h index 7ea8972eab0..3af8be54109 100644 --- a/src/ripple/basics/random.h +++ b/src/ripple/basics/random.h @@ -20,7 +20,6 @@ #ifndef RIPPLE_BASICS_RANDOM_H_INCLUDED #define RIPPLE_BASICS_RANDOM_H_INCLUDED -#include // #include #include #include @@ -29,6 +28,7 @@ #include #include #include +#include namespace ripple { diff --git a/src/ripple/basics/safe_cast.h b/src/ripple/basics/safe_cast.h index d4bbefec300..e52bceca759 100644 --- a/src/ripple/basics/safe_cast.h +++ b/src/ripple/basics/safe_cast.h @@ -20,7 +20,6 @@ #ifndef RIPPLE_BASICS_SAFE_CAST_H_INCLUDED #define RIPPLE_BASICS_SAFE_CAST_H_INCLUDED -#include #include namespace ripple { diff --git a/src/ripple/basics/strHex.h b/src/ripple/basics/strHex.h index f3618a359d4..e48ea921557 100644 --- a/src/ripple/basics/strHex.h +++ b/src/ripple/basics/strHex.h @@ -17,36 +17,14 @@ */ //============================================================================== -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2011 The Bitcoin developers -// Distributed under the MIT/X11 software license, see the accompanying -// file license.txt or http://www.opensource.org/licenses/mit-license.php. - #ifndef RIPPLE_BASICS_STRHEX_H_INCLUDED #define RIPPLE_BASICS_STRHEX_H_INCLUDED -#include -#include - #include #include namespace ripple { -/** Converts an integer to the corresponding hex digit - @param iDigit 0-15 inclusive - @return a character from '0'-'9' or 'A'-'F'. -*/ -inline char -charHex(unsigned int digit) -{ - static char const xtab[] = "0123456789ABCDEF"; - - assert(digit < 16); - - return xtab[digit]; -} - /** @{ */ /** Converts a hex digit to the corresponding integer @param cDigit one of '0'-'9', 'A'-'F' or 'a'-'f' @@ -84,15 +62,6 @@ strHex(T const& from) return strHex(from.begin(), from.end()); } -inline std::string -strHex(const std::uint64_t uiHost) -{ - uint64_t uBig = boost::endian::native_to_big(uiHost); - - auto const begin = (unsigned char*)&uBig; - auto const end = begin + sizeof(uBig); - return strHex(begin, end); -} } // namespace ripple #endif diff --git a/src/ripple/beast/asio/io_latency_probe.h b/src/ripple/beast/asio/io_latency_probe.h index b18fb823e30..bbde13af687 100644 --- a/src/ripple/beast/asio/io_latency_probe.h +++ b/src/ripple/beast/asio/io_latency_probe.h @@ -22,7 +22,6 @@ #include #include -#include #include #include #include @@ -137,12 +136,7 @@ class io_latency_probe } if (wait) -#ifdef BOOST_NO_CXX11_LAMBDAS - while (m_count != 0) - m_cond.wait(lock); -#else m_cond.wait(lock, [this] { return this->m_count == 0; }); -#endif } void diff --git a/src/ripple/beast/container/detail/aged_ordered_container.h b/src/ripple/beast/container/detail/aged_ordered_container.h index a6cafa89737..ed6585dd501 100644 --- a/src/ripple/beast/container/detail/aged_ordered_container.h +++ b/src/ripple/beast/container/detail/aged_ordered_container.h @@ -25,7 +25,6 @@ #include #include #include -#include // #include #include #include @@ -34,6 +33,7 @@ #include #include #include +#include #include namespace beast { diff --git a/src/ripple/beast/crypto/detail/mac_facade.h b/src/ripple/beast/crypto/detail/mac_facade.h deleted file mode 100644 index f58c2fa2340..00000000000 --- a/src/ripple/beast/crypto/detail/mac_facade.h +++ /dev/null @@ -1,82 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of Beast: https://github.com/vinniefalco/Beast - Copyright 2013, Vinnie Falco - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef BEAST_CRYPTO_MAC_FACADE_H_INCLUDED -#define BEAST_CRYPTO_MAC_FACADE_H_INCLUDED - -#include -#include -#include -#include - -namespace beast { -namespace detail { - -// Message Authentication Code (MAC) facade -template -class mac_facade -{ -private: - Context ctx_; - -public: - static beast::endian const endian = beast::endian::native; - - static std::size_t const digest_size = Context::digest_size; - - using result_type = std::array; - - mac_facade() noexcept - { - init(ctx_); - } - - ~mac_facade() - { - erase(std::integral_constant{}); - } - - void - operator()(void const* data, std::size_t size) noexcept - { - update(ctx_, data, size); - } - - explicit operator result_type() noexcept - { - result_type digest; - finish(ctx_, &digest[0]); - return digest; - } - -private: - inline void erase(std::false_type) noexcept - { - } - - inline void erase(std::true_type) noexcept - { - secure_erase(&ctx_, sizeof(ctx_)); - } -}; - -} // namespace detail -} // namespace beast - -#endif diff --git a/src/ripple/beast/crypto/detail/ripemd_context.h b/src/ripple/beast/crypto/detail/ripemd_context.h deleted file mode 100644 index 8eb7a573dd5..00000000000 --- a/src/ripple/beast/crypto/detail/ripemd_context.h +++ /dev/null @@ -1,449 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of Beast: https://github.com/vinniefalco/Beast - Copyright 2013, Vinnie Falco - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef BEAST_CRYPTO_RIPEMD_CONTEXT_H_INCLUDED -#define BEAST_CRYPTO_RIPEMD_CONTEXT_H_INCLUDED - -#include -#include -#include - -namespace beast { -namespace detail { - -// Based on -// https://code.google.com/p/blockchain/source/browse/trunk/RIPEMD160.cpp -/* - Copyright (c) Katholieke Universiteit Leuven - 1996 All Rights Reserved - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to - deal in the Software without restriction, including without limitation the - rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - sell copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - IN THE SOFTWARE. -*/ -// This implementation has been modified from the -// original. It has been updated for C++11. - -struct ripemd160_context -{ - explicit ripemd160_context() = default; - - static unsigned int const block_size = 64; - static unsigned int const digest_size = 20; - - unsigned int tot_len; - unsigned int len; - unsigned char block[256]; - std::uint32_t h[5]; -}; - -// ROL(x, n) cyclically rotates x over n bits to the left -// x must be of an unsigned 32 bits type and 0 <= n < 32. -#define BEAST_RIPEMD_ROL(x, n) (((x) << (n)) | ((x) >> (32 - (n)))) - -// the five basic functions F(), G() and H() -#define BEAST_RIPEMD_F(x, y, z) ((x) ^ (y) ^ (z)) -#define BEAST_RIPEMD_G(x, y, z) (((x) & (y)) | (~(x) & (z))) -#define BEAST_RIPEMD_H(x, y, z) (((x) | ~(y)) ^ (z)) -#define BEAST_RIPEMD_I(x, y, z) (((x) & (z)) | ((y) & ~(z))) -#define BEAST_RIPEMD_J(x, y, z) ((x) ^ ((y) | ~(z))) - -// the ten basic operations FF() through III() -#define BEAST_RIPEMD_FF(a, b, c, d, e, x, s) \ - { \ - (a) += BEAST_RIPEMD_F((b), (c), (d)) + (x); \ - (a) = BEAST_RIPEMD_ROL((a), (s)) + (e); \ - (c) = BEAST_RIPEMD_ROL((c), 10); \ - } -#define BEAST_RIPEMD_GG(a, b, c, d, e, x, s) \ - { \ - (a) += BEAST_RIPEMD_G((b), (c), (d)) + (x) + 0x5a827999UL; \ - (a) = BEAST_RIPEMD_ROL((a), (s)) + (e); \ - (c) = BEAST_RIPEMD_ROL((c), 10); \ - } -#define BEAST_RIPEMD_HH(a, b, c, d, e, x, s) \ - { \ - (a) += BEAST_RIPEMD_H((b), (c), (d)) + (x) + 0x6ed9eba1UL; \ - (a) = BEAST_RIPEMD_ROL((a), (s)) + (e); \ - (c) = BEAST_RIPEMD_ROL((c), 10); \ - } -#define BEAST_RIPEMD_II(a, b, c, d, e, x, s) \ - { \ - (a) += BEAST_RIPEMD_I((b), (c), (d)) + (x) + 0x8f1bbcdcUL; \ - (a) = BEAST_RIPEMD_ROL((a), (s)) + (e); \ - (c) = BEAST_RIPEMD_ROL((c), 10); \ - } -#define BEAST_RIPEMD_JJ(a, b, c, d, e, x, s) \ - { \ - (a) += BEAST_RIPEMD_J((b), (c), (d)) + (x) + 0xa953fd4eUL; \ - (a) = BEAST_RIPEMD_ROL((a), (s)) + (e); \ - (c) = BEAST_RIPEMD_ROL((c), 10); \ - } -#define BEAST_RIPEMD_FFF(a, b, c, d, e, x, s) \ - { \ - (a) += BEAST_RIPEMD_F((b), (c), (d)) + (x); \ - (a) = BEAST_RIPEMD_ROL((a), (s)) + (e); \ - (c) = BEAST_RIPEMD_ROL((c), 10); \ - } -#define BEAST_RIPEMD_GGG(a, b, c, d, e, x, s) \ - { \ - (a) += BEAST_RIPEMD_G((b), (c), (d)) + (x) + 0x7a6d76e9UL; \ - (a) = BEAST_RIPEMD_ROL((a), (s)) + (e); \ - (c) = BEAST_RIPEMD_ROL((c), 10); \ - } -#define BEAST_RIPEMD_HHH(a, b, c, d, e, x, s) \ - { \ - (a) += BEAST_RIPEMD_H((b), (c), (d)) + (x) + 0x6d703ef3UL; \ - (a) = BEAST_RIPEMD_ROL((a), (s)) + (e); \ - (c) = BEAST_RIPEMD_ROL((c), 10); \ - } -#define BEAST_RIPEMD_III(a, b, c, d, e, x, s) \ - { \ - (a) += BEAST_RIPEMD_I((b), (c), (d)) + (x) + 0x5c4dd124UL; \ - (a) = BEAST_RIPEMD_ROL((a), (s)) + (e); \ - (c) = BEAST_RIPEMD_ROL((c), 10); \ - } -#define BEAST_RIPEMD_JJJ(a, b, c, d, e, x, s) \ - { \ - (a) += BEAST_RIPEMD_J((b), (c), (d)) + (x) + 0x50a28be6UL; \ - (a) = BEAST_RIPEMD_ROL((a), (s)) + (e); \ - (c) = BEAST_RIPEMD_ROL((c), 10); \ - } - -template -void -ripemd_load(std::array& X, unsigned char const* p) -{ - for (int i = 0; i < 16; ++i) - { - X[i] = ((std::uint32_t) * ((p) + 3) << 24) | - ((std::uint32_t) * ((p) + 2) << 16) | - ((std::uint32_t) * ((p) + 1) << 8) | ((std::uint32_t) * (p)); - p += 4; - } -} - -template -void -ripemd_compress( - ripemd160_context& ctx, - std::array& X) noexcept -{ - std::uint32_t aa = ctx.h[0]; - std::uint32_t bb = ctx.h[1]; - std::uint32_t cc = ctx.h[2]; - std::uint32_t dd = ctx.h[3]; - std::uint32_t ee = ctx.h[4]; - std::uint32_t aaa = ctx.h[0]; - std::uint32_t bbb = ctx.h[1]; - std::uint32_t ccc = ctx.h[2]; - std::uint32_t ddd = ctx.h[3]; - std::uint32_t eee = ctx.h[4]; - - // round 1 - BEAST_RIPEMD_FF(aa, bb, cc, dd, ee, X[0], 11); - BEAST_RIPEMD_FF(ee, aa, bb, cc, dd, X[1], 14); - BEAST_RIPEMD_FF(dd, ee, aa, bb, cc, X[2], 15); - BEAST_RIPEMD_FF(cc, dd, ee, aa, bb, X[3], 12); - BEAST_RIPEMD_FF(bb, cc, dd, ee, aa, X[4], 5); - BEAST_RIPEMD_FF(aa, bb, cc, dd, ee, X[5], 8); - BEAST_RIPEMD_FF(ee, aa, bb, cc, dd, X[6], 7); - BEAST_RIPEMD_FF(dd, ee, aa, bb, cc, X[7], 9); - BEAST_RIPEMD_FF(cc, dd, ee, aa, bb, X[8], 11); - BEAST_RIPEMD_FF(bb, cc, dd, ee, aa, X[9], 13); - BEAST_RIPEMD_FF(aa, bb, cc, dd, ee, X[10], 14); - BEAST_RIPEMD_FF(ee, aa, bb, cc, dd, X[11], 15); - BEAST_RIPEMD_FF(dd, ee, aa, bb, cc, X[12], 6); - BEAST_RIPEMD_FF(cc, dd, ee, aa, bb, X[13], 7); - BEAST_RIPEMD_FF(bb, cc, dd, ee, aa, X[14], 9); - BEAST_RIPEMD_FF(aa, bb, cc, dd, ee, X[15], 8); - - // round 2 - BEAST_RIPEMD_GG(ee, aa, bb, cc, dd, X[7], 7); - BEAST_RIPEMD_GG(dd, ee, aa, bb, cc, X[4], 6); - BEAST_RIPEMD_GG(cc, dd, ee, aa, bb, X[13], 8); - BEAST_RIPEMD_GG(bb, cc, dd, ee, aa, X[1], 13); - BEAST_RIPEMD_GG(aa, bb, cc, dd, ee, X[10], 11); - BEAST_RIPEMD_GG(ee, aa, bb, cc, dd, X[6], 9); - BEAST_RIPEMD_GG(dd, ee, aa, bb, cc, X[15], 7); - BEAST_RIPEMD_GG(cc, dd, ee, aa, bb, X[3], 15); - BEAST_RIPEMD_GG(bb, cc, dd, ee, aa, X[12], 7); - BEAST_RIPEMD_GG(aa, bb, cc, dd, ee, X[0], 12); - BEAST_RIPEMD_GG(ee, aa, bb, cc, dd, X[9], 15); - BEAST_RIPEMD_GG(dd, ee, aa, bb, cc, X[5], 9); - BEAST_RIPEMD_GG(cc, dd, ee, aa, bb, X[2], 11); - BEAST_RIPEMD_GG(bb, cc, dd, ee, aa, X[14], 7); - BEAST_RIPEMD_GG(aa, bb, cc, dd, ee, X[11], 13); - BEAST_RIPEMD_GG(ee, aa, bb, cc, dd, X[8], 12); - - // round 3 - BEAST_RIPEMD_HH(dd, ee, aa, bb, cc, X[3], 11); - BEAST_RIPEMD_HH(cc, dd, ee, aa, bb, X[10], 13); - BEAST_RIPEMD_HH(bb, cc, dd, ee, aa, X[14], 6); - BEAST_RIPEMD_HH(aa, bb, cc, dd, ee, X[4], 7); - BEAST_RIPEMD_HH(ee, aa, bb, cc, dd, X[9], 14); - BEAST_RIPEMD_HH(dd, ee, aa, bb, cc, X[15], 9); - BEAST_RIPEMD_HH(cc, dd, ee, aa, bb, X[8], 13); - BEAST_RIPEMD_HH(bb, cc, dd, ee, aa, X[1], 15); - BEAST_RIPEMD_HH(aa, bb, cc, dd, ee, X[2], 14); - BEAST_RIPEMD_HH(ee, aa, bb, cc, dd, X[7], 8); - BEAST_RIPEMD_HH(dd, ee, aa, bb, cc, X[0], 13); - BEAST_RIPEMD_HH(cc, dd, ee, aa, bb, X[6], 6); - BEAST_RIPEMD_HH(bb, cc, dd, ee, aa, X[13], 5); - BEAST_RIPEMD_HH(aa, bb, cc, dd, ee, X[11], 12); - BEAST_RIPEMD_HH(ee, aa, bb, cc, dd, X[5], 7); - BEAST_RIPEMD_HH(dd, ee, aa, bb, cc, X[12], 5); - - // round 4 - BEAST_RIPEMD_II(cc, dd, ee, aa, bb, X[1], 11); - BEAST_RIPEMD_II(bb, cc, dd, ee, aa, X[9], 12); - BEAST_RIPEMD_II(aa, bb, cc, dd, ee, X[11], 14); - BEAST_RIPEMD_II(ee, aa, bb, cc, dd, X[10], 15); - BEAST_RIPEMD_II(dd, ee, aa, bb, cc, X[0], 14); - BEAST_RIPEMD_II(cc, dd, ee, aa, bb, X[8], 15); - BEAST_RIPEMD_II(bb, cc, dd, ee, aa, X[12], 9); - BEAST_RIPEMD_II(aa, bb, cc, dd, ee, X[4], 8); - BEAST_RIPEMD_II(ee, aa, bb, cc, dd, X[13], 9); - BEAST_RIPEMD_II(dd, ee, aa, bb, cc, X[3], 14); - BEAST_RIPEMD_II(cc, dd, ee, aa, bb, X[7], 5); - BEAST_RIPEMD_II(bb, cc, dd, ee, aa, X[15], 6); - BEAST_RIPEMD_II(aa, bb, cc, dd, ee, X[14], 8); - BEAST_RIPEMD_II(ee, aa, bb, cc, dd, X[5], 6); - BEAST_RIPEMD_II(dd, ee, aa, bb, cc, X[6], 5); - BEAST_RIPEMD_II(cc, dd, ee, aa, bb, X[2], 12); - - // round 5 - BEAST_RIPEMD_JJ(bb, cc, dd, ee, aa, X[4], 9); - BEAST_RIPEMD_JJ(aa, bb, cc, dd, ee, X[0], 15); - BEAST_RIPEMD_JJ(ee, aa, bb, cc, dd, X[5], 5); - BEAST_RIPEMD_JJ(dd, ee, aa, bb, cc, X[9], 11); - BEAST_RIPEMD_JJ(cc, dd, ee, aa, bb, X[7], 6); - BEAST_RIPEMD_JJ(bb, cc, dd, ee, aa, X[12], 8); - BEAST_RIPEMD_JJ(aa, bb, cc, dd, ee, X[2], 13); - BEAST_RIPEMD_JJ(ee, aa, bb, cc, dd, X[10], 12); - BEAST_RIPEMD_JJ(dd, ee, aa, bb, cc, X[14], 5); - BEAST_RIPEMD_JJ(cc, dd, ee, aa, bb, X[1], 12); - BEAST_RIPEMD_JJ(bb, cc, dd, ee, aa, X[3], 13); - BEAST_RIPEMD_JJ(aa, bb, cc, dd, ee, X[8], 14); - BEAST_RIPEMD_JJ(ee, aa, bb, cc, dd, X[11], 11); - BEAST_RIPEMD_JJ(dd, ee, aa, bb, cc, X[6], 8); - BEAST_RIPEMD_JJ(cc, dd, ee, aa, bb, X[15], 5); - BEAST_RIPEMD_JJ(bb, cc, dd, ee, aa, X[13], 6); - - // parallel round 1 - BEAST_RIPEMD_JJJ(aaa, bbb, ccc, ddd, eee, X[5], 8); - BEAST_RIPEMD_JJJ(eee, aaa, bbb, ccc, ddd, X[14], 9); - BEAST_RIPEMD_JJJ(ddd, eee, aaa, bbb, ccc, X[7], 9); - BEAST_RIPEMD_JJJ(ccc, ddd, eee, aaa, bbb, X[0], 11); - BEAST_RIPEMD_JJJ(bbb, ccc, ddd, eee, aaa, X[9], 13); - BEAST_RIPEMD_JJJ(aaa, bbb, ccc, ddd, eee, X[2], 15); - BEAST_RIPEMD_JJJ(eee, aaa, bbb, ccc, ddd, X[11], 15); - BEAST_RIPEMD_JJJ(ddd, eee, aaa, bbb, ccc, X[4], 5); - BEAST_RIPEMD_JJJ(ccc, ddd, eee, aaa, bbb, X[13], 7); - BEAST_RIPEMD_JJJ(bbb, ccc, ddd, eee, aaa, X[6], 7); - BEAST_RIPEMD_JJJ(aaa, bbb, ccc, ddd, eee, X[15], 8); - BEAST_RIPEMD_JJJ(eee, aaa, bbb, ccc, ddd, X[8], 11); - BEAST_RIPEMD_JJJ(ddd, eee, aaa, bbb, ccc, X[1], 14); - BEAST_RIPEMD_JJJ(ccc, ddd, eee, aaa, bbb, X[10], 14); - BEAST_RIPEMD_JJJ(bbb, ccc, ddd, eee, aaa, X[3], 12); - BEAST_RIPEMD_JJJ(aaa, bbb, ccc, ddd, eee, X[12], 6); - - // parallel round 2 - BEAST_RIPEMD_III(eee, aaa, bbb, ccc, ddd, X[6], 9); - BEAST_RIPEMD_III(ddd, eee, aaa, bbb, ccc, X[11], 13); - BEAST_RIPEMD_III(ccc, ddd, eee, aaa, bbb, X[3], 15); - BEAST_RIPEMD_III(bbb, ccc, ddd, eee, aaa, X[7], 7); - BEAST_RIPEMD_III(aaa, bbb, ccc, ddd, eee, X[0], 12); - BEAST_RIPEMD_III(eee, aaa, bbb, ccc, ddd, X[13], 8); - BEAST_RIPEMD_III(ddd, eee, aaa, bbb, ccc, X[5], 9); - BEAST_RIPEMD_III(ccc, ddd, eee, aaa, bbb, X[10], 11); - BEAST_RIPEMD_III(bbb, ccc, ddd, eee, aaa, X[14], 7); - BEAST_RIPEMD_III(aaa, bbb, ccc, ddd, eee, X[15], 7); - BEAST_RIPEMD_III(eee, aaa, bbb, ccc, ddd, X[8], 12); - BEAST_RIPEMD_III(ddd, eee, aaa, bbb, ccc, X[12], 7); - BEAST_RIPEMD_III(ccc, ddd, eee, aaa, bbb, X[4], 6); - BEAST_RIPEMD_III(bbb, ccc, ddd, eee, aaa, X[9], 15); - BEAST_RIPEMD_III(aaa, bbb, ccc, ddd, eee, X[1], 13); - BEAST_RIPEMD_III(eee, aaa, bbb, ccc, ddd, X[2], 11); - - // parallel round 3 - BEAST_RIPEMD_HHH(ddd, eee, aaa, bbb, ccc, X[15], 9); - BEAST_RIPEMD_HHH(ccc, ddd, eee, aaa, bbb, X[5], 7); - BEAST_RIPEMD_HHH(bbb, ccc, ddd, eee, aaa, X[1], 15); - BEAST_RIPEMD_HHH(aaa, bbb, ccc, ddd, eee, X[3], 11); - BEAST_RIPEMD_HHH(eee, aaa, bbb, ccc, ddd, X[7], 8); - BEAST_RIPEMD_HHH(ddd, eee, aaa, bbb, ccc, X[14], 6); - BEAST_RIPEMD_HHH(ccc, ddd, eee, aaa, bbb, X[6], 6); - BEAST_RIPEMD_HHH(bbb, ccc, ddd, eee, aaa, X[9], 14); - BEAST_RIPEMD_HHH(aaa, bbb, ccc, ddd, eee, X[11], 12); - BEAST_RIPEMD_HHH(eee, aaa, bbb, ccc, ddd, X[8], 13); - BEAST_RIPEMD_HHH(ddd, eee, aaa, bbb, ccc, X[12], 5); - BEAST_RIPEMD_HHH(ccc, ddd, eee, aaa, bbb, X[2], 14); - BEAST_RIPEMD_HHH(bbb, ccc, ddd, eee, aaa, X[10], 13); - BEAST_RIPEMD_HHH(aaa, bbb, ccc, ddd, eee, X[0], 13); - BEAST_RIPEMD_HHH(eee, aaa, bbb, ccc, ddd, X[4], 7); - BEAST_RIPEMD_HHH(ddd, eee, aaa, bbb, ccc, X[13], 5); - - // parallel round 4 - BEAST_RIPEMD_GGG(ccc, ddd, eee, aaa, bbb, X[8], 15); - BEAST_RIPEMD_GGG(bbb, ccc, ddd, eee, aaa, X[6], 5); - BEAST_RIPEMD_GGG(aaa, bbb, ccc, ddd, eee, X[4], 8); - BEAST_RIPEMD_GGG(eee, aaa, bbb, ccc, ddd, X[1], 11); - BEAST_RIPEMD_GGG(ddd, eee, aaa, bbb, ccc, X[3], 14); - BEAST_RIPEMD_GGG(ccc, ddd, eee, aaa, bbb, X[11], 14); - BEAST_RIPEMD_GGG(bbb, ccc, ddd, eee, aaa, X[15], 6); - BEAST_RIPEMD_GGG(aaa, bbb, ccc, ddd, eee, X[0], 14); - BEAST_RIPEMD_GGG(eee, aaa, bbb, ccc, ddd, X[5], 6); - BEAST_RIPEMD_GGG(ddd, eee, aaa, bbb, ccc, X[12], 9); - BEAST_RIPEMD_GGG(ccc, ddd, eee, aaa, bbb, X[2], 12); - BEAST_RIPEMD_GGG(bbb, ccc, ddd, eee, aaa, X[13], 9); - BEAST_RIPEMD_GGG(aaa, bbb, ccc, ddd, eee, X[9], 12); - BEAST_RIPEMD_GGG(eee, aaa, bbb, ccc, ddd, X[7], 5); - BEAST_RIPEMD_GGG(ddd, eee, aaa, bbb, ccc, X[10], 15); - BEAST_RIPEMD_GGG(ccc, ddd, eee, aaa, bbb, X[14], 8); - - // parallel round 5 - BEAST_RIPEMD_FFF(bbb, ccc, ddd, eee, aaa, X[12], 8); - BEAST_RIPEMD_FFF(aaa, bbb, ccc, ddd, eee, X[15], 5); - BEAST_RIPEMD_FFF(eee, aaa, bbb, ccc, ddd, X[10], 12); - BEAST_RIPEMD_FFF(ddd, eee, aaa, bbb, ccc, X[4], 9); - BEAST_RIPEMD_FFF(ccc, ddd, eee, aaa, bbb, X[1], 12); - BEAST_RIPEMD_FFF(bbb, ccc, ddd, eee, aaa, X[5], 5); - BEAST_RIPEMD_FFF(aaa, bbb, ccc, ddd, eee, X[8], 14); - BEAST_RIPEMD_FFF(eee, aaa, bbb, ccc, ddd, X[7], 6); - BEAST_RIPEMD_FFF(ddd, eee, aaa, bbb, ccc, X[6], 8); - BEAST_RIPEMD_FFF(ccc, ddd, eee, aaa, bbb, X[2], 13); - BEAST_RIPEMD_FFF(bbb, ccc, ddd, eee, aaa, X[13], 6); - BEAST_RIPEMD_FFF(aaa, bbb, ccc, ddd, eee, X[14], 5); - BEAST_RIPEMD_FFF(eee, aaa, bbb, ccc, ddd, X[0], 15); - BEAST_RIPEMD_FFF(ddd, eee, aaa, bbb, ccc, X[3], 13); - BEAST_RIPEMD_FFF(ccc, ddd, eee, aaa, bbb, X[9], 11); - BEAST_RIPEMD_FFF(bbb, ccc, ddd, eee, aaa, X[11], 11); - - // combine results - ddd += cc + ctx.h[1]; // final result for h[0] - ctx.h[1] = ctx.h[2] + dd + eee; - ctx.h[2] = ctx.h[3] + ee + aaa; - ctx.h[3] = ctx.h[4] + aa + bbb; - ctx.h[4] = ctx.h[0] + bb + ccc; - ctx.h[0] = ddd; -} - -template -void -init(ripemd160_context& ctx) noexcept -{ - ctx.len = 0; - ctx.tot_len = 0; - ctx.h[0] = 0x67452301UL; - ctx.h[1] = 0xefcdab89UL; - ctx.h[2] = 0x98badcfeUL; - ctx.h[3] = 0x10325476UL; - ctx.h[4] = 0xc3d2e1f0UL; -} - -template -void -update(ripemd160_context& ctx, void const* message, std::size_t size) noexcept -{ - auto const pm = reinterpret_cast(message); - unsigned int block_nb; - unsigned int new_len, rem_len, tmp_len; - const unsigned char* shifted_message; - tmp_len = ripemd160_context::block_size - ctx.len; - rem_len = size < tmp_len ? size : tmp_len; - std::memcpy(&ctx.block[ctx.len], pm, rem_len); - if (ctx.len + size < ripemd160_context::block_size) - { - ctx.len += size; - return; - } - new_len = size - rem_len; - block_nb = new_len / ripemd160_context::block_size; - shifted_message = pm + rem_len; - std::array X; - ripemd_load(X, ctx.block); - ripemd_compress(ctx, X); - for (int i = 0; i < block_nb; ++i) - { - ripemd_load(X, shifted_message + i * ripemd160_context::block_size); - ripemd_compress(ctx, X); - } - rem_len = new_len % ripemd160_context::block_size; - std::memcpy( - ctx.block, - &shifted_message[block_nb * ripemd160_context::block_size], - rem_len); - ctx.len = rem_len; - ctx.tot_len += (block_nb + 1) * ripemd160_context::block_size; -} - -template -void -finish(ripemd160_context& ctx, void* digest) noexcept -{ - std::array X; - X.fill(0); - // put leftovers into X - auto p = &ctx.block[0]; - // uint8_t i goes into word X[i div 4] at pos. 8*(i mod 4) - for (int i = 0; i < ctx.len; ++i) - X[i >> 2] ^= (std::uint32_t)*p++ << (8 * (i & 3)); - ctx.tot_len += ctx.len; - // append the bit m_n == 1 - X[(ctx.tot_len >> 2) & 15] ^= (uint32_t)1 << (8 * (ctx.tot_len & 3) + 7); - // length goes to next block? - if ((ctx.tot_len & 63) > 55) - { - ripemd_compress(ctx, X); - X.fill(0); - } - // append length in bits*/ - X[14] = ctx.tot_len << 3; - X[15] = (ctx.tot_len >> 29) | (0 << 3); - ripemd_compress(ctx, X); - - std::uint8_t* pd = reinterpret_cast(digest); - for (std::uint32_t i = 0; i < 20; i += 4) - { - pd[i] = (std::uint8_t)(ctx.h[i >> 2]); // implicit cast to uint8_t - pd[i + 1] = (std::uint8_t)(ctx.h[i >> 2] >> 8); // extracts the 8 least - pd[i + 2] = (std::uint8_t)(ctx.h[i >> 2] >> 16); // significant bits. - pd[i + 3] = (std::uint8_t)(ctx.h[i >> 2] >> 24); - } -} - -} // namespace detail -} // namespace beast - -#endif diff --git a/src/ripple/beast/crypto/detail/sha2_context.h b/src/ripple/beast/crypto/detail/sha2_context.h deleted file mode 100644 index 6fc8018fc67..00000000000 --- a/src/ripple/beast/crypto/detail/sha2_context.h +++ /dev/null @@ -1,419 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of Beast: https://github.com/vinniefalco/Beast - Copyright 2013, Vinnie Falco - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef BEAST_CRYPTO_SHA512_CONTEXT_H_INCLUDED -#define BEAST_CRYPTO_SHA512_CONTEXT_H_INCLUDED - -#include -#include - -namespace beast { -namespace detail { - -// Based on https://github.com/ogay/sha2 -// This implementation has been modified from the -// original. It has been updated for C++11. - -/* - * Updated to C++, zedwood.com 2012 - * Based on Olivier Gay's version - * See Modified BSD License below: - * - * FIPS 180-2 SHA-224/256/384/512 implementation - * Issue date: 04/30/2005 - * http://www.ouah.org/ogay/sha2/ - * - * Copyright (C) 2005, 2007 Olivier Gay - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the project nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -struct sha256_context -{ - explicit sha256_context() = default; - - static unsigned int const block_size = 64; - static unsigned int const digest_size = 32; - - unsigned int tot_len; - unsigned int len; - unsigned char block[2 * block_size]; - std::uint32_t h[8]; -}; - -struct sha512_context -{ - explicit sha512_context() = default; - - static unsigned int const block_size = 128; - static unsigned int const digest_size = 64; - - unsigned int tot_len; - unsigned int len; - unsigned char block[2 * block_size]; - std::uint64_t h[8]; -}; - -#define BEAST_SHA2_SHFR(x, n) (x >> n) -#define BEAST_SHA2_ROTR(x, n) ((x >> n) | (x << ((sizeof(x) << 3) - n))) -#define BEAST_SHA2_ROTL(x, n) ((x << n) | (x >> ((sizeof(x) << 3) - n))) -#define BEAST_SHA2_CH(x, y, z) ((x & y) ^ (~x & z)) -#define BEAST_SHA2_MAJ(x, y, z) ((x & y) ^ (x & z) ^ (y & z)) -#define BEAST_SHA256_F1(x) \ - (BEAST_SHA2_ROTR(x, 2) ^ BEAST_SHA2_ROTR(x, 13) ^ BEAST_SHA2_ROTR(x, 22)) -#define BEAST_SHA256_F2(x) \ - (BEAST_SHA2_ROTR(x, 6) ^ BEAST_SHA2_ROTR(x, 11) ^ BEAST_SHA2_ROTR(x, 25)) -#define BEAST_SHA256_F3(x) \ - (BEAST_SHA2_ROTR(x, 7) ^ BEAST_SHA2_ROTR(x, 18) ^ BEAST_SHA2_SHFR(x, 3)) -#define BEAST_SHA256_F4(x) \ - (BEAST_SHA2_ROTR(x, 17) ^ BEAST_SHA2_ROTR(x, 19) ^ BEAST_SHA2_SHFR(x, 10)) -#define BEAST_SHA512_F1(x) \ - (BEAST_SHA2_ROTR(x, 28) ^ BEAST_SHA2_ROTR(x, 34) ^ BEAST_SHA2_ROTR(x, 39)) -#define BEAST_SHA512_F2(x) \ - (BEAST_SHA2_ROTR(x, 14) ^ BEAST_SHA2_ROTR(x, 18) ^ BEAST_SHA2_ROTR(x, 41)) -#define BEAST_SHA512_F3(x) \ - (BEAST_SHA2_ROTR(x, 1) ^ BEAST_SHA2_ROTR(x, 8) ^ BEAST_SHA2_SHFR(x, 7)) -#define BEAST_SHA512_F4(x) \ - (BEAST_SHA2_ROTR(x, 19) ^ BEAST_SHA2_ROTR(x, 61) ^ BEAST_SHA2_SHFR(x, 6)) -#define BEAST_SHA2_PACK32(str, x) \ - { \ - *(x) = ((std::uint32_t) * ((str) + 3)) | \ - ((std::uint32_t) * ((str) + 2) << 8) | \ - ((std::uint32_t) * ((str) + 1) << 16) | \ - ((std::uint32_t) * ((str) + 0) << 24); \ - } -#define BEAST_SHA2_UNPACK32(x, str) \ - { \ - *((str) + 3) = (std::uint8_t)((x)); \ - *((str) + 2) = (std::uint8_t)((x) >> 8); \ - *((str) + 1) = (std::uint8_t)((x) >> 16); \ - *((str) + 0) = (std::uint8_t)((x) >> 24); \ - } -#define BEAST_SHA2_PACK64(str, x) \ - { \ - *(x) = ((std::uint64_t) * ((str) + 7)) | \ - ((std::uint64_t) * ((str) + 6) << 8) | \ - ((std::uint64_t) * ((str) + 5) << 16) | \ - ((std::uint64_t) * ((str) + 4) << 24) | \ - ((std::uint64_t) * ((str) + 3) << 32) | \ - ((std::uint64_t) * ((str) + 2) << 40) | \ - ((std::uint64_t) * ((str) + 1) << 48) | \ - ((std::uint64_t) * ((str) + 0) << 56); \ - } -#define BEAST_SHA2_UNPACK64(x, str) \ - { \ - *((str) + 7) = (std::uint8_t)((x)); \ - *((str) + 6) = (std::uint8_t)((x) >> 8); \ - *((str) + 5) = (std::uint8_t)((x) >> 16); \ - *((str) + 4) = (std::uint8_t)((x) >> 24); \ - *((str) + 3) = (std::uint8_t)((x) >> 32); \ - *((str) + 2) = (std::uint8_t)((x) >> 40); \ - *((str) + 1) = (std::uint8_t)((x) >> 48); \ - *((str) + 0) = (std::uint8_t)((x) >> 56); \ - } - -//------------------------------------------------------------------------------ - -// SHA256 - -template -void -sha256_transform( - sha256_context& ctx, - unsigned char const* message, - unsigned int block_nb) noexcept -{ - static unsigned long long const K[64] = { - 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, - 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, - 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, - 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, - 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, - 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, - 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, - 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, - 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, - 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, - 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; - std::uint32_t w[64]; - std::uint32_t wv[8]; - std::uint32_t t1, t2; - unsigned char const* sub_block; - int i, j; - for (i = 0; i < (int)block_nb; i++) - { - sub_block = message + (i << 6); - for (j = 0; j < 16; j++) - BEAST_SHA2_PACK32(&sub_block[j << 2], &w[j]); - for (j = 16; j < 64; j++) - w[j] = BEAST_SHA256_F4(w[j - 2]) + w[j - 7] + - BEAST_SHA256_F3(w[j - 15]) + w[j - 16]; - for (j = 0; j < 8; j++) - wv[j] = ctx.h[j]; - for (j = 0; j < 64; j++) - { - t1 = wv[7] + BEAST_SHA256_F2(wv[4]) + - BEAST_SHA2_CH(wv[4], wv[5], wv[6]) + K[j] + w[j]; - t2 = BEAST_SHA256_F1(wv[0]) + BEAST_SHA2_MAJ(wv[0], wv[1], wv[2]); - wv[7] = wv[6]; - wv[6] = wv[5]; - wv[5] = wv[4]; - wv[4] = wv[3] + t1; - wv[3] = wv[2]; - wv[2] = wv[1]; - wv[1] = wv[0]; - wv[0] = t1 + t2; - } - for (j = 0; j < 8; j++) - ctx.h[j] += wv[j]; - } -} - -template -void -init(sha256_context& ctx) noexcept -{ - ctx.len = 0; - ctx.tot_len = 0; - ctx.h[0] = 0x6a09e667; - ctx.h[1] = 0xbb67ae85; - ctx.h[2] = 0x3c6ef372; - ctx.h[3] = 0xa54ff53a; - ctx.h[4] = 0x510e527f; - ctx.h[5] = 0x9b05688c; - ctx.h[6] = 0x1f83d9ab; - ctx.h[7] = 0x5be0cd19; -} - -template -void -update(sha256_context& ctx, void const* message, std::size_t size) noexcept -{ - auto const pm = reinterpret_cast(message); - unsigned int block_nb; - unsigned int new_len, rem_len, tmp_len; - const unsigned char* shifted_message; - tmp_len = sha256_context::block_size - ctx.len; - rem_len = size < tmp_len ? size : tmp_len; - std::memcpy(&ctx.block[ctx.len], pm, rem_len); - if (ctx.len + size < sha256_context::block_size) - { - ctx.len += size; - return; - } - new_len = size - rem_len; - block_nb = new_len / sha256_context::block_size; - shifted_message = pm + rem_len; - sha256_transform(ctx, ctx.block, 1); - sha256_transform(ctx, shifted_message, block_nb); - rem_len = new_len % sha256_context::block_size; - std::memcpy(ctx.block, &shifted_message[block_nb << 6], rem_len); - ctx.len = rem_len; - ctx.tot_len += (block_nb + 1) << 6; -} - -template -void -finish(sha256_context& ctx, void* digest) noexcept -{ - auto const pd = reinterpret_cast(digest); - unsigned int block_nb; - unsigned int pm_len; - unsigned int len_b; - int i; - block_nb = - (1 + - ((sha256_context::block_size - 9) < - (ctx.len % sha256_context::block_size))); - len_b = (ctx.tot_len + ctx.len) << 3; - pm_len = block_nb << 6; - std::memset(ctx.block + ctx.len, 0, pm_len - ctx.len); - ctx.block[ctx.len] = 0x80; - BEAST_SHA2_UNPACK32(len_b, ctx.block + pm_len - 4); - sha256_transform(ctx, ctx.block, block_nb); - for (i = 0; i < 8; i++) - BEAST_SHA2_UNPACK32(ctx.h[i], &pd[i << 2]); -} - -//------------------------------------------------------------------------------ - -// SHA512 - -template -void -sha512_transform( - sha512_context& ctx, - unsigned char const* message, - unsigned int block_nb) noexcept -{ - static unsigned long long const K[80] = { - 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 0xb5c0fbcfec4d3b2fULL, - 0xe9b5dba58189dbbcULL, 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, - 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, 0xd807aa98a3030242ULL, - 0x12835b0145706fbeULL, 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, - 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, 0x9bdc06a725c71235ULL, - 0xc19bf174cf692694ULL, 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, - 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, 0x2de92c6f592b0275ULL, - 0x4a7484aa6ea6e483ULL, 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, - 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, 0xb00327c898fb213fULL, - 0xbf597fc7beef0ee4ULL, 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, - 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, 0x27b70a8546d22ffcULL, - 0x2e1b21385c26c926ULL, 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, - 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, 0x81c2c92e47edaee6ULL, - 0x92722c851482353bULL, 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, - 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, 0xd192e819d6ef5218ULL, - 0xd69906245565a910ULL, 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, - 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, 0x2748774cdf8eeb99ULL, - 0x34b0bcb5e19b48a8ULL, 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, - 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, 0x748f82ee5defb2fcULL, - 0x78a5636f43172f60ULL, 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, - 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, 0xbef9a3f7b2c67915ULL, - 0xc67178f2e372532bULL, 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, - 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, 0x06f067aa72176fbaULL, - 0x0a637dc5a2c898a6ULL, 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, - 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, 0x3c9ebe0a15c9bebcULL, - 0x431d67c49c100d4cULL, 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, - 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL}; - - std::uint64_t w[80]; - std::uint64_t wv[8]; - std::uint64_t t1, t2; - unsigned char const* sub_block; - int i, j; - for (i = 0; i < (int)block_nb; i++) - { - sub_block = message + (i << 7); - for (j = 0; j < 16; j++) - BEAST_SHA2_PACK64(&sub_block[j << 3], &w[j]); - for (j = 16; j < 80; j++) - w[j] = BEAST_SHA512_F4(w[j - 2]) + w[j - 7] + - BEAST_SHA512_F3(w[j - 15]) + w[j - 16]; - for (j = 0; j < 8; j++) - wv[j] = ctx.h[j]; - for (j = 0; j < 80; j++) - { - t1 = wv[7] + BEAST_SHA512_F2(wv[4]) + - BEAST_SHA2_CH(wv[4], wv[5], wv[6]) + K[j] + w[j]; - t2 = BEAST_SHA512_F1(wv[0]) + BEAST_SHA2_MAJ(wv[0], wv[1], wv[2]); - wv[7] = wv[6]; - wv[6] = wv[5]; - wv[5] = wv[4]; - wv[4] = wv[3] + t1; - wv[3] = wv[2]; - wv[2] = wv[1]; - wv[1] = wv[0]; - wv[0] = t1 + t2; - } - for (j = 0; j < 8; j++) - ctx.h[j] += wv[j]; - } -} - -template -void -init(sha512_context& ctx) noexcept -{ - ctx.len = 0; - ctx.tot_len = 0; - ctx.h[0] = 0x6a09e667f3bcc908ULL; - ctx.h[1] = 0xbb67ae8584caa73bULL; - ctx.h[2] = 0x3c6ef372fe94f82bULL; - ctx.h[3] = 0xa54ff53a5f1d36f1ULL; - ctx.h[4] = 0x510e527fade682d1ULL; - ctx.h[5] = 0x9b05688c2b3e6c1fULL; - ctx.h[6] = 0x1f83d9abfb41bd6bULL; - ctx.h[7] = 0x5be0cd19137e2179ULL; -} - -template -void -update(sha512_context& ctx, void const* message, std::size_t size) noexcept -{ - auto const pm = reinterpret_cast(message); - unsigned int block_nb; - unsigned int new_len, rem_len, tmp_len; - const unsigned char* shifted_message; - tmp_len = sha512_context::block_size - ctx.len; - rem_len = size < tmp_len ? size : tmp_len; - std::memcpy(&ctx.block[ctx.len], pm, rem_len); - if (ctx.len + size < sha512_context::block_size) - { - ctx.len += size; - return; - } - new_len = size - rem_len; - block_nb = new_len / sha512_context::block_size; - shifted_message = pm + rem_len; - sha512_transform(ctx, ctx.block, 1); - sha512_transform(ctx, shifted_message, block_nb); - rem_len = new_len % sha512_context::block_size; - std::memcpy(ctx.block, &shifted_message[block_nb << 7], rem_len); - ctx.len = rem_len; - ctx.tot_len += (block_nb + 1) << 7; -} - -template -void -finish(sha512_context& ctx, void* digest) noexcept -{ - auto const pd = reinterpret_cast(digest); - unsigned int block_nb; - unsigned int pm_len; - unsigned int len_b; - int i; - block_nb = 1 + - ((sha512_context::block_size - 17) < - (ctx.len % sha512_context::block_size)); - len_b = (ctx.tot_len + ctx.len) << 3; - pm_len = block_nb << 7; - std::memset(ctx.block + ctx.len, 0, pm_len - ctx.len); - ctx.block[ctx.len] = 0x80; - BEAST_SHA2_UNPACK32(len_b, ctx.block + pm_len - 4); - sha512_transform(ctx, ctx.block, block_nb); - for (i = 0; i < 8; i++) - BEAST_SHA2_UNPACK64(ctx.h[i], &pd[i << 3]); -} - -} // namespace detail -} // namespace beast - -#endif diff --git a/src/ripple/beast/crypto/ripemd.h b/src/ripple/beast/crypto/ripemd.h deleted file mode 100644 index 3111deee85d..00000000000 --- a/src/ripple/beast/crypto/ripemd.h +++ /dev/null @@ -1,35 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of Beast: https://github.com/vinniefalco/Beast - Copyright 2013, Vinnie Falco - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef BEAST_CRYPTO_RIPEMD_H_INCLUDED -#define BEAST_CRYPTO_RIPEMD_H_INCLUDED - -#include -#include - -namespace beast { - -using ripemd160_hasher = detail::mac_facade; - -// secure version -using ripemd160_hasher_s = detail::mac_facade; - -} // namespace beast - -#endif diff --git a/src/ripple/beast/crypto/sha2.h b/src/ripple/beast/crypto/sha2.h deleted file mode 100644 index c7cd187a512..00000000000 --- a/src/ripple/beast/crypto/sha2.h +++ /dev/null @@ -1,40 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of Beast: https://github.com/vinniefalco/Beast - Copyright 2013, Vinnie Falco - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef BEAST_CRYPTO_SHA2_H_INCLUDED -#define BEAST_CRYPTO_SHA2_H_INCLUDED - -#include -#include - -namespace beast { - -using sha256_hasher = detail::mac_facade; - -// secure version -using sha256_hasher_s = detail::mac_facade; - -using sha512_hasher = detail::mac_facade; - -// secure version -using sha512_hasher_s = detail::mac_facade; - -} // namespace beast - -#endif diff --git a/src/ripple/beast/cxx17/type_traits.h b/src/ripple/beast/cxx17/type_traits.h deleted file mode 100644 index 0dfeee45bdb..00000000000 --- a/src/ripple/beast/cxx17/type_traits.h +++ /dev/null @@ -1,41 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of Beast: https://github.com/vinniefalco/Beast - Copyright 2013, Vinnie Falco - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef BEAST_CXX17_TYPE_TRAITS_H_INCLUDED -#define BEAST_CXX17_TYPE_TRAITS_H_INCLUDED - -#include -#include -#include - -#if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 6000 - -namespace std { - -template -struct _LIBCPP_TEMPLATE_VIS is_invocable_r - : integral_constant::value> -{ -}; - -} // namespace std - -#endif - -#endif diff --git a/src/ripple/beast/hash/endian.h b/src/ripple/beast/hash/endian.h deleted file mode 100644 index cda8f4b0d44..00000000000 --- a/src/ripple/beast/hash/endian.h +++ /dev/null @@ -1,54 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of Beast: https://github.com/vinniefalco/Beast - Copyright 2014, Howard Hinnant , - Vinnie Falco #include #include +#include #include #include #include @@ -77,7 +77,10 @@ inline void maybe_reverse_bytes(T& t, Hasher&) { maybe_reverse_bytes( - t, std::integral_constant{}); + t, + std::integral_constant< + bool, + Hasher::endian != boost::endian::order::native>{}); } } // namespace detail @@ -182,7 +185,8 @@ struct is_contiguously_hashable : public std::integral_constant< bool, is_uniquely_represented::value && - (sizeof(T) == 1 || HashAlgorithm::endian == endian::native)> + (sizeof(T) == 1 || + HashAlgorithm::endian == boost::endian::order::native)> { explicit is_contiguously_hashable() = default; }; @@ -192,7 +196,8 @@ struct is_contiguously_hashable : public std::integral_constant< bool, is_uniquely_represented::value && - (sizeof(T) == 1 || HashAlgorithm::endian == endian::native)> + (sizeof(T) == 1 || + HashAlgorithm::endian == boost::endian::order::native)> { explicit is_contiguously_hashable() = default; }; diff --git a/src/ripple/beast/hash/xxhasher.h b/src/ripple/beast/hash/xxhasher.h index d6f6335ace7..512c451a051 100644 --- a/src/ripple/beast/hash/xxhasher.h +++ b/src/ripple/beast/hash/xxhasher.h @@ -20,8 +20,8 @@ #ifndef BEAST_HASH_XXHASHER_H_INCLUDED #define BEAST_HASH_XXHASHER_H_INCLUDED -#include #include +#include #include #include @@ -38,7 +38,7 @@ class xxhasher public: using result_type = std::size_t; - static beast::endian const endian = beast::endian::native; + static constexpr auto const endian = boost::endian::order::native; xxhasher() noexcept { diff --git a/src/ripple/beast/rfc2616.h b/src/ripple/beast/rfc2616.h index bb176b944ff..deac23550b0 100644 --- a/src/ripple/beast/rfc2616.h +++ b/src/ripple/beast/rfc2616.h @@ -29,7 +29,6 @@ #include #include #include -#include // for std::tie, remove ASAP #include #include @@ -38,27 +37,6 @@ namespace rfc2616 { namespace detail { -/* Routines for performing RFC2616 compliance. - RFC2616: - Hypertext Transfer Protocol -- HTTP/1.1 - http://www.w3.org/Protocols/rfc2616/rfc2616 -*/ - -struct ci_equal_pred -{ - explicit ci_equal_pred() = default; - - bool - operator()(char c1, char c2) - { - // VFALCO TODO Use a table lookup here - return std::tolower(static_cast(c1)) == - std::tolower(static_cast(c2)); - } -}; - -} // namespace detail - /** Returns `true` if `c` is linear white space. This excludes the CRLF sequence allowed for line continuations. @@ -86,57 +64,6 @@ is_white(char c) return false; } -/** Returns `true` if `c` is a control character. */ -inline bool -is_control(char c) -{ - return c <= 31 || c >= 127; -} - -/** Returns `true` if `c` is a separator. */ -inline bool -is_separator(char c) -{ - // VFALCO Could use a static table - switch (c) - { - case '(': - case ')': - case '<': - case '>': - case '@': - case ',': - case ';': - case ':': - case '\\': - case '"': - case '{': - case '}': - case ' ': - case '\t': - return true; - }; - return false; -} - -/** Returns `true` if `c` is a character. */ -inline bool -is_char(char c) -{ -#ifdef __CHAR_UNSIGNED__ /* -funsigned-char */ - return c >= 0 && c <= 127; -#else - return c >= 0; -#endif -} - -template -FwdIter -trim_left(FwdIter first, FwdIter last) -{ - return std::find_if_not(first, last, is_white); -} - template FwdIter trim_right(FwdIter first, FwdIter last) @@ -152,34 +79,6 @@ trim_right(FwdIter first, FwdIter last) return first; } -template -void -trim_right_in_place(std::basic_string& s) -{ - s.resize(std::distance(s.begin(), trim_right(s.begin(), s.end()))); -} - -template -std::pair -trim(FwdIter first, FwdIter last) -{ - first = trim_left(first, last); - last = trim_right(first, last); - return std::make_pair(first, last); -} - -template -String -trim(String const& s) -{ - using std::begin; - using std::end; - auto first = begin(s); - auto last = end(s); - std::tie(first, last) = trim(first, last); - return {first, last}; -} - template String trim_right(String const& s) @@ -192,11 +91,7 @@ trim_right(String const& s) return {first, last}; } -inline std::string -trim(std::string const& s) -{ - return trim(s); -} +} // namespace detail /** Parse a character sequence of values separated by commas. Double quotes and escape sequences will be converted. Excess white @@ -214,8 +109,11 @@ template < Result split(FwdIt first, FwdIt last, Char delim) { - Result result; + using namespace detail; using string = typename Result::value_type; + + Result result; + FwdIt iter = first; string e; while (iter != last) @@ -297,179 +195,6 @@ split_commas(boost::beast::string_view const& s) return split_commas(s.begin(), s.end()); } -//------------------------------------------------------------------------------ - -/** Iterates through a comma separated list. - - Meets the requirements of ForwardIterator. - - List defined in rfc2616 2.1. - - @note Values returned may contain backslash escapes. -*/ -class list_iterator -{ - using iter_type = boost::string_ref::const_iterator; - - iter_type it_; - iter_type end_; - boost::string_ref value_; - -public: - using value_type = boost::string_ref; - using pointer = value_type const*; - using reference = value_type const&; - using difference_type = std::ptrdiff_t; - using iterator_category = std::forward_iterator_tag; - - list_iterator(iter_type begin, iter_type end) : it_(begin), end_(end) - { - if (it_ != end_) - increment(); - } - - bool - operator==(list_iterator const& other) const - { - return other.it_ == it_ && other.end_ == end_ && - other.value_.size() == value_.size(); - } - - bool - operator!=(list_iterator const& other) const - { - return !(*this == other); - } - - reference - operator*() const - { - return value_; - } - - pointer - operator->() const - { - return &*(*this); - } - - list_iterator& - operator++() - { - increment(); - return *this; - } - - list_iterator - operator++(int) - { - auto temp = *this; - ++(*this); - return temp; - } - -private: - template - void - increment(); -}; - -template -void -list_iterator::increment() -{ - value_.clear(); - while (it_ != end_) - { - if (*it_ == '"') - { - // quoted-string - ++it_; - if (it_ == end_) - return; - if (*it_ != '"') - { - auto start = it_; - for (;;) - { - ++it_; - if (it_ == end_) - { - value_ = boost::string_ref( - &*start, std::distance(start, it_)); - return; - } - if (*it_ == '"') - { - value_ = boost::string_ref( - &*start, std::distance(start, it_)); - ++it_; - return; - } - } - } - ++it_; - } - else if (*it_ == ',') - { - it_++; - continue; - } - else if (is_lws(*it_)) - { - ++it_; - continue; - } - else - { - auto start = it_; - for (;;) - { - ++it_; - if (it_ == end_ || *it_ == ',' || is_lws(*it_)) - { - value_ = - boost::string_ref(&*start, std::distance(start, it_)); - return; - } - } - } - } -} - -/** Returns true if two strings are equal. - - A case-insensitive comparison is used. -*/ -inline bool -ci_equal(boost::string_ref s1, boost::string_ref s2) -{ - return boost::range::equal(s1, s2, detail::ci_equal_pred{}); -} - -/** Returns a range representing the list. */ -inline boost::iterator_range -make_list(boost::string_ref const& field) -{ - return boost::iterator_range{ - list_iterator{field.begin(), field.end()}, - list_iterator{field.end(), field.end()}}; -} - -/** Returns true if the specified token exists in the list. - - A case-insensitive comparison is used. -*/ -template -bool -token_in_list(boost::string_ref const& value, boost::string_ref const& token) -{ - for (auto const& item : make_list(value)) - if (ci_equal(item, token)) - return true; - return false; -} - template bool is_keep_alive(boost::beast::http::message const& m) diff --git a/src/ripple/consensus/Consensus.h b/src/ripple/consensus/Consensus.h index 5f88033ecfb..4b64cfbdad5 100644 --- a/src/ripple/consensus/Consensus.h +++ b/src/ripple/consensus/Consensus.h @@ -738,8 +738,7 @@ Consensus::peerProposalInternal( if (deadNodes_.find(peerID) != deadNodes_.end()) { - using std::to_string; - JLOG(j_.info()) << "Position from dead node: " << to_string(peerID); + JLOG(j_.info()) << "Position from dead node: " << peerID; return false; } @@ -758,9 +757,7 @@ Consensus::peerProposalInternal( if (newPeerProp.isBowOut()) { - using std::to_string; - - JLOG(j_.info()) << "Peer bows out: " << to_string(peerID); + JLOG(j_.info()) << "Peer " << peerID << " bows out"; if (result_) { for (auto& it : result_->disputes) @@ -1559,10 +1556,7 @@ Consensus::haveConsensus() } else { - using std::to_string; - - JLOG(j_.debug()) << to_string(nodeId) << " has " - << to_string(peerProp.position()); + JLOG(j_.debug()) << nodeId << " has " << peerProp.position(); ++disagree; } } diff --git a/src/ripple/core/Config.h b/src/ripple/core/Config.h index f66bf3b8bda..bf282f1a13e 100644 --- a/src/ripple/core/Config.h +++ b/src/ripple/core/Config.h @@ -143,7 +143,14 @@ class Config : public BasicConfig // True to ask peers not to relay current IP. bool PEER_PRIVATE = false; + // peers_max is a legacy configuration, which is going to be replaced + // with individual inbound peers peers_in_max and outbound peers + // peers_out_max configuration. for now we support both the legacy and + // the new configuration. if peers_max is configured then peers_in_max and + // peers_out_max are ignored. std::size_t PEERS_MAX = 0; + std::size_t PEERS_OUT_MAX = 0; + std::size_t PEERS_IN_MAX = 0; std::chrono::seconds WEBSOCKET_PING_FREQ = std::chrono::minutes{5}; @@ -198,6 +205,12 @@ class Config : public BasicConfig std::string SERVER_DOMAIN; + // How long can a peer remain in the "unknown" state + std::chrono::seconds MAX_UNKNOWN_TIME{600}; + + // How long can a peer remain in the "diverged" state + std::chrono::seconds MAX_DIVERGED_TIME{300}; + public: Config() : j_{beast::Journal::getNullSink()} { diff --git a/src/ripple/core/ConfigSections.h b/src/ripple/core/ConfigSections.h index e3b864d8583..9791c073473 100644 --- a/src/ripple/core/ConfigSections.h +++ b/src/ripple/core/ConfigSections.h @@ -66,12 +66,15 @@ struct ConfigSection #define SECTION_NETWORK_QUORUM "network_quorum" #define SECTION_NODE_SEED "node_seed" #define SECTION_NODE_SIZE "node_size" +#define SECTION_OVERLAY "overlay" #define SECTION_PATH_SEARCH_OLD "path_search_old" #define SECTION_PATH_SEARCH "path_search" #define SECTION_PATH_SEARCH_FAST "path_search_fast" #define SECTION_PATH_SEARCH_MAX "path_search_max" #define SECTION_PEER_PRIVATE "peer_private" #define SECTION_PEERS_MAX "peers_max" +#define SECTION_PEERS_IN_MAX "peers_in_max" +#define SECTION_PEERS_OUT_MAX "peers_out_max" #define SECTION_REDUCE_RELAY "reduce_relay" #define SECTION_RELAY_PROPOSALS "relay_proposals" #define SECTION_RELAY_VALIDATIONS "relay_validations" diff --git a/src/ripple/core/impl/Config.cpp b/src/ripple/core/impl/Config.cpp index fe0da432ace..c4f28873383 100644 --- a/src/ripple/core/impl/Config.cpp +++ b/src/ripple/core/impl/Config.cpp @@ -366,7 +366,45 @@ Config::loadFromString(std::string const& fileContents) PEER_PRIVATE = beast::lexicalCastThrow(strTemp); if (getSingleSection(secConfig, SECTION_PEERS_MAX, strTemp, j_)) + { PEERS_MAX = beast::lexicalCastThrow(strTemp); + } + else + { + std::optional peers_in_max{}; + if (getSingleSection(secConfig, SECTION_PEERS_IN_MAX, strTemp, j_)) + { + peers_in_max = beast::lexicalCastThrow(strTemp); + if (*peers_in_max > 1000) + Throw( + "Invalid value specified in [" SECTION_PEERS_IN_MAX + "] section; the value must be less or equal than 1000"); + } + + std::optional peers_out_max{}; + if (getSingleSection(secConfig, SECTION_PEERS_OUT_MAX, strTemp, j_)) + { + peers_out_max = beast::lexicalCastThrow(strTemp); + if (*peers_out_max < 10 || *peers_out_max > 1000) + Throw( + "Invalid value specified in [" SECTION_PEERS_OUT_MAX + "] section; the value must be in range 10-1000"); + } + + // if one section is configured then the other must be configured too + if ((peers_in_max && !peers_out_max) || + (peers_out_max && !peers_in_max)) + Throw("Both sections [" SECTION_PEERS_IN_MAX + "]" + "and [" SECTION_PEERS_OUT_MAX + "] must be configured"); + + if (peers_in_max && peers_out_max) + { + PEERS_IN_MAX = *peers_in_max; + PEERS_OUT_MAX = *peers_out_max; + } + } if (getSingleSection(secConfig, SECTION_NODE_SIZE, strTemp, j_)) { @@ -427,8 +465,7 @@ Config::loadFromString(std::string const& fileContents) if (exists(SECTION_VALIDATION_SEED) && exists(SECTION_VALIDATOR_TOKEN)) Throw("Cannot have both [" SECTION_VALIDATION_SEED - "] " - "and [" SECTION_VALIDATOR_TOKEN + "] and [" SECTION_VALIDATOR_TOKEN "] config sections"); if (getSingleSection(secConfig, SECTION_NETWORK_QUORUM, strTemp, j_)) @@ -512,6 +549,51 @@ Config::loadFromString(std::string const& fileContents) SERVER_DOMAIN = strTemp; } + if (exists(SECTION_OVERLAY)) + { + auto const sec = section(SECTION_OVERLAY); + + using namespace std::chrono; + + try + { + if (auto val = sec.get("max_unknown_time")) + MAX_UNKNOWN_TIME = + seconds{beast::lexicalCastThrow(*val)}; + } + catch (...) + { + Throw( + "Invalid value 'max_unknown_time' in " SECTION_OVERLAY + ": must be of the form '' representing seconds."); + } + + if (MAX_UNKNOWN_TIME < seconds{300} || MAX_UNKNOWN_TIME > seconds{1800}) + Throw( + "Invalid value 'max_unknown_time' in " SECTION_OVERLAY + ": the time must be between 300 and 1800 seconds, inclusive."); + + try + { + if (auto val = sec.get("max_diverged_time")) + MAX_DIVERGED_TIME = + seconds{beast::lexicalCastThrow(*val)}; + } + catch (...) + { + Throw( + "Invalid value 'max_diverged_time' in " SECTION_OVERLAY + ": must be of the form '' representing seconds."); + } + + if (MAX_DIVERGED_TIME < seconds{60} || MAX_DIVERGED_TIME > seconds{900}) + { + Throw( + "Invalid value 'max_diverged_time' in " SECTION_OVERLAY + ": the time must be between 60 and 900 seconds, inclusive."); + } + } + if (getSingleSection( secConfig, SECTION_AMENDMENT_MAJORITY_TIME, strTemp, j_)) { diff --git a/src/ripple/json/impl/json_value.cpp b/src/ripple/json/impl/json_value.cpp index aaf8a23b214..024ad2c88a0 100644 --- a/src/ripple/json/impl/json_value.cpp +++ b/src/ripple/json/impl/json_value.cpp @@ -480,13 +480,13 @@ Value::asString() const return value_.bool_ ? "true" : "false"; case intValue: - return beast::lexicalCastThrow(value_.int_); + return std::to_string(value_.int_); case uintValue: - return beast::lexicalCastThrow(value_.uint_); + return std::to_string(value_.uint_); case realValue: - return beast::lexicalCastThrow(value_.real_); + return std::to_string(value_.real_); case arrayValue: case objectValue: diff --git a/src/ripple/ledger/detail/ReadViewFwdRange.h b/src/ripple/ledger/detail/ReadViewFwdRange.h index 3e7408568e7..af00944cfe8 100644 --- a/src/ripple/ledger/detail/ReadViewFwdRange.h +++ b/src/ripple/ledger/detail/ReadViewFwdRange.h @@ -141,10 +141,6 @@ class ReadViewFwdRange ReadViewFwdRange& operator=(ReadViewFwdRange const&) = default; - // VFALCO Otherwise causes errors on clang - // private: - // friend class ReadView; - explicit ReadViewFwdRange(ReadView const& view) : view_(&view) { } diff --git a/src/ripple/net/InfoSub.h b/src/ripple/net/InfoSub.h index aaf6293d2b6..b1ef6e25be1 100644 --- a/src/ripple/net/InfoSub.h +++ b/src/ripple/net/InfoSub.h @@ -40,12 +40,6 @@ class PathRequest; class InfoSub : public CountedObject { public: - static char const* - getCountedObjectName() - { - return "InfoSub"; - } - using pointer = std::shared_ptr; // VFALCO TODO Standardize on the names of weak / strong pointer type diff --git a/src/ripple/net/impl/HTTPClient.cpp b/src/ripple/net/impl/HTTPClient.cpp index 79779065ce0..c9f136cb0dc 100644 --- a/src/ripple/net/impl/HTTPClient.cpp +++ b/src/ripple/net/impl/HTTPClient.cpp @@ -144,7 +144,7 @@ class HTTPClientImp : public std::enable_shared_from_this, auto query = std::make_shared( mDeqSites[0], - beast::lexicalCast(mPort), + std::to_string(mPort), boost::asio::ip::resolver_query_base::numeric_service); mQuery = query; diff --git a/src/ripple/net/impl/RPCCall.cpp b/src/ripple/net/impl/RPCCall.cpp index 815ccf0ac5f..9a79515a252 100644 --- a/src/ripple/net/impl/RPCCall.cpp +++ b/src/ripple/net/impl/RPCCall.cpp @@ -818,7 +818,7 @@ class RPCParser { // verify the channel id is a valid 256 bit number uint256 channelId; - if (!channelId.SetHexExact(jvParams[index].asString())) + if (!channelId.parseHex(jvParams[index].asString())) return rpcError(rpcCHANNEL_MALFORMED); jvRequest[jss::channel_id] = to_string(channelId); index++; @@ -850,7 +850,7 @@ class RPCParser { // verify the channel id is a valid 256 bit number uint256 channelId; - if (!channelId.SetHexExact(jvParams[1u].asString())) + if (!channelId.parseHex(jvParams[1u].asString())) return rpcError(rpcCHANNEL_MALFORMED); } jvRequest[jss::channel_id] = jvParams[1u].asString(); diff --git a/src/ripple/nodestore/NodeObject.h b/src/ripple/nodestore/NodeObject.h index 7c33169939b..2bd73d8dee5 100644 --- a/src/ripple/nodestore/NodeObject.h +++ b/src/ripple/nodestore/NodeObject.h @@ -48,12 +48,6 @@ enum NodeObjectType : std::uint32_t { class NodeObject : public CountedObject { public: - static char const* - getCountedObjectName() - { - return "NodeObject"; - } - static constexpr std::size_t keyBytes = 32; private: diff --git a/src/ripple/nodestore/backend/RocksDBFactory.cpp b/src/ripple/nodestore/backend/RocksDBFactory.cpp index 4fc36b7e1b7..04b9699902f 100644 --- a/src/ripple/nodestore/backend/RocksDBFactory.cpp +++ b/src/ripple/nodestore/backend/RocksDBFactory.cpp @@ -376,8 +376,7 @@ class RocksDBBackend : public Backend, public BatchWriter::Callback { // Uh oh, corrupted data! JLOG(m_journal.fatal()) - << "Corrupt NodeObject #" - << from_hex_text(it->key().data()); + << "Corrupt NodeObject #" << it->key().ToString(true); } } else diff --git a/src/ripple/nodestore/impl/Database.cpp b/src/ripple/nodestore/impl/Database.cpp index 178d4bb7f75..3bec8dfffc6 100644 --- a/src/ripple/nodestore/impl/Database.cpp +++ b/src/ripple/nodestore/impl/Database.cpp @@ -244,11 +244,11 @@ Database::storeLedger( } bool error = false; - auto visit = [&](SHAMapAbstractNode& node) { + auto visit = [&](SHAMapTreeNode& node) { if (!isStopping()) { if (auto nodeObject = srcDB.fetchNodeObject( - node.getNodeHash().as_uint256(), srcLedger.info().seq)) + node.getHash().as_uint256(), srcLedger.info().seq)) { batch.emplace_back(std::move(nodeObject)); if (batch.size() < batchWritePreallocationSize || storeBatch()) diff --git a/src/ripple/nodestore/impl/DatabaseShardImp.cpp b/src/ripple/nodestore/impl/DatabaseShardImp.cpp index f05ddf9e563..27a933c7b25 100644 --- a/src/ripple/nodestore/impl/DatabaseShardImp.cpp +++ b/src/ripple/nodestore/impl/DatabaseShardImp.cpp @@ -939,6 +939,9 @@ DatabaseShardImp::import(Database& source) true, boost::none); success = true; + + if (shardIndex < shardBoundaryIndex()) + ++numHistShards; } catch (std::exception const& e) { @@ -1457,10 +1460,20 @@ DatabaseShardImp::setFileStats() fdRequired_ = sumFd; avgShardFileSz_ = (numShards == 0 ? fileSz_ : fileSz_ / numShards); + if (!canAdd_) + return; + if (auto const count = numHistoricalShards(lock); count >= maxHistoricalShards_) { - JLOG(j_.warn()) << "maximum number of historical shards reached"; + if (maxHistoricalShards_) + { + // In order to avoid excessive output, don't produce + // this warning if the server isn't configured to + // store historical shards. + JLOG(j_.warn()) << "maximum number of historical shards reached"; + } + canAdd_ = false; } else if (!sufficientStorage( @@ -1470,6 +1483,8 @@ DatabaseShardImp::setFileStats() { JLOG(j_.warn()) << "maximum shard store size exceeds available storage space"; + + canAdd_ = false; } } @@ -1609,14 +1624,17 @@ DatabaseShardImp::removeFailedShard(std::shared_ptr& shard) std::uint32_t DatabaseShardImp::shardBoundaryIndex() const { + auto const validIndex = app_.getLedgerMaster().getValidLedgerIndex(); + + if (validIndex < earliestLedgerSeq()) + return 0; + // Shards with an index earlier than the recent shard boundary index // are considered historical. The three shards at or later than // this index consist of the two most recently validated shards // and the shard still in the process of being built by live // transactions. - return NodeStore::seqToShardIndex( - app_.getLedgerMaster().getValidLedgerIndex(), ledgersPerShard_) - - 1; + return seqToShardIndex(validIndex) - 1; } std::uint32_t diff --git a/src/ripple/nodestore/impl/Shard.cpp b/src/ripple/nodestore/impl/Shard.cpp index a7f49bebe39..7694362f133 100644 --- a/src/ripple/nodestore/impl/Shard.cpp +++ b/src/ripple/nodestore/impl/Shard.cpp @@ -405,11 +405,11 @@ Shard::storeLedger( } bool error = false; - auto visit = [&](SHAMapAbstractNode& node) { + auto visit = [&](SHAMapTreeNode const& node) { if (!stop_) { if (auto nodeObject = srcDB.fetchNodeObject( - node.getNodeHash().as_uint256(), srcLedger->info().seq)) + node.getHash().as_uint256(), srcLedger->info().seq)) { batch.emplace_back(std::move(nodeObject)); if (batch.size() < batchWritePreallocationSize || storeBatch()) @@ -671,7 +671,7 @@ Shard::finalize( if (!sHash) return fail("missing LastLedgerHash"); - if (!hash.SetHexExact(*sHash) || hash.isZero()) + if (!hash.parseHex(*sHash) || hash.isZero()) return fail("invalid LastLedgerHash"); if (blobPresent != soci::i_ok) @@ -1159,7 +1159,7 @@ Shard::storeSQLite( session << (STTx::getMetaSQLInsertReplaceHeader() + item.first->getMetaSQL( - ledgerSeq, sqlEscape(s.modData())) + + ledgerSeq, sqlBlobLiteral(s.modData())) + ';'); } } @@ -1288,10 +1288,10 @@ Shard::verifyLedger( return fail("Invalid ledger account hash"); bool error{false}; - auto visit = [this, &error](SHAMapAbstractNode& node) { + auto visit = [this, &error](SHAMapTreeNode const& node) { if (stop_) return false; - if (!verifyFetch(node.getNodeHash().as_uint256())) + if (!verifyFetch(node.getHash().as_uint256())) error = true; return !error; }; diff --git a/src/ripple/overlay/Message.h b/src/ripple/overlay/Message.h index 724fad07e6e..5ce66585897 100644 --- a/src/ripple/overlay/Message.h +++ b/src/ripple/overlay/Message.h @@ -64,6 +64,10 @@ class Message : public std::enable_shared_from_this int type, boost::optional const& validator = {}); + /** Retrieve the size of the packed but uncompressed message data. */ + std::size_t + getBufferSize(); + /** Retrieve the packed message data. If compressed message is requested but * the message is not compressible then the uncompressed buffer is returned. * @param compressed Request compressed (Compress::On) or diff --git a/src/ripple/overlay/Overlay.h b/src/ripple/overlay/Overlay.h index fd1b9ca34f9..cf37cf72bf3 100644 --- a/src/ripple/overlay/Overlay.h +++ b/src/ripple/overlay/Overlay.h @@ -120,16 +120,11 @@ class Overlay : public Stoppable, public beast::PropertyStream::Source virtual PeerSequence getActivePeers() const = 0; - /** Calls the checkSanity function on each peer - @param index the value to pass to the peer's checkSanity function + /** Calls the checkTracking function on each peer + @param index the value to pass to the peer's checkTracking function */ virtual void - checkSanity(std::uint32_t index) = 0; - - /** Calls the check function on each peer - */ - virtual void - check() = 0; + checkTracking(std::uint32_t index) = 0; /** Returns the peer with the matching short id, or null. */ virtual std::shared_ptr diff --git a/src/ripple/overlay/impl/Message.cpp b/src/ripple/overlay/impl/Message.cpp index b5b24c3d3e4..d5aec1a2d96 100644 --- a/src/ripple/overlay/impl/Message.cpp +++ b/src/ripple/overlay/impl/Message.cpp @@ -177,6 +177,12 @@ Message::setHeader( } } +std::size_t +Message::getBufferSize() +{ + return buffer_.size(); +} + std::vector const& Message::getBuffer(Compressed tryCompressed) { diff --git a/src/ripple/overlay/impl/OverlayImpl.cpp b/src/ripple/overlay/impl/OverlayImpl.cpp index af8c90711ed..489a69ed7c6 100644 --- a/src/ripple/overlay/impl/OverlayImpl.cpp +++ b/src/ripple/overlay/impl/OverlayImpl.cpp @@ -100,9 +100,6 @@ OverlayImpl::Timer::on_timer(error_code ec) overlay_.sendEndpoints(); overlay_.autoConnect(); - if ((++overlay_.timer_count_ % Tuning::checkSeconds) == 0) - overlay_.check(); - if ((overlay_.timer_count_ % Tuning::checkIdlePeers) == 0) overlay_.deleteIdlePeers(); @@ -499,37 +496,11 @@ OverlayImpl::checkStopped() void OverlayImpl::onPrepare() { - PeerFinder::Config config; - - if (app_.config().PEERS_MAX != 0) - config.maxPeers = app_.config().PEERS_MAX; - - config.outPeers = config.calcOutPeers(); - - auto const port = serverHandler_.setup().overlay.port; - - config.peerPrivate = app_.config().PEER_PRIVATE; - - // Servers with peer privacy don't want to allow incoming connections - config.wantIncoming = (!config.peerPrivate) && (port != 0); - - // This will cause servers configured as validators to request that - // peers they connect to never report their IP address. We set this - // after we set the 'wantIncoming' because we want a "soft" version - // of peer privacy unless the operator explicitly asks for it. - if (!app_.getValidationPublicKey().empty()) - config.peerPrivate = true; - - // if it's a private peer or we are running as standalone - // automatic connections would defeat the purpose. - config.autoConnect = - !app_.config().standalone() && !app_.config().PEER_PRIVATE; - config.listeningPort = port; - config.features = ""; - config.ipLimit = setup_.ipLimit; - - // Enforce business rules - config.applyTuning(); + PeerFinder::Config config = PeerFinder::Config::makeConfig( + app_.config(), + serverHandler_.setup().overlay.port, + !app_.getValidationPublicKey().empty(), + setup_.ipLimit); m_peerFinder->setConfig(config); @@ -1207,16 +1178,10 @@ OverlayImpl::getActivePeers() const } void -OverlayImpl::checkSanity(std::uint32_t index) +OverlayImpl::checkTracking(std::uint32_t index) { for_each( - [index](std::shared_ptr&& sp) { sp->checkSanity(index); }); -} - -void -OverlayImpl::check() -{ - for_each([](std::shared_ptr&& sp) { sp->check(); }); + [index](std::shared_ptr&& sp) { sp->checkTracking(index); }); } std::shared_ptr @@ -1298,6 +1263,36 @@ OverlayImpl::relay( return {}; } +std::shared_ptr +OverlayImpl::getManifestsMessage() +{ + std::lock_guard g(manifestLock_); + + if (auto seq = app_.validatorManifests().sequence(); + seq != manifestListSeq_) + { + protocol::TMManifests tm; + + app_.validatorManifests().for_each_manifest( + [&tm](std::size_t s) { tm.mutable_list()->Reserve(s); }, + [&tm, &hr = app_.getHashRouter()](Manifest const& manifest) { + tm.add_list()->set_stobject( + manifest.serialized.data(), manifest.serialized.size()); + hr.addSuppression(manifest.hash()); + }); + + manifestMessage_.reset(); + + if (tm.list_size() != 0) + manifestMessage_ = + std::make_shared(tm, protocol::mtMANIFESTS); + + manifestListSeq_ = seq; + } + + return manifestMessage_; +} + //------------------------------------------------------------------------------ void diff --git a/src/ripple/overlay/impl/OverlayImpl.h b/src/ripple/overlay/impl/OverlayImpl.h index abbae3c925a..65f5ffc52f4 100644 --- a/src/ripple/overlay/impl/OverlayImpl.h +++ b/src/ripple/overlay/impl/OverlayImpl.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -127,6 +128,13 @@ class OverlayImpl : public Overlay, public squelch::SquelchHandler squelch::Slots slots_; + // A message with the list of manifests we send to peers + std::shared_ptr manifestMessage_; + // Used to track whether we need to update the cached list of manifests + std::optional manifestListSeq_; + // Protects the message and the sequence list of manifests + std::mutex manifestLock_; + //-------------------------------------------------------------------------- public: @@ -192,10 +200,7 @@ class OverlayImpl : public Overlay, public squelch::SquelchHandler PeerSequence getActivePeers() const override; - void - check() override; - - void checkSanity(std::uint32_t) override; + void checkTracking(std::uint32_t) override; std::shared_ptr findPeerByShortID(Peer::id_t const& id) const override; @@ -221,6 +226,9 @@ class OverlayImpl : public Overlay, public squelch::SquelchHandler uint256 const& uid, PublicKey const& validator) override; + std::shared_ptr + getManifestsMessage(); + //-------------------------------------------------------------------------- // // OverlayImpl diff --git a/src/ripple/overlay/impl/PeerImp.cpp b/src/ripple/overlay/impl/PeerImp.cpp index 8ed6955f1a5..7a2834dd4a0 100644 --- a/src/ripple/overlay/impl/PeerImp.cpp +++ b/src/ripple/overlay/impl/PeerImp.cpp @@ -47,6 +47,7 @@ #include #include +#include #include #include @@ -54,6 +55,14 @@ using namespace std::chrono_literals; namespace ripple { +namespace { +/** The threshold above which we treat a peer connection as high latency */ +std::chrono::milliseconds constexpr peerHighLatency{300}; + +/** How often we PING the peer to check for latency and sendq probe */ +std::chrono::seconds constexpr peerTimerInterval{60}; +} // namespace + PeerImp::PeerImp( Application& app, id_t id, @@ -78,12 +87,12 @@ PeerImp::PeerImp( , timer_(waitable_timer{socket_.get_executor()}) , remote_address_(slot->remote_endpoint()) , overlay_(overlay) - , m_inbound(true) + , inbound_(true) , protocol_(protocol) - , state_(State::active) - , sanity_(Sanity::unknown) - , insaneTime_(clock_type::now()) + , tracking_(Tracking::unknown) + , trackingTime_(clock_type::now()) , publicKey_(publicKey) + , lastPingTime_(clock_type::now()) , creationTime_(clock_type::now()) , usage_(consumer) , fee_(Resource::feeLightPeer) @@ -101,8 +110,7 @@ PeerImp::~PeerImp() const bool inCluster{cluster()}; overlay_.deletePeer(id_); - if (state_ == State::active) - overlay_.onPeerDeactivate(id_); + overlay_.onPeerDeactivate(id_); overlay_.peerFinder().on_closed(slot_); overlay_.remove(slot_); @@ -125,17 +133,15 @@ PeerImp::run() if (!strand_.running_in_this_thread()) return post(strand_, std::bind(&PeerImp::run, shared_from_this())); - // We need to decipher auto parseLedgerHash = [](std::string const& value) -> boost::optional { - uint256 ret; - if (ret.SetHexExact(value)) - return {ret}; - - auto const s = base64_decode(value); - if (s.size() != uint256::size()) - return boost::none; - return uint256{s}; + if (uint256 ret; ret.parseHex(value)) + return ret; + + if (auto const s = base64_decode(value); s.size() == uint256::size()) + return uint256{s}; + + return boost::none; }; boost::optional closed; @@ -170,18 +176,10 @@ PeerImp::run() previousLedgerHash_ = *previous; } - if (m_inbound) - { + if (inbound_) doAccept(); - } else - { - assert(state_ == State::active); - // XXX Set timer: connection is in grace period to be useful. - // XXX Set timer: connection idle (idle may vary depending on connection - // type.) doProtocolStart(); - } // Request shard info from peer protocol::TMGetPeerShardInfo tmGPS; @@ -203,7 +201,7 @@ PeerImp::stop() // at a higher level, but inbound connections are more numerous and // uncontrolled so to prevent log flooding the severity is reduced. // - if (m_inbound) + if (inbound_) { JLOG(journal_.debug()) << "Stop"; } @@ -304,7 +302,7 @@ PeerImp::cluster() const std::string PeerImp::getVersion() const { - if (m_inbound) + if (inbound_) return headers_["User-Agent"].to_string(); return headers_["Server"].to_string(); } @@ -317,7 +315,7 @@ PeerImp::json() ret[jss::public_key] = toBase58(TokenType::NodePublic, publicKey_); ret[jss::address] = remote_address_.to_string(); - if (m_inbound) + if (inbound_) ret[jss::inbound] = true; if (cluster()) @@ -358,17 +356,17 @@ PeerImp::json() ret[jss::complete_ledgers] = std::to_string(minSeq) + " - " + std::to_string(maxSeq); - switch (sanity_.load()) + switch (tracking_.load()) { - case Sanity::insane: - ret[jss::sanity] = "insane"; + case Tracking::diverged: + ret[jss::track] = "diverged"; break; - case Sanity::unknown: - ret[jss::sanity] = "unknown"; + case Tracking::unknown: + ret[jss::track] = "unknown"; break; - case Sanity::sane: + case Tracking::converged: // Nothing to do here break; } @@ -446,7 +444,7 @@ PeerImp::hasLedger(uint256 const& hash, std::uint32_t seq) const { std::lock_guard sl(recentLock_); if ((seq != 0) && (seq >= minLedger_) && (seq <= maxLedger_) && - (sanity_.load() == Sanity::sane)) + (tracking_.load() == Tracking::converged)) return true; if (std::find(recentLedgers_.begin(), recentLedgers_.end(), hash) != recentLedgers_.end()) @@ -498,7 +496,7 @@ bool PeerImp::hasRange(std::uint32_t uMin, std::uint32_t uMax) { std::lock_guard sl(recentLock_); - return (sanity_ != Sanity::insane) && (uMin >= minLedger_) && + return (tracking_ != Tracking::diverged) && (uMin >= minLedger_) && (uMax <= maxLedger_); } @@ -515,7 +513,7 @@ PeerImp::close() timer_.cancel(ec); socket_.close(ec); overlay_.incPeerDisconnect(); - if (m_inbound) + if (inbound_) { JLOG(journal_.debug()) << "Closed"; } @@ -602,7 +600,7 @@ void PeerImp::setTimer() { error_code ec; - timer_.expires_from_now(std::chrono::seconds(Tuning::timerSeconds), ec); + timer_.expires_from_now(peerTimerInterval, ec); if (ec) { @@ -655,49 +653,41 @@ PeerImp::onTimer(error_code const& ec) return; } - bool failedNoPing{false}; - boost::optional pingSeq; - // Operations on lastPingSeq_, lastPingTime_, no_ping_, and latency_ - // must be guarded by recentLock_. + if (auto const t = tracking_.load(); !inbound_ && t != Tracking::converged) { - std::lock_guard sl(recentLock_); - if (no_ping_++ >= Tuning::noPing) - { - failedNoPing = true; - } - else if (!lastPingSeq_) + clock_type::duration duration; + { - // Make the sequence unpredictable enough to prevent guessing - lastPingSeq_ = rand_int(); - lastPingTime_ = clock_type::now(); - pingSeq = lastPingSeq_; + std::lock_guard sl(recentLock_); + duration = clock_type::now() - trackingTime_; } - else - { - // We have an outstanding ping, raise latency - auto const minLatency = - std::chrono::duration_cast( - clock_type::now() - lastPingTime_); - if (latency_ < minLatency) - latency_ = minLatency; + if ((t == Tracking::diverged && + (duration > app_.config().MAX_DIVERGED_TIME)) || + (t == Tracking::unknown && + (duration > app_.config().MAX_UNKNOWN_TIME))) + { + overlay_.peerFinder().on_failure(slot_); + fail("Not useful"); + return; } } - if (failedNoPing) + // Already waiting for PONG + if (lastPingSeq_) { - fail("No ping reply received"); + fail("Ping Timeout"); return; } - if (pingSeq) - { - protocol::TMPing message; - message.set_type(protocol::TMPing::ptPING); - message.set_seq(*pingSeq); + lastPingTime_ = clock_type::now(); + lastPingSeq_ = rand_int(); - send(std::make_shared(message, protocol::mtPING)); - } + protocol::TMPing message; + message.set_type(protocol::TMPing::ptPING); + message.set_seq(*lastPingSeq_); + + send(std::make_shared(message, protocol::mtPING)); setTimer(); } @@ -718,7 +708,6 @@ PeerImp::onShutdown(error_code ec) } //------------------------------------------------------------------------------ - void PeerImp::doAccept() { @@ -733,14 +722,6 @@ PeerImp::doAccept() if (!sharedValue) return fail("makeSharedValue: Unexpected failure"); - // TODO Apply headers to connection state. - - boost::beast::ostream(write_buffer_) << makeResponse( - !overlay_.peerFinder().config().peerPrivate, - request_, - remote_address_.address(), - *sharedValue); - JLOG(journal_.info()) << "Protocol: " << to_string(protocol_); JLOG(journal_.info()) << "Public Key: " << toBase58(TokenType::NodePublic, publicKey_); @@ -748,7 +729,7 @@ PeerImp::doAccept() if (auto member = app_.cluster().member(publicKey_)) { { - std::unique_lock lock{nameMutex_}; + std::unique_lock lock{nameMutex_}; name_ = *member; } JLOG(journal_.info()) << "Cluster name: " << *member; @@ -760,75 +741,60 @@ PeerImp::doAccept() // XXX Set timer: connection idle (idle may vary depending on connection // type.) - onWriteResponse(error_code(), 0); -} - -http_response_type -PeerImp::makeResponse( - bool crawl, - http_request_type const& req, - beast::IP::Address remote_ip, - uint256 const& sharedValue) -{ - http_response_type resp; - resp.result(boost::beast::http::status::switching_protocols); - resp.version(req.version()); - resp.insert("Connection", "Upgrade"); - resp.insert("Upgrade", to_string(protocol_)); - resp.insert("Connect-As", "Peer"); - resp.insert("Server", BuildInfo::getFullVersionString()); - resp.insert("Crawl", crawl ? "public" : "private"); - if (req["X-Offer-Compression"] == "lz4" && app_.config().COMPRESSION) - resp.insert("X-Offer-Compression", "lz4"); - - buildHandshake( - resp, - sharedValue, - overlay_.setup().networkID, - overlay_.setup().public_ip, - remote_ip, - app_); - - return resp; -} - -// Called repeatedly to send the bytes in the response -void -PeerImp::onWriteResponse(error_code ec, std::size_t bytes_transferred) -{ - if (!socket_.is_open()) - return; - if (ec == boost::asio::error::operation_aborted) - return; - if (ec) - return fail("onWriteResponse", ec); - if (auto stream = journal_.trace()) - { - if (bytes_transferred > 0) - stream << "onWriteResponse: " << bytes_transferred << " bytes"; - else - stream << "onWriteResponse"; - } - - write_buffer_.consume(bytes_transferred); - if (write_buffer_.size() == 0) - return doProtocolStart(); - - stream_.async_write_some( - write_buffer_.data(), - bind_executor( - strand_, - std::bind( - &PeerImp::onWriteResponse, - shared_from_this(), - std::placeholders::_1, - std::placeholders::_2))); + auto write_buffer = [this, sharedValue]() { + auto buf = std::make_shared(); + + http_response_type resp; + resp.result(boost::beast::http::status::switching_protocols); + resp.version(request_.version()); + resp.insert("Connection", "Upgrade"); + resp.insert("Upgrade", to_string(protocol_)); + resp.insert("Connect-As", "Peer"); + resp.insert("Server", BuildInfo::getFullVersionString()); + resp.insert( + "Crawl", + overlay_.peerFinder().config().peerPrivate ? "private" : "public"); + + if (request_["X-Offer-Compression"] == "lz4" && + app_.config().COMPRESSION) + resp.insert("X-Offer-Compression", "lz4"); + + buildHandshake( + resp, + *sharedValue, + overlay_.setup().networkID, + overlay_.setup().public_ip, + remote_address_.address(), + app_); + + boost::beast::ostream(*buf) << resp; + + return buf; + }(); + + // Write the whole buffer and only start protocol when that's done. + boost::asio::async_write( + stream_, + write_buffer->data(), + boost::asio::transfer_all(), + [this, write_buffer, self = shared_from_this()]( + error_code ec, std::size_t bytes_transferred) { + if (!socket_.is_open()) + return; + if (ec == boost::asio::error::operation_aborted) + return; + if (ec) + return fail("onWriteResponse", ec); + if (write_buffer->size() == bytes_transferred) + return doProtocolStart(); + return fail("Failed to write header"); + }); } std::string PeerImp::name() const { - std::shared_lock read_lock{nameMutex_}; + std::shared_lock read_lock{nameMutex_}; return name_; } @@ -868,30 +834,15 @@ PeerImp::doProtocolStart() << "Sending validator list for " << strHex(pubKey) << " with sequence " << sequence << " to " << remote_address_.to_string() << " (" << id_ << ")"; - auto m = std::make_shared(vl, protocol::mtVALIDATORLIST); - send(m); + send(std::make_shared(vl, protocol::mtVALIDATORLIST)); // Don't send it next time. app_.getHashRouter().addSuppressionPeer(hash, id_); setPublisherListSequence(pubKey, sequence); }); } - protocol::TMManifests tm; - - app_.validatorManifests().for_each_manifest( - [&tm](std::size_t s) { tm.mutable_list()->Reserve(s); }, - [&tm, &hr = app_.getHashRouter()](Manifest const& manifest) { - auto const& s = manifest.serialized; - auto& tm_e = *tm.add_list(); - tm_e.set_stobject(s.data(), s.size()); - hr.addSuppression(manifest.hash()); - }); - - if (tm.list_size() > 0) - { - auto m = std::make_shared(tm, protocol::mtMANIFESTS); + if (auto m = overlay_.getManifestsMessage()) send(m); - } } // Called repeatedly with protocol message data @@ -921,11 +872,13 @@ PeerImp::onReadMessage(error_code ec, std::size_t bytes_transferred) read_buffer_.commit(bytes_transferred); + auto hint = Tuning::readBufferBytes; + while (read_buffer_.size() > 0) { std::size_t bytes_consumed; std::tie(bytes_consumed, ec) = - invokeProtocolMessage(read_buffer_.data(), *this); + invokeProtocolMessage(read_buffer_.data(), *this, hint); if (ec) return fail("onReadMessage", ec); if (!socket_.is_open()) @@ -936,9 +889,10 @@ PeerImp::onReadMessage(error_code ec, std::size_t bytes_transferred) break; read_buffer_.consume(bytes_consumed); } + // Timeout on writes only stream_.async_read_some( - read_buffer_.prepare(Tuning::readBufferBytes), + read_buffer_.prepare(std::max(Tuning::readBufferBytes, hint)), bind_executor( strand_, std::bind( @@ -1053,31 +1007,25 @@ PeerImp::onMessage(std::shared_ptr const& m) return; } - if (m->type() == protocol::TMPing::ptPONG) + if (m->type() == protocol::TMPing::ptPONG && m->has_seq()) { - // Operations on lastPingSeq_, lastPingTime_, no_ping_, and latency_ - // must be guarded by recentLock_. - std::lock_guard sl(recentLock_); - - if (m->has_seq() && m->seq() == lastPingSeq_) + // Only reset the ping sequence if we actually received a + // PONG with the correct cookie. That way, any peers which + // respond with incorrect cookies will eventually time out. + if (m->seq() == lastPingSeq_) { - no_ping_ = 0; - - // Only reset the ping sequence if we actually received a - // PONG with the correct cookie. That way, any peers which - // respond with incorrect cookies will eventually time out. lastPingSeq_.reset(); // Update latency estimate - auto const estimate = - std::chrono::duration_cast( - clock_type::now() - lastPingTime_); + auto const rtt = std::chrono::round( + clock_type::now() - lastPingTime_); + + std::lock_guard sl(recentLock_); - // Calculate the cumulative moving average of the latency: if (latency_) - latency_ = (*latency_ * 7 + estimate) / 8; + latency_ = (*latency_ * 7 + rtt) / 8; else - latency_ = estimate; + latency_ = rtt; } return; @@ -1377,9 +1325,9 @@ PeerImp::onMessage(std::shared_ptr const& m) void PeerImp::onMessage(std::shared_ptr const& m) { - // Don't allow endpoints from peers that are not known sane or are + // Don't allow endpoints from peers that are not known tracking or are // not using a version of the message that we support: - if (sanity_.load() != Sanity::sane || m->version() != 2) + if (tracking_.load() != Tracking::converged || m->version() != 2) return; std::vector endpoints; @@ -1414,7 +1362,7 @@ PeerImp::onMessage(std::shared_ptr const& m) void PeerImp::onMessage(std::shared_ptr const& m) { - if (sanity_.load() == Sanity::insane) + if (tracking_.load() == Tracking::diverged) return; if (app_.getOPs().isNeedNetworkLedger()) @@ -1629,21 +1577,22 @@ PeerImp::onMessage(std::shared_ptr const& m) if (!isTrusted) { - if (sanity_.load() == Sanity::insane) + if (tracking_.load() == Tracking::diverged) { - JLOG(p_journal_.debug()) << "Proposal: Dropping UNTRUSTED (insane)"; + JLOG(p_journal_.debug()) + << "Proposal: Dropping untrusted (peer divergence)"; return; } if (!cluster() && app_.getFeeTrack().isLoadedLocal()) { - JLOG(p_journal_.debug()) << "Proposal: Dropping UNTRUSTED (load)"; + JLOG(p_journal_.debug()) << "Proposal: Dropping untrusted (load)"; return; } } JLOG(p_journal_.trace()) - << "Proposal: " << (isTrusted ? "trusted" : "UNTRUSTED"); + << "Proposal: " << (isTrusted ? "trusted" : "untrusted"); auto proposal = RCLCxPeerPos( publicKey, @@ -1764,7 +1713,7 @@ PeerImp::onMessage(std::shared_ptr const& m) if (m->has_ledgerseq() && app_.getLedgerMaster().getValidatedLedgerAge() < 2min) { - checkSanity( + checkTracking( m->ledgerseq(), app_.getLedgerMaster().getValidLedgerIndex()); } @@ -1843,11 +1792,11 @@ PeerImp::onMessage(std::shared_ptr const& m) } void -PeerImp::checkSanity(std::uint32_t validationSeq) +PeerImp::checkTracking(std::uint32_t validationSeq) { std::uint32_t serverSeq; { - // Extract the seqeuence number of the highest + // Extract the sequence number of the highest // ledger this peer has std::lock_guard sl(recentLock_); @@ -1857,66 +1806,29 @@ PeerImp::checkSanity(std::uint32_t validationSeq) { // Compare the peer's ledger sequence to the // sequence of a recently-validated ledger - checkSanity(serverSeq, validationSeq); + checkTracking(serverSeq, validationSeq); } } void -PeerImp::checkSanity(std::uint32_t seq1, std::uint32_t seq2) +PeerImp::checkTracking(std::uint32_t seq1, std::uint32_t seq2) { int diff = std::max(seq1, seq2) - std::min(seq1, seq2); - if (diff < Tuning::saneLedgerLimit) + if (diff < Tuning::convergedLedgerLimit) { // The peer's ledger sequence is close to the validation's - sanity_ = Sanity::sane; + tracking_ = Tracking::converged; } - if ((diff > Tuning::insaneLedgerLimit) && - (sanity_.load() != Sanity::insane)) + if ((diff > Tuning::divergedLedgerLimit) && + (tracking_.load() != Tracking::diverged)) { // The peer's ledger sequence is way off the validation's std::lock_guard sl(recentLock_); - sanity_ = Sanity::insane; - insaneTime_ = clock_type::now(); - } -} - -// Should this connection be rejected -// and considered a failure -void -PeerImp::check() -{ - if (m_inbound || (sanity_.load() == Sanity::sane)) - return; - - clock_type::time_point insaneTime; - { - std::lock_guard sl(recentLock_); - - insaneTime = insaneTime_; - } - - bool reject = false; - - if (sanity_.load() == Sanity::insane) - reject = (insaneTime - clock_type::now()) > - std::chrono::seconds(Tuning::maxInsaneTime); - - if (sanity_.load() == Sanity::unknown) - reject = (insaneTime - clock_type::now()) > - std::chrono::seconds(Tuning::maxUnknownTime); - - if (reject) - { - overlay_.peerFinder().on_failure(slot_); - post( - strand_, - std::bind( - (void (PeerImp::*)(std::string const&)) & PeerImp::fail, - shared_from_this(), - "Not useful")); + tracking_ = Tracking::diverged; + trackingTime_ = clock_type::now(); } } @@ -2140,10 +2052,10 @@ PeerImp::onMessage(std::shared_ptr const& m) auto const isTrusted = app_.validators().trusted(val->getSignerPublic()); - if (!isTrusted && (sanity_.load() == Sanity::insane)) + if (!isTrusted && (tracking_.load() == Tracking::diverged)) { JLOG(p_journal_.debug()) - << "Validation: dropping untrusted from insane peer"; + << "Validation: dropping untrusted from diverged peer"; } if (isTrusted || cluster() || !app_.getFeeTrack().isLoadedLocal()) { @@ -2822,24 +2734,21 @@ PeerImp::getLedger(std::shared_ptr const& m) { // return account state root node if possible Serializer rootNode(768); - if (stateMap.getRootNode(rootNode, snfWIRE)) - { - reply.add_nodes()->set_nodedata( - rootNode.getDataPtr(), rootNode.getLength()); - if (ledger->info().txHash != beast::zero) - { - auto const& txMap = ledger->txMap(); + stateMap.serializeRoot(rootNode); + reply.add_nodes()->set_nodedata( + rootNode.getDataPtr(), rootNode.getLength()); - if (txMap.getHash() != beast::zero) - { - rootNode.erase(); + if (ledger->info().txHash != beast::zero) + { + auto const& txMap = ledger->txMap(); + if (txMap.getHash() != beast::zero) + { + rootNode.erase(); - if (txMap.getRootNode(rootNode, snfWIRE)) - reply.add_nodes()->set_nodedata( - rootNode.getDataPtr(), - rootNode.getLength()); - } + txMap.serializeRoot(rootNode); + reply.add_nodes()->set_nodedata( + rootNode.getDataPtr(), rootNode.getLength()); } } } @@ -2882,12 +2791,12 @@ PeerImp::getLedger(std::shared_ptr const& m) (reply.nodes().size() < Tuning::maxReplyNodes)); ++i) { - SHAMapNodeID mn(packet.nodeids(i).data(), packet.nodeids(i).size()); + auto const mn = deserializeSHAMapNodeID(packet.nodeids(i)); - if (!mn.isValid()) + if (!mn) { JLOG(p_journal_.warn()) << "GetLedger: Invalid node " << logMe; - charge(Resource::feeInvalidRequest); + charge(Resource::feeBadData); return; } @@ -2896,7 +2805,7 @@ PeerImp::getLedger(std::shared_ptr const& m) try { - if (map->getNodeFat(mn, nodeIDs, rawNodes, fatLeaves, depth)) + if (map->getNodeFat(*mn, nodeIDs, rawNodes, fatLeaves, depth)) { assert(nodeIDs.size() == rawNodes.size()); JLOG(p_journal_.trace()) << "GetLedger: getNodeFat got " @@ -2909,10 +2818,8 @@ PeerImp::getLedger(std::shared_ptr const& m) nodeIDIterator != nodeIDs.end(); ++nodeIDIterator, ++rawNodeIterator) { - Serializer nID(33); - nodeIDIterator->addIDRaw(nID); protocol::TMLedgerNode* node = reply.add_nodes(); - node->set_nodeid(nID.getDataPtr(), nID.getLength()); + node->set_nodeid(nodeIDIterator->getRawString()); node->set_nodedata( &rawNodeIterator->front(), rawNodeIterator->size()); } @@ -2940,7 +2847,7 @@ PeerImp::getLedger(std::shared_ptr const& m) info += ", no hash specified"; JLOG(p_journal_.warn()) - << "getNodeFat( " << mn << ") throws exception: " << info; + << "getNodeFat( " << *mn << ") throws exception: " << info; } } @@ -2994,7 +2901,7 @@ bool PeerImp::isHighLatency() const { std::lock_guard sl(recentLock_); - return latency_ >= Tuning::peerHighLatency; + return latency_ >= peerHighLatency; } void diff --git a/src/ripple/overlay/impl/PeerImp.h b/src/ripple/overlay/impl/PeerImp.h index dad36584204..77baca9e2bf 100644 --- a/src/ripple/overlay/impl/PeerImp.h +++ b/src/ripple/overlay/impl/PeerImp.h @@ -37,9 +37,9 @@ #include #include #include +#include #include #include -#include namespace ripple { @@ -48,30 +48,8 @@ class PeerImp : public Peer, public OverlayImpl::Child { public: - /** Type of connection. - This affects how messages are routed. - */ - enum class Type { legacy, leaf, peer }; - - /** Current state */ - enum class State { - /** A connection is being established (outbound) */ - connecting - - /** Connection has been successfully established */ - , - connected - - /** Handshake has been received from this peer */ - , - handshaked - - /** Running the Ripple protocol actively */ - , - active - }; - - enum class Sanity { insane, unknown, sane }; + /** Whether the peer's view of the ledger converges or diverges from ours */ + enum class Tracking { diverged, unknown, converged }; struct ShardInfo { @@ -79,8 +57,6 @@ class PeerImp : public Peer, RangeSet shardIndexes; }; - using ptr = std::shared_ptr; - private: using clock_type = std::chrono::steady_clock; using error_code = boost::system::error_code; @@ -105,8 +81,6 @@ class PeerImp : public Peer, boost::asio::strand strand_; waitable_timer timer_; - // Type type_ = Type::legacy; - // Updated at each stage of the connection process to reflect // the current conditions as closely as possible. beast::IP::Endpoint const remote_address_; @@ -114,19 +88,18 @@ class PeerImp : public Peer, // These are up here to prevent warnings about order of initializations // OverlayImpl& overlay_; - bool const m_inbound; + bool const inbound_; // Protocol version to use for this link ProtocolVersion protocol_; - State state_; // Current state - std::atomic sanity_; - clock_type::time_point insaneTime_; + std::atomic tracking_; + clock_type::time_point trackingTime_; bool detaching_ = false; // Node public key of peer. PublicKey const publicKey_; std::string name_; - std::shared_timed_mutex mutable nameMutex_; + boost::shared_mutex mutable nameMutex_; // The indices of the smallest and largest ledgers this peer has available // @@ -162,16 +135,13 @@ class PeerImp : public Peer, // o maxLedger_ // o recentLedgers_ // o recentTxSets_ - // o insaneTime_ + // o trackingTime_ // o latency_ // // The following variables are being protected preemptively: // // o name_ // o last_status_ - // o lastPingSeq_ - // o lastPingTime_ - // o no_ping_ // // June 2019 @@ -184,11 +154,9 @@ class PeerImp : public Peer, http_request_type request_; http_response_type response_; boost::beast::http::fields const& headers_; - boost::beast::multi_buffer write_buffer_; std::queue> send_queue_; bool gracefulClose_ = false; int large_sendq_ = 0; - int no_ping_ = 0; std::unique_ptr load_event_; // The highest sequence of each PublisherList that has // been sent to or received from this peer. @@ -220,7 +188,7 @@ class PeerImp : public Peer, total_bytes() const; private: - std::shared_mutex mutable mutex_; + boost::shared_mutex mutable mutex_; boost::circular_buffer rollingAvg_{30, 0ull}; clock_type::time_point intervalStart_{clock_type::now()}; std::uint64_t totalBytes_{0}; @@ -330,17 +298,14 @@ class PeerImp : public Peer, bool cluster() const override; - void - check(); - - /** Check if the peer is sane + /** Check if the peer is tracking @param validationSeq The ledger sequence of a recently-validated ledger */ void - checkSanity(std::uint32_t validationSeq); + checkTracking(std::uint32_t validationSeq); void - checkSanity(std::uint32_t seq1, std::uint32_t seq2); + checkTracking(std::uint32_t seq1, std::uint32_t seq2); PublicKey const& getNodePublic() const override @@ -467,16 +432,6 @@ class PeerImp : public Peer, void doAccept(); - http_response_type - makeResponse( - bool crawl, - http_request_type const& req, - beast::IP::Address remote_ip, - uint256 const& sharedValue); - - void - onWriteResponse(error_code ec, std::size_t bytes_transferred); - std::string name() const; @@ -561,18 +516,6 @@ class PeerImp : public Peer, onMessage(std::shared_ptr const& m); private: - State - state() const - { - return state_; - } - - void - state(State new_state) - { - state_ = new_state; - } - //-------------------------------------------------------------------------- // lockedRecentLock is passed as a reminder to callers that recentLock_ // must be locked. @@ -633,12 +576,12 @@ PeerImp::PeerImp( , timer_(waitable_timer{socket_.get_executor()}) , remote_address_(slot->remote_endpoint()) , overlay_(overlay) - , m_inbound(false) + , inbound_(false) , protocol_(protocol) - , state_(State::active) - , sanity_(Sanity::unknown) - , insaneTime_(clock_type::now()) + , tracking_(Tracking::unknown) + , trackingTime_(clock_type::now()) , publicKey_(publicKey) + , lastPingTime_(clock_type::now()) , creationTime_(clock_type::now()) , usage_(usage) , fee_(Resource::feeLightPeer) diff --git a/src/ripple/overlay/impl/ProtocolMessage.h b/src/ripple/overlay/impl/ProtocolMessage.h index eb7dd18f092..f5114a2452c 100644 --- a/src/ripple/overlay/impl/ProtocolMessage.h +++ b/src/ripple/overlay/impl/ProtocolMessage.h @@ -258,11 +258,19 @@ invoke(MessageHeader const& header, Buffers const& buffers, Handler& handler) If there is insufficient data to produce a complete protocol message, zero is returned for the number of bytes consumed. + @param buffers The buffer that contains the data we've received + @param handler The handler that will be used to process the message + @param hint If possible, a hint as to the amount of data to read next. The + returned value MAY be zero, which means "no hint" + @return The number of bytes consumed, or the error code if any. */ template std::pair -invokeProtocolMessage(Buffers const& buffers, Handler& handler) +invokeProtocolMessage( + Buffers const& buffers, + Handler& handler, + std::size_t& hint) { std::pair result = {0, {}}; @@ -303,7 +311,10 @@ invokeProtocolMessage(Buffers const& buffers, Handler& handler) // We don't have the whole message yet. This isn't an error but we have // nothing to do. if (header->total_wire_size > size) + { + hint = header->total_wire_size - size; return result; + } bool success; diff --git a/src/ripple/overlay/impl/Tuning.h b/src/ripple/overlay/impl/Tuning.h index bd199fc0a73..4d3467ae49c 100644 --- a/src/ripple/overlay/impl/Tuning.h +++ b/src/ripple/overlay/impl/Tuning.h @@ -27,42 +27,22 @@ namespace ripple { namespace Tuning { enum { - /** Size of buffer used to read from the socket. */ - readBufferBytes = 4096, - - /** How long a server can remain insane before we - disconnected it (if outbound) */ - maxInsaneTime = 60, - - /** How long a server can remain unknown before we - disconnect it (if outbound) */ - maxUnknownTime = 300, - /** How many ledgers off a server can be and we will - still consider it sane */ - saneLedgerLimit = 24, + still consider it converged */ + convergedLedgerLimit = 24, /** How many ledgers off a server has to be before we - consider it insane */ - insaneLedgerLimit = 128, + consider it diverged */ + divergedLedgerLimit = 128, /** The maximum number of ledger entries in a single reply */ maxReplyNodes = 8192, - /** How often we check connections (seconds) */ - checkSeconds = 32, - - /** How often we latency/sendq probe connections */ - timerSeconds = 8, - /** How many timer intervals a sendq has to stay large before we disconnect */ sendqIntervals = 4, - /** How many timer intervals we can go without a ping reply */ - noPing = 10, - /** How many messages on a send queue before we refuse queries */ dropSendQueue = 192, @@ -76,8 +56,8 @@ enum { checkIdlePeers = 4, }; -/** The threshold above which we treat a peer connection as high latency */ -std::chrono::milliseconds constexpr peerHighLatency{300}; +/** Size of buffer used to read from the socket. */ +std::size_t constexpr readBufferBytes = 16384; } // namespace Tuning diff --git a/src/ripple/peerfinder/PeerfinderManager.h b/src/ripple/peerfinder/PeerfinderManager.h index 9c175cc9765..bcca3c919c9 100644 --- a/src/ripple/peerfinder/PeerfinderManager.h +++ b/src/ripple/peerfinder/PeerfinderManager.h @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -43,19 +44,19 @@ struct Config This includes both inbound and outbound, but does not include fixed peers. */ - int maxPeers; + std::size_t maxPeers; /** The number of automatic outbound connections to maintain. Outbound connections are only maintained if autoConnect - is `true`. The value can be fractional; The decision to round up - or down will be made using a per-process pseudorandom number and - a probability proportional to the fractional part. - Example: - If outPeers is 9.3, then 30% of nodes will maintain 9 outbound - connections, while 70% of nodes will maintain 10 outbound - connections. + is `true`. */ - double outPeers; + std::size_t outPeers; + + /** The number of automatic inbound connections to maintain. + Inbound connections are only maintained if wantIncoming + is `true`. + */ + std::size_t inPeers; /** `true` if we want our IP address kept private. */ bool peerPrivate = true; @@ -81,7 +82,7 @@ struct Config Config(); /** Returns a suitable value for outPeers according to the rules. */ - double + std::size_t calcOutPeers() const; /** Adjusts the values so they follow the business rules. */ @@ -91,6 +92,20 @@ struct Config /** Write the configuration into a property stream */ void onWrite(beast::PropertyStream::Map& map); + + /** Make PeerFinder::Config from configuration parameters + * @param config server's configuration + * @param port server's listening port + * @param validationPublicKey true if validation public key is not empty + * @param ipLimit limit of incoming connections per IP + * @return PeerFinder::Config + */ + static Config + makeConfig( + ripple::Config const& config, + std::uint16_t port, + bool validationPublicKey, + int ipLimit); }; //------------------------------------------------------------------------------ diff --git a/src/ripple/peerfinder/impl/Counts.h b/src/ripple/peerfinder/impl/Counts.h index 7053c5789ca..5d9e318594f 100644 --- a/src/ripple/peerfinder/impl/Counts.h +++ b/src/ripple/peerfinder/impl/Counts.h @@ -48,8 +48,6 @@ class Counts , m_acceptCount(0) , m_closingCount(0) { - m_roundingThreshold = - std::generate_canonical(default_prng()); } //-------------------------------------------------------------------------- @@ -136,28 +134,9 @@ class Counts void onConfig(Config const& config) { - // Calculate the number of outbound peers we want. If we dont want or - // can't accept incoming, this will simply be equal to maxPeers. - // Otherwise we calculate a fractional amount based on percentages and - // pseudo-randomly round up or down. - // + m_out_max = config.outPeers; if (config.wantIncoming) - { - // Round outPeers upwards using a Bernoulli distribution - m_out_max = std::floor(config.outPeers); - if (m_roundingThreshold < (config.outPeers - m_out_max)) - ++m_out_max; - } - else - { - m_out_max = config.maxPeers; - } - - // Calculate the largest number of inbound connections we could take. - if (config.maxPeers >= m_out_max) - m_in_max = config.maxPeers - m_out_max; - else - m_in_max = 0; + m_in_max = config.inPeers; } /** Returns the number of accepted connections that haven't handshaked. */ @@ -350,13 +329,6 @@ class Counts // Number of connections that are gracefully closing. int m_closingCount; - - /** Fractional threshold below which we round down. - This is used to round the value of Config::outPeers up or down in - such a way that the network-wide average number of outgoing - connections approximates the recommended, fractional value. - */ - double m_roundingThreshold; }; } // namespace PeerFinder diff --git a/src/ripple/peerfinder/impl/PeerfinderConfig.cpp b/src/ripple/peerfinder/impl/PeerfinderConfig.cpp index 3cd41e24f2c..8b90bc97184 100644 --- a/src/ripple/peerfinder/impl/PeerfinderConfig.cpp +++ b/src/ripple/peerfinder/impl/PeerfinderConfig.cpp @@ -26,6 +26,7 @@ namespace PeerFinder { Config::Config() : maxPeers(Tuning::defaultMaxPeers) , outPeers(calcOutPeers()) + , inPeers(0) , wantIncoming(true) , autoConnect(true) , listeningPort(0) @@ -33,22 +34,17 @@ Config::Config() { } -double +std::size_t Config::calcOutPeers() const { return std::max( - maxPeers * Tuning::outPercent * 0.01, double(Tuning::minOutCount)); + (maxPeers * Tuning::outPercent + 50) / 100, + std::size_t(Tuning::minOutCount)); } void Config::applyTuning() { - if (maxPeers < Tuning::minOutCount) - maxPeers = Tuning::minOutCount; - outPeers = calcOutPeers(); - - auto const inPeers = maxPeers - outPeers; - if (ipLimit == 0) { // Unless a limit is explicitly set, we allow between @@ -78,5 +74,67 @@ Config::onWrite(beast::PropertyStream::Map& map) map["ip_limit"] = ipLimit; } +Config +Config::makeConfig( + ripple::Config const& cfg, + std::uint16_t port, + bool validationPublicKey, + int ipLimit) +{ + PeerFinder::Config config; + + config.peerPrivate = cfg.PEER_PRIVATE; + + // Servers with peer privacy don't want to allow incoming connections + config.wantIncoming = (!config.peerPrivate) && (port != 0); + + if (!cfg.PEERS_OUT_MAX && !cfg.PEERS_IN_MAX) + { + if (cfg.PEERS_MAX != 0) + config.maxPeers = cfg.PEERS_MAX; + + if (config.maxPeers < Tuning::minOutCount) + config.maxPeers = Tuning::minOutCount; + config.outPeers = config.calcOutPeers(); + + // Calculate the number of outbound peers we want. If we dont want + // or can't accept incoming, this will simply be equal to maxPeers. + if (!config.wantIncoming) + config.outPeers = config.maxPeers; + + // Calculate the largest number of inbound connections we could + // take. + if (config.maxPeers >= config.outPeers) + config.inPeers = config.maxPeers - config.outPeers; + else + config.inPeers = 0; + } + else + { + config.outPeers = cfg.PEERS_OUT_MAX; + config.inPeers = cfg.PEERS_IN_MAX; + config.maxPeers = 0; + } + + // This will cause servers configured as validators to request that + // peers they connect to never report their IP address. We set this + // after we set the 'wantIncoming' because we want a "soft" version + // of peer privacy unless the operator explicitly asks for it. + if (validationPublicKey) + config.peerPrivate = true; + + // if it's a private peer or we are running as standalone + // automatic connections would defeat the purpose. + config.autoConnect = !cfg.standalone() && !cfg.PEER_PRIVATE; + config.listeningPort = port; + config.features = ""; + config.ipLimit = ipLimit; + + // Enforce business rules + config.applyTuning(); + + return config; +} + } // namespace PeerFinder } // namespace ripple diff --git a/src/ripple/protocol/AccountID.h b/src/ripple/protocol/AccountID.h index f8393bc2637..e105084c8c9 100644 --- a/src/ripple/protocol/AccountID.h +++ b/src/ripple/protocol/AccountID.h @@ -57,29 +57,6 @@ template <> boost::optional parseBase58(std::string const& s); -// Compatibility with legacy code -bool -deprecatedParseBase58(AccountID& account, Json::Value const& jv); - -/** Parse AccountID from hexadecimal string - - If the string is not exactly 40 - hex digits, boost::none is returned. - - @return boost::none if a parse error occurs -*/ -template <> -boost::optional -parseHex(std::string const& s); - -/** Parse AccountID from hex or checked base58 string. - - @return boost::none if a parse error occurs -*/ -template <> -boost::optional -parseHexOrBase58(std::string const& s); - /** Compute AccountID from public key. The account ID is computed as the 160-bit hash of the diff --git a/src/ripple/protocol/SField.h b/src/ripple/protocol/SField.h index 97d7499145d..d6c51014654 100644 --- a/src/ripple/protocol/SField.h +++ b/src/ripple/protocol/SField.h @@ -314,17 +314,17 @@ operator~(TypedField const& f) //------------------------------------------------------------------------------ -using SF_U8 = TypedField>; -using SF_U16 = TypedField>; -using SF_U32 = TypedField>; -using SF_U64 = TypedField>; -using SF_U128 = TypedField>; -using SF_U160 = TypedField>; -using SF_U256 = TypedField>; -using SF_Account = TypedField; -using SF_Amount = TypedField; -using SF_Blob = TypedField; -using SF_Vec256 = TypedField; +using SF_UINT8 = TypedField>; +using SF_UINT16 = TypedField>; +using SF_UINT32 = TypedField>; +using SF_UINT64 = TypedField>; +using SF_HASH128 = TypedField>; +using SF_HASH160 = TypedField>; +using SF_HASH256 = TypedField>; +using SF_ACCOUNT = TypedField; +using SF_AMOUNT = TypedField; +using SF_VL = TypedField; +using SF_VECTOR256 = TypedField; //------------------------------------------------------------------------------ @@ -336,164 +336,164 @@ extern SField const sfValidation; extern SField const sfMetadata; // 8-bit integers -extern SF_U8 const sfCloseResolution; -extern SF_U8 const sfMethod; -extern SF_U8 const sfTransactionResult; -extern SF_U8 const sfTickSize; -extern SF_U8 const sfUNLModifyDisabling; +extern SF_UINT8 const sfCloseResolution; +extern SF_UINT8 const sfMethod; +extern SF_UINT8 const sfTransactionResult; +extern SF_UINT8 const sfTickSize; +extern SF_UINT8 const sfUNLModifyDisabling; // 16-bit integers -extern SF_U16 const sfLedgerEntryType; -extern SF_U16 const sfTransactionType; -extern SF_U16 const sfSignerWeight; +extern SF_UINT16 const sfLedgerEntryType; +extern SF_UINT16 const sfTransactionType; +extern SF_UINT16 const sfSignerWeight; // 16-bit integers (uncommon) -extern SF_U16 const sfVersion; +extern SF_UINT16 const sfVersion; // 32-bit integers (common) -extern SF_U32 const sfFlags; -extern SF_U32 const sfSourceTag; -extern SF_U32 const sfSequence; -extern SF_U32 const sfPreviousTxnLgrSeq; -extern SF_U32 const sfLedgerSequence; -extern SF_U32 const sfCloseTime; -extern SF_U32 const sfParentCloseTime; -extern SF_U32 const sfSigningTime; -extern SF_U32 const sfExpiration; -extern SF_U32 const sfTransferRate; -extern SF_U32 const sfWalletSize; -extern SF_U32 const sfOwnerCount; -extern SF_U32 const sfDestinationTag; +extern SF_UINT32 const sfFlags; +extern SF_UINT32 const sfSourceTag; +extern SF_UINT32 const sfSequence; +extern SF_UINT32 const sfPreviousTxnLgrSeq; +extern SF_UINT32 const sfLedgerSequence; +extern SF_UINT32 const sfCloseTime; +extern SF_UINT32 const sfParentCloseTime; +extern SF_UINT32 const sfSigningTime; +extern SF_UINT32 const sfExpiration; +extern SF_UINT32 const sfTransferRate; +extern SF_UINT32 const sfWalletSize; +extern SF_UINT32 const sfOwnerCount; +extern SF_UINT32 const sfDestinationTag; // 32-bit integers (uncommon) -extern SF_U32 const sfHighQualityIn; -extern SF_U32 const sfHighQualityOut; -extern SF_U32 const sfLowQualityIn; -extern SF_U32 const sfLowQualityOut; -extern SF_U32 const sfQualityIn; -extern SF_U32 const sfQualityOut; -extern SF_U32 const sfStampEscrow; -extern SF_U32 const sfBondAmount; -extern SF_U32 const sfLoadFee; -extern SF_U32 const sfOfferSequence; -extern SF_U32 const sfFirstLedgerSequence; -extern SF_U32 const sfLastLedgerSequence; -extern SF_U32 const sfTransactionIndex; -extern SF_U32 const sfOperationLimit; -extern SF_U32 const sfReferenceFeeUnits; -extern SF_U32 const sfReserveBase; -extern SF_U32 const sfReserveIncrement; -extern SF_U32 const sfSetFlag; -extern SF_U32 const sfClearFlag; -extern SF_U32 const sfSignerQuorum; -extern SF_U32 const sfCancelAfter; -extern SF_U32 const sfFinishAfter; -extern SF_U32 const sfSignerListID; -extern SF_U32 const sfSettleDelay; -extern SF_U32 const sfTicketCount; -extern SF_U32 const sfTicketSequence; +extern SF_UINT32 const sfHighQualityIn; +extern SF_UINT32 const sfHighQualityOut; +extern SF_UINT32 const sfLowQualityIn; +extern SF_UINT32 const sfLowQualityOut; +extern SF_UINT32 const sfQualityIn; +extern SF_UINT32 const sfQualityOut; +extern SF_UINT32 const sfStampEscrow; +extern SF_UINT32 const sfBondAmount; +extern SF_UINT32 const sfLoadFee; +extern SF_UINT32 const sfOfferSequence; +extern SF_UINT32 const sfFirstLedgerSequence; +extern SF_UINT32 const sfLastLedgerSequence; +extern SF_UINT32 const sfTransactionIndex; +extern SF_UINT32 const sfOperationLimit; +extern SF_UINT32 const sfReferenceFeeUnits; +extern SF_UINT32 const sfReserveBase; +extern SF_UINT32 const sfReserveIncrement; +extern SF_UINT32 const sfSetFlag; +extern SF_UINT32 const sfClearFlag; +extern SF_UINT32 const sfSignerQuorum; +extern SF_UINT32 const sfCancelAfter; +extern SF_UINT32 const sfFinishAfter; +extern SF_UINT32 const sfSignerListID; +extern SF_UINT32 const sfSettleDelay; +extern SF_UINT32 const sfTicketCount; +extern SF_UINT32 const sfTicketSequence; // 64-bit integers -extern SF_U64 const sfIndexNext; -extern SF_U64 const sfIndexPrevious; -extern SF_U64 const sfBookNode; -extern SF_U64 const sfOwnerNode; -extern SF_U64 const sfBaseFee; -extern SF_U64 const sfExchangeRate; -extern SF_U64 const sfLowNode; -extern SF_U64 const sfHighNode; -extern SF_U64 const sfDestinationNode; -extern SF_U64 const sfCookie; -extern SF_U64 const sfServerVersion; +extern SF_UINT64 const sfIndexNext; +extern SF_UINT64 const sfIndexPrevious; +extern SF_UINT64 const sfBookNode; +extern SF_UINT64 const sfOwnerNode; +extern SF_UINT64 const sfBaseFee; +extern SF_UINT64 const sfExchangeRate; +extern SF_UINT64 const sfLowNode; +extern SF_UINT64 const sfHighNode; +extern SF_UINT64 const sfDestinationNode; +extern SF_UINT64 const sfCookie; +extern SF_UINT64 const sfServerVersion; // 128-bit -extern SF_U128 const sfEmailHash; +extern SF_HASH128 const sfEmailHash; // 160-bit (common) -extern SF_U160 const sfTakerPaysCurrency; -extern SF_U160 const sfTakerPaysIssuer; -extern SF_U160 const sfTakerGetsCurrency; -extern SF_U160 const sfTakerGetsIssuer; +extern SF_HASH160 const sfTakerPaysCurrency; +extern SF_HASH160 const sfTakerPaysIssuer; +extern SF_HASH160 const sfTakerGetsCurrency; +extern SF_HASH160 const sfTakerGetsIssuer; // 256-bit (common) -extern SF_U256 const sfLedgerHash; -extern SF_U256 const sfParentHash; -extern SF_U256 const sfTransactionHash; -extern SF_U256 const sfAccountHash; -extern SF_U256 const sfPreviousTxnID; -extern SF_U256 const sfLedgerIndex; -extern SF_U256 const sfWalletLocator; -extern SF_U256 const sfRootIndex; -extern SF_U256 const sfAccountTxnID; +extern SF_HASH256 const sfLedgerHash; +extern SF_HASH256 const sfParentHash; +extern SF_HASH256 const sfTransactionHash; +extern SF_HASH256 const sfAccountHash; +extern SF_HASH256 const sfPreviousTxnID; +extern SF_HASH256 const sfLedgerIndex; +extern SF_HASH256 const sfWalletLocator; +extern SF_HASH256 const sfRootIndex; +extern SF_HASH256 const sfAccountTxnID; // 256-bit (uncommon) -extern SF_U256 const sfBookDirectory; -extern SF_U256 const sfInvoiceID; -extern SF_U256 const sfNickname; -extern SF_U256 const sfAmendment; -extern SF_U256 const sfDigest; -extern SF_U256 const sfPayChannel; -extern SF_U256 const sfConsensusHash; -extern SF_U256 const sfCheckID; -extern SF_U256 const sfValidatedHash; +extern SF_HASH256 const sfBookDirectory; +extern SF_HASH256 const sfInvoiceID; +extern SF_HASH256 const sfNickname; +extern SF_HASH256 const sfAmendment; +extern SF_HASH256 const sfDigest; +extern SF_HASH256 const sfChannel; +extern SF_HASH256 const sfConsensusHash; +extern SF_HASH256 const sfCheckID; +extern SF_HASH256 const sfValidatedHash; // currency amount (common) -extern SF_Amount const sfAmount; -extern SF_Amount const sfBalance; -extern SF_Amount const sfLimitAmount; -extern SF_Amount const sfTakerPays; -extern SF_Amount const sfTakerGets; -extern SF_Amount const sfLowLimit; -extern SF_Amount const sfHighLimit; -extern SF_Amount const sfFee; -extern SF_Amount const sfSendMax; -extern SF_Amount const sfDeliverMin; +extern SF_AMOUNT const sfAmount; +extern SF_AMOUNT const sfBalance; +extern SF_AMOUNT const sfLimitAmount; +extern SF_AMOUNT const sfTakerPays; +extern SF_AMOUNT const sfTakerGets; +extern SF_AMOUNT const sfLowLimit; +extern SF_AMOUNT const sfHighLimit; +extern SF_AMOUNT const sfFee; +extern SF_AMOUNT const sfSendMax; +extern SF_AMOUNT const sfDeliverMin; // currency amount (uncommon) -extern SF_Amount const sfMinimumOffer; -extern SF_Amount const sfRippleEscrow; -extern SF_Amount const sfDeliveredAmount; +extern SF_AMOUNT const sfMinimumOffer; +extern SF_AMOUNT const sfRippleEscrow; +extern SF_AMOUNT const sfDeliveredAmount; // variable length (common) -extern SF_Blob const sfPublicKey; -extern SF_Blob const sfMessageKey; -extern SF_Blob const sfSigningPubKey; -extern SF_Blob const sfTxnSignature; -extern SF_Blob const sfSignature; -extern SF_Blob const sfDomain; -extern SF_Blob const sfFundCode; -extern SF_Blob const sfRemoveCode; -extern SF_Blob const sfExpireCode; -extern SF_Blob const sfCreateCode; -extern SF_Blob const sfMemoType; -extern SF_Blob const sfMemoData; -extern SF_Blob const sfMemoFormat; +extern SF_VL const sfPublicKey; +extern SF_VL const sfMessageKey; +extern SF_VL const sfSigningPubKey; +extern SF_VL const sfTxnSignature; +extern SF_VL const sfSignature; +extern SF_VL const sfDomain; +extern SF_VL const sfFundCode; +extern SF_VL const sfRemoveCode; +extern SF_VL const sfExpireCode; +extern SF_VL const sfCreateCode; +extern SF_VL const sfMemoType; +extern SF_VL const sfMemoData; +extern SF_VL const sfMemoFormat; // variable length (uncommon) -extern SF_Blob const sfFulfillment; -extern SF_Blob const sfCondition; -extern SF_Blob const sfMasterSignature; -extern SF_Blob const sfUNLModifyValidator; -extern SF_Blob const sfValidatorToDisable; -extern SF_Blob const sfValidatorToReEnable; +extern SF_VL const sfFulfillment; +extern SF_VL const sfCondition; +extern SF_VL const sfMasterSignature; +extern SF_VL const sfUNLModifyValidator; +extern SF_VL const sfValidatorToDisable; +extern SF_VL const sfValidatorToReEnable; // account -extern SF_Account const sfAccount; -extern SF_Account const sfOwner; -extern SF_Account const sfDestination; -extern SF_Account const sfIssuer; -extern SF_Account const sfAuthorize; -extern SF_Account const sfUnauthorize; -extern SF_Account const sfTarget; -extern SF_Account const sfRegularKey; +extern SF_ACCOUNT const sfAccount; +extern SF_ACCOUNT const sfOwner; +extern SF_ACCOUNT const sfDestination; +extern SF_ACCOUNT const sfIssuer; +extern SF_ACCOUNT const sfAuthorize; +extern SF_ACCOUNT const sfUnauthorize; +extern SF_ACCOUNT const sfTarget; +extern SF_ACCOUNT const sfRegularKey; // path set extern SField const sfPaths; // vector of 256-bit -extern SF_Vec256 const sfIndexes; -extern SF_Vec256 const sfHashes; -extern SF_Vec256 const sfAmendments; +extern SF_VECTOR256 const sfIndexes; +extern SF_VECTOR256 const sfHashes; +extern SF_VECTOR256 const sfAmendments; // inner object // OBJECT/1 is reserved for end of object diff --git a/src/ripple/protocol/STArray.h b/src/ripple/protocol/STArray.h index ad62b5af6be..12c69864970 100644 --- a/src/ripple/protocol/STArray.h +++ b/src/ripple/protocol/STArray.h @@ -33,12 +33,6 @@ class STArray final : public STBase, public CountedObject list_type v_; public: - static char const* - getCountedObjectName() - { - return "STArray"; - } - using size_type = list_type::size_type; using iterator = list_type::iterator; using const_iterator = list_type::const_iterator; diff --git a/src/ripple/protocol/STBitString.h b/src/ripple/protocol/STBitString.h index a4a847d0b08..06ace27d98f 100644 --- a/src/ripple/protocol/STBitString.h +++ b/src/ripple/protocol/STBitString.h @@ -45,16 +45,6 @@ class STBitString final : public STBase { } - STBitString(SField const& n, const char* v) : STBase(n) - { - value_.SetHex(v); - } - - STBitString(SField const& n, std::string const& v) : STBase(n) - { - value_.SetHex(v); - } - STBitString(SerialIter& sit, SField const& name) : STBitString(name, sit.getBitString()) { diff --git a/src/ripple/protocol/STLedgerEntry.h b/src/ripple/protocol/STLedgerEntry.h index 1ddb0122015..0a0e3a8054f 100644 --- a/src/ripple/protocol/STLedgerEntry.h +++ b/src/ripple/protocol/STLedgerEntry.h @@ -32,12 +32,6 @@ class STLedgerEntry final : public STObject, public CountedObject friend Invariants_test; // this test wants access to the private type_ public: - static char const* - getCountedObjectName() - { - return "STLedgerEntry"; - } - using pointer = std::shared_ptr; using ref = const std::shared_ptr&; diff --git a/src/ripple/protocol/STObject.h b/src/ripple/protocol/STObject.h index 4ac67ded5c8..ad63dcc396e 100644 --- a/src/ripple/protocol/STObject.h +++ b/src/ripple/protocol/STObject.h @@ -260,12 +260,6 @@ class STObject : public STBase, public CountedObject using std::runtime_error::runtime_error; }; - static char const* - getCountedObjectName() - { - return "STObject"; - } - STObject(STObject&&); STObject(STObject const&) = default; STObject(const SOTemplate& type, SField const& name); diff --git a/src/ripple/protocol/STTx.h b/src/ripple/protocol/STTx.h index cecbd2c0a30..0d90b23245a 100644 --- a/src/ripple/protocol/STTx.h +++ b/src/ripple/protocol/STTx.h @@ -42,12 +42,6 @@ enum TxnSql : char { class STTx final : public STObject, public CountedObject { public: - static char const* - getCountedObjectName() - { - return "STTx"; - } - static std::size_t const minMultiSigners = 1; static std::size_t const maxMultiSigners = 8; diff --git a/src/ripple/protocol/STValidation.h b/src/ripple/protocol/STValidation.h index be1537ad04b..866d1e78505 100644 --- a/src/ripple/protocol/STValidation.h +++ b/src/ripple/protocol/STValidation.h @@ -43,12 +43,6 @@ constexpr std::uint32_t vfFullyCanonicalSig = 0x80000000; class STValidation final : public STObject, public CountedObject { public: - static char const* - getCountedObjectName() - { - return "STValidation"; - } - /** Construct a STValidation from a peer. Construct a STValidation from serialized data previously shared by a diff --git a/src/ripple/protocol/Serializer.h b/src/ripple/protocol/Serializer.h index 812517f2566..e51e1bb14f4 100644 --- a/src/ripple/protocol/Serializer.h +++ b/src/ripple/protocol/Serializer.h @@ -411,15 +411,18 @@ template base_uint SerialIter::getBitString() { - base_uint u; auto const n = Bits / 8; + if (remain_ < n) Throw("invalid SerialIter getBitString"); - std::memcpy(u.begin(), p_, n); + + auto const x = p_; + p_ += n; used_ += n; remain_ -= n; - return u; + + return base_uint::fromVoid(x); } } // namespace ripple diff --git a/src/ripple/protocol/Sign.h b/src/ripple/protocol/Sign.h index 112255af81e..6af29530486 100644 --- a/src/ripple/protocol/Sign.h +++ b/src/ripple/protocol/Sign.h @@ -45,7 +45,7 @@ sign( HashPrefix const& prefix, KeyType type, SecretKey const& sk, - SF_Blob const& sigField = sfSignature); + SF_VL const& sigField = sfSignature); /** Returns `true` if STObject contains valid signature @@ -60,7 +60,7 @@ verify( STObject const& st, HashPrefix const& prefix, PublicKey const& pk, - SF_Blob const& sigField = sfSignature); + SF_VL const& sigField = sfSignature); /** Return a Serializer suitable for computing a multisigning TxnSignature. */ Serializer diff --git a/src/ripple/protocol/digest.h b/src/ripple/protocol/digest.h index bd6190f1852..6507057dcdc 100644 --- a/src/ripple/protocol/digest.h +++ b/src/ripple/protocol/digest.h @@ -21,21 +21,20 @@ #define RIPPLE_PROTOCOL_DIGEST_H_INCLUDED #include -#include -#include -#include +#include +#include #include #include namespace ripple { -/* Message digest functions used in the Ripple Protocol +/** Message digest functions used in the codebase - Modeled to meet the requirements of `Hasher` in the - `hash_append` interface, currently in proposal: + @note These are modeled to meet the requirements of `Hasher` in the + `hash_append` interface, discussed in proposal: - N3980 "Types Don't Know #" - http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2014/n3980.html + N3980 "Types Don't Know #" + http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2014/n3980.html */ //------------------------------------------------------------------------------ @@ -47,7 +46,7 @@ namespace ripple { struct openssl_ripemd160_hasher { public: - static beast::endian const endian = beast::endian::native; + static constexpr auto const endian = boost::endian::order::native; using result_type = std::array; @@ -69,7 +68,7 @@ struct openssl_ripemd160_hasher struct openssl_sha512_hasher { public: - static beast::endian const endian = beast::endian::native; + static constexpr auto const endian = boost::endian::order::native; using result_type = std::array; @@ -91,7 +90,7 @@ struct openssl_sha512_hasher struct openssl_sha256_hasher { public: - static beast::endian const endian = beast::endian::native; + static constexpr auto const endian = boost::endian::order::native; using result_type = std::array; @@ -108,17 +107,9 @@ struct openssl_sha256_hasher //------------------------------------------------------------------------------ -// Aliases to choose the correct digest implementation - -#if USE_BEAST_HASHER -using ripemd160_hasher = beast::ripemd160_hasher; -using sha256_hasher = beast::sha256_hasher; -using sha512_hasher = beast::sha512_hasher; -#else using ripemd160_hasher = openssl_ripemd160_hasher; using sha256_hasher = openssl_sha256_hasher; using sha512_hasher = openssl_sha512_hasher; -#endif //------------------------------------------------------------------------------ @@ -143,7 +134,7 @@ struct ripesha_hasher sha256_hasher h_; public: - static beast::endian const endian = beast::endian::native; + static constexpr auto const endian = boost::endian::order::native; using result_type = std::array; @@ -178,7 +169,7 @@ struct basic_sha512_half_hasher sha512_hasher h_; public: - static beast::endian const endian = beast::endian::big; + static constexpr auto const endian = boost::endian::order::big; using result_type = uint256; @@ -196,9 +187,7 @@ struct basic_sha512_half_hasher explicit operator result_type() noexcept { auto const digest = sha512_hasher::result_type(h_); - result_type result; - std::copy(digest.begin(), digest.begin() + 32, result.begin()); - return result; + return result_type::fromVoid(digest.data()); } private: diff --git a/src/ripple/protocol/impl/AccountID.cpp b/src/ripple/protocol/impl/AccountID.cpp index 47193513558..5f9b7a1b76f 100644 --- a/src/ripple/protocol/impl/AccountID.cpp +++ b/src/ripple/protocol/impl/AccountID.cpp @@ -36,47 +36,9 @@ boost::optional parseBase58(std::string const& s) { auto const result = decodeBase58Token(s, TokenType::AccountID); - if (result.empty()) + if (result.size() != AccountID::bytes) return boost::none; - AccountID id; - if (result.size() != id.size()) - return boost::none; - std::memcpy(id.data(), result.data(), result.size()); - return id; -} - -bool -deprecatedParseBase58(AccountID& account, Json::Value const& jv) -{ - if (!jv.isString()) - return false; - auto const result = parseBase58(jv.asString()); - if (!result) - return false; - account = *result; - return true; -} - -template <> -boost::optional -parseHex(std::string const& s) -{ - if (s.size() != 40) - return boost::none; - AccountID id; - if (!id.SetHex(s, true)) - return boost::none; - return id; -} - -template <> -boost::optional -parseHexOrBase58(std::string const& s) -{ - auto result = parseHex(s); - if (!result) - result = parseBase58(s); - return result; + return AccountID{result}; } //------------------------------------------------------------------------------ @@ -117,13 +79,11 @@ parseHexOrBase58(std::string const& s) AccountID calcAccountID(PublicKey const& pk) { + static_assert(AccountID::bytes == sizeof(ripesha_hasher::result_type)); + ripesha_hasher rsh; rsh(pk.data(), pk.size()); - auto const d = static_cast(rsh); - AccountID id; - static_assert(sizeof(d) == id.size(), ""); - std::memcpy(id.data(), d.data(), d.size()); - return id; + return AccountID{static_cast(rsh)}; } AccountID const& @@ -143,11 +103,8 @@ noAccount() bool to_issuer(AccountID& issuer, std::string const& s) { - if (s.size() == (160 / 4)) - { - issuer.SetHex(s); + if (issuer.parseHex(s)) return true; - } auto const account = parseBase58(s); if (!account) return false; diff --git a/src/ripple/protocol/impl/BuildInfo.cpp b/src/ripple/protocol/impl/BuildInfo.cpp index 633c7b6858d..a6479654a43 100644 --- a/src/ripple/protocol/impl/BuildInfo.cpp +++ b/src/ripple/protocol/impl/BuildInfo.cpp @@ -33,7 +33,7 @@ namespace BuildInfo { // and follow the format described at http://semver.org/ //------------------------------------------------------------------------------ // clang-format off -char const* const versionString = "1.7.0-b4" +char const* const versionString = "1.7.0-b5" // clang-format on #if defined(DEBUG) || defined(SANITIZER) diff --git a/src/ripple/protocol/impl/Indexes.cpp b/src/ripple/protocol/impl/Indexes.cpp index 20174d97881..02148a5f158 100644 --- a/src/ripple/protocol/impl/Indexes.cpp +++ b/src/ripple/protocol/impl/Indexes.cpp @@ -96,8 +96,13 @@ getBookBase(Book const& book) uint256 getQualityNext(uint256 const& uBase) { - static uint256 const uNext(from_hex_text("10000000000000000")); - return uBase + uNext; + static uint256 const nextq = []() { + uint256 x; + (void)x.parseHex( + "0000000000000000000000000000000000000000000000010000000000000000"); + return x; + }(); + return uBase + nextq; } std::uint64_t diff --git a/src/ripple/protocol/impl/PublicKey.cpp b/src/ripple/protocol/impl/PublicKey.cpp index 4dfdc312b6f..0469c6ed646 100644 --- a/src/ripple/protocol/impl/PublicKey.cpp +++ b/src/ripple/protocol/impl/PublicKey.cpp @@ -298,13 +298,11 @@ verify( NodeID calcNodeID(PublicKey const& pk) { + static_assert(NodeID::bytes == sizeof(ripesha_hasher::result_type)); + ripesha_hasher h; h(pk.data(), pk.size()); - auto const digest = static_cast(h); - static_assert(NodeID::bytes == sizeof(ripesha_hasher::result_type), ""); - NodeID result; - std::memcpy(result.data(), digest.data(), digest.size()); - return result; + return NodeID{static_cast(h)}; } } // namespace ripple diff --git a/src/ripple/protocol/impl/SField.cpp b/src/ripple/protocol/impl/SField.cpp index 3f0454b31eb..590ffeb65a4 100644 --- a/src/ripple/protocol/impl/SField.cpp +++ b/src/ripple/protocol/impl/SField.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include namespace ripple { @@ -40,255 +41,250 @@ static SField::private_access_tag_t access; // Construct all compile-time SFields, and register them in the knownCodeToField // database: +// Use macros for most SField construction to enforce naming conventions. +#pragma push_macro("CONSTRUCT_UNTYPED_SFIELD") +#undef CONSTRUCT_UNTYPED_SFIELD + +// It would be possible to design the macros so that sfName and txtName would +// be constructed from a single macro parameter. We chose not to take that +// path because then you cannot grep for the exact SField name and find +// where it is constructed. These macros allow that grep to succeed. +#define CONSTRUCT_UNTYPED_SFIELD(sfName, txtName, stiSuffix, fieldValue, ...) \ + SField const sfName( \ + access, STI_##stiSuffix, fieldValue, txtName, ##__VA_ARGS__); \ + static_assert( \ + std::string_view(#sfName) == "sf" txtName, \ + "Declaration of SField does not match its text name") + +#pragma push_macro("CONSTRUCT_TYPED_SFIELD") +#undef CONSTRUCT_TYPED_SFIELD + +#define CONSTRUCT_TYPED_SFIELD(sfName, txtName, stiSuffix, fieldValue, ...) \ + SF_##stiSuffix const sfName( \ + access, STI_##stiSuffix, fieldValue, txtName, ##__VA_ARGS__); \ + static_assert( \ + std::string_view(#sfName) == "sf" txtName, \ + "Declaration of SField does not match its text name") + +// clang-format off + +// SFields which, for historical reasons, do not follow naming conventions. SField const sfInvalid(access, -1); SField const sfGeneric(access, 0); -SField const sfLedgerEntry(access, STI_LEDGERENTRY, 257, "LedgerEntry"); -SField const sfTransaction(access, STI_TRANSACTION, 257, "Transaction"); -SField const sfValidation(access, STI_VALIDATION, 257, "Validation"); -SField const sfMetadata(access, STI_METADATA, 257, "Metadata"); SField const sfHash(access, STI_HASH256, 257, "hash"); SField const sfIndex(access, STI_HASH256, 258, "index"); +// Untyped SFields +CONSTRUCT_UNTYPED_SFIELD(sfLedgerEntry, "LedgerEntry", LEDGERENTRY, 257); +CONSTRUCT_UNTYPED_SFIELD(sfTransaction, "Transaction", TRANSACTION, 257); +CONSTRUCT_UNTYPED_SFIELD(sfValidation, "Validation", VALIDATION, 257); +CONSTRUCT_UNTYPED_SFIELD(sfMetadata, "Metadata", METADATA, 257); + // 8-bit integers -SF_U8 const sfCloseResolution(access, STI_UINT8, 1, "CloseResolution"); -SF_U8 const sfMethod(access, STI_UINT8, 2, "Method"); -SF_U8 const sfTransactionResult(access, STI_UINT8, 3, "TransactionResult"); +CONSTRUCT_TYPED_SFIELD(sfCloseResolution, "CloseResolution", UINT8, 1); +CONSTRUCT_TYPED_SFIELD(sfMethod, "Method", UINT8, 2); +CONSTRUCT_TYPED_SFIELD(sfTransactionResult, "TransactionResult", UINT8, 3); // 8-bit integers (uncommon) -SF_U8 const sfTickSize(access, STI_UINT8, 16, "TickSize"); -SF_U8 const sfUNLModifyDisabling(access, STI_UINT8, 17, "UNLModifyDisabling"); +CONSTRUCT_TYPED_SFIELD(sfTickSize, "TickSize", UINT8, 16); +CONSTRUCT_TYPED_SFIELD(sfUNLModifyDisabling, "UNLModifyDisabling", UINT8, 17); // 16-bit integers -SF_U16 const sfLedgerEntryType( - access, - STI_UINT16, - 1, - "LedgerEntryType", - SField::sMD_Never); -SF_U16 const sfTransactionType(access, STI_UINT16, 2, "TransactionType"); -SF_U16 const sfSignerWeight(access, STI_UINT16, 3, "SignerWeight"); +CONSTRUCT_TYPED_SFIELD(sfLedgerEntryType, "LedgerEntryType", UINT16, 1, SField::sMD_Never); +CONSTRUCT_TYPED_SFIELD(sfTransactionType, "TransactionType", UINT16, 2); +CONSTRUCT_TYPED_SFIELD(sfSignerWeight, "SignerWeight", UINT16, 3); // 16-bit integers (uncommon) -SF_U16 const sfVersion(access, STI_UINT16, 16, "Version"); +CONSTRUCT_TYPED_SFIELD(sfVersion, "Version", UINT16, 16); // 32-bit integers (common) -SF_U32 const sfFlags(access, STI_UINT32, 2, "Flags"); -SF_U32 const sfSourceTag(access, STI_UINT32, 3, "SourceTag"); -SF_U32 const sfSequence(access, STI_UINT32, 4, "Sequence"); -SF_U32 const sfPreviousTxnLgrSeq( - access, - STI_UINT32, - 5, - "PreviousTxnLgrSeq", - SField::sMD_DeleteFinal); -SF_U32 const sfLedgerSequence(access, STI_UINT32, 6, "LedgerSequence"); -SF_U32 const sfCloseTime(access, STI_UINT32, 7, "CloseTime"); -SF_U32 const sfParentCloseTime(access, STI_UINT32, 8, "ParentCloseTime"); -SF_U32 const sfSigningTime(access, STI_UINT32, 9, "SigningTime"); -SF_U32 const sfExpiration(access, STI_UINT32, 10, "Expiration"); -SF_U32 const sfTransferRate(access, STI_UINT32, 11, "TransferRate"); -SF_U32 const sfWalletSize(access, STI_UINT32, 12, "WalletSize"); -SF_U32 const sfOwnerCount(access, STI_UINT32, 13, "OwnerCount"); -SF_U32 const sfDestinationTag(access, STI_UINT32, 14, "DestinationTag"); +CONSTRUCT_TYPED_SFIELD(sfFlags, "Flags", UINT32, 2); +CONSTRUCT_TYPED_SFIELD(sfSourceTag, "SourceTag", UINT32, 3); +CONSTRUCT_TYPED_SFIELD(sfSequence, "Sequence", UINT32, 4); +CONSTRUCT_TYPED_SFIELD(sfPreviousTxnLgrSeq, "PreviousTxnLgrSeq", UINT32, 5, SField::sMD_DeleteFinal); +CONSTRUCT_TYPED_SFIELD(sfLedgerSequence, "LedgerSequence", UINT32, 6); +CONSTRUCT_TYPED_SFIELD(sfCloseTime, "CloseTime", UINT32, 7); +CONSTRUCT_TYPED_SFIELD(sfParentCloseTime, "ParentCloseTime", UINT32, 8); +CONSTRUCT_TYPED_SFIELD(sfSigningTime, "SigningTime", UINT32, 9); +CONSTRUCT_TYPED_SFIELD(sfExpiration, "Expiration", UINT32, 10); +CONSTRUCT_TYPED_SFIELD(sfTransferRate, "TransferRate", UINT32, 11); +CONSTRUCT_TYPED_SFIELD(sfWalletSize, "WalletSize", UINT32, 12); +CONSTRUCT_TYPED_SFIELD(sfOwnerCount, "OwnerCount", UINT32, 13); +CONSTRUCT_TYPED_SFIELD(sfDestinationTag, "DestinationTag", UINT32, 14); // 32-bit integers (uncommon) -SF_U32 const sfHighQualityIn(access, STI_UINT32, 16, "HighQualityIn"); -SF_U32 const sfHighQualityOut(access, STI_UINT32, 17, "HighQualityOut"); -SF_U32 const sfLowQualityIn(access, STI_UINT32, 18, "LowQualityIn"); -SF_U32 const sfLowQualityOut(access, STI_UINT32, 19, "LowQualityOut"); -SF_U32 const sfQualityIn(access, STI_UINT32, 20, "QualityIn"); -SF_U32 const sfQualityOut(access, STI_UINT32, 21, "QualityOut"); -SF_U32 const sfStampEscrow(access, STI_UINT32, 22, "StampEscrow"); -SF_U32 const sfBondAmount(access, STI_UINT32, 23, "BondAmount"); -SF_U32 const sfLoadFee(access, STI_UINT32, 24, "LoadFee"); -SF_U32 const sfOfferSequence(access, STI_UINT32, 25, "OfferSequence"); -SF_U32 const - sfFirstLedgerSequence(access, STI_UINT32, 26, "FirstLedgerSequence"); -SF_U32 const sfLastLedgerSequence(access, STI_UINT32, 27, "LastLedgerSequence"); -SF_U32 const sfTransactionIndex(access, STI_UINT32, 28, "TransactionIndex"); -SF_U32 const sfOperationLimit(access, STI_UINT32, 29, "OperationLimit"); -SF_U32 const sfReferenceFeeUnits(access, STI_UINT32, 30, "ReferenceFeeUnits"); -SF_U32 const sfReserveBase(access, STI_UINT32, 31, "ReserveBase"); -SF_U32 const sfReserveIncrement(access, STI_UINT32, 32, "ReserveIncrement"); -SF_U32 const sfSetFlag(access, STI_UINT32, 33, "SetFlag"); -SF_U32 const sfClearFlag(access, STI_UINT32, 34, "ClearFlag"); -SF_U32 const sfSignerQuorum(access, STI_UINT32, 35, "SignerQuorum"); -SF_U32 const sfCancelAfter(access, STI_UINT32, 36, "CancelAfter"); -SF_U32 const sfFinishAfter(access, STI_UINT32, 37, "FinishAfter"); -SF_U32 const sfSignerListID(access, STI_UINT32, 38, "SignerListID"); -SF_U32 const sfSettleDelay(access, STI_UINT32, 39, "SettleDelay"); -SF_U32 const sfTicketCount(access, STI_UINT32, 40, "TicketCount"); -SF_U32 const sfTicketSequence(access, STI_UINT32, 41, "TicketSequence"); +CONSTRUCT_TYPED_SFIELD(sfHighQualityIn, "HighQualityIn", UINT32, 16); +CONSTRUCT_TYPED_SFIELD(sfHighQualityOut, "HighQualityOut", UINT32, 17); +CONSTRUCT_TYPED_SFIELD(sfLowQualityIn, "LowQualityIn", UINT32, 18); +CONSTRUCT_TYPED_SFIELD(sfLowQualityOut, "LowQualityOut", UINT32, 19); +CONSTRUCT_TYPED_SFIELD(sfQualityIn, "QualityIn", UINT32, 20); +CONSTRUCT_TYPED_SFIELD(sfQualityOut, "QualityOut", UINT32, 21); +CONSTRUCT_TYPED_SFIELD(sfStampEscrow, "StampEscrow", UINT32, 22); +CONSTRUCT_TYPED_SFIELD(sfBondAmount, "BondAmount", UINT32, 23); +CONSTRUCT_TYPED_SFIELD(sfLoadFee, "LoadFee", UINT32, 24); +CONSTRUCT_TYPED_SFIELD(sfOfferSequence, "OfferSequence", UINT32, 25); +CONSTRUCT_TYPED_SFIELD(sfFirstLedgerSequence, "FirstLedgerSequence", UINT32, 26); +CONSTRUCT_TYPED_SFIELD(sfLastLedgerSequence, "LastLedgerSequence", UINT32, 27); +CONSTRUCT_TYPED_SFIELD(sfTransactionIndex, "TransactionIndex", UINT32, 28); +CONSTRUCT_TYPED_SFIELD(sfOperationLimit, "OperationLimit", UINT32, 29); +CONSTRUCT_TYPED_SFIELD(sfReferenceFeeUnits, "ReferenceFeeUnits", UINT32, 30); +CONSTRUCT_TYPED_SFIELD(sfReserveBase, "ReserveBase", UINT32, 31); +CONSTRUCT_TYPED_SFIELD(sfReserveIncrement, "ReserveIncrement", UINT32, 32); +CONSTRUCT_TYPED_SFIELD(sfSetFlag, "SetFlag", UINT32, 33); +CONSTRUCT_TYPED_SFIELD(sfClearFlag, "ClearFlag", UINT32, 34); +CONSTRUCT_TYPED_SFIELD(sfSignerQuorum, "SignerQuorum", UINT32, 35); +CONSTRUCT_TYPED_SFIELD(sfCancelAfter, "CancelAfter", UINT32, 36); +CONSTRUCT_TYPED_SFIELD(sfFinishAfter, "FinishAfter", UINT32, 37); +CONSTRUCT_TYPED_SFIELD(sfSignerListID, "SignerListID", UINT32, 38); +CONSTRUCT_TYPED_SFIELD(sfSettleDelay, "SettleDelay", UINT32, 39); +CONSTRUCT_TYPED_SFIELD(sfTicketCount, "TicketCount", UINT32, 40); +CONSTRUCT_TYPED_SFIELD(sfTicketSequence, "TicketSequence", UINT32, 41); // 64-bit integers -SF_U64 const sfIndexNext(access, STI_UINT64, 1, "IndexNext"); -SF_U64 const sfIndexPrevious(access, STI_UINT64, 2, "IndexPrevious"); -SF_U64 const sfBookNode(access, STI_UINT64, 3, "BookNode"); -SF_U64 const sfOwnerNode(access, STI_UINT64, 4, "OwnerNode"); -SF_U64 const sfBaseFee(access, STI_UINT64, 5, "BaseFee"); -SF_U64 const sfExchangeRate(access, STI_UINT64, 6, "ExchangeRate"); -SF_U64 const sfLowNode(access, STI_UINT64, 7, "LowNode"); -SF_U64 const sfHighNode(access, STI_UINT64, 8, "HighNode"); -SF_U64 const sfDestinationNode(access, STI_UINT64, 9, "DestinationNode"); -SF_U64 const sfCookie(access, STI_UINT64, 10, "Cookie"); -SF_U64 const sfServerVersion(access, STI_UINT64, 11, "ServerVersion"); +CONSTRUCT_TYPED_SFIELD(sfIndexNext, "IndexNext", UINT64, 1); +CONSTRUCT_TYPED_SFIELD(sfIndexPrevious, "IndexPrevious", UINT64, 2); +CONSTRUCT_TYPED_SFIELD(sfBookNode, "BookNode", UINT64, 3); +CONSTRUCT_TYPED_SFIELD(sfOwnerNode, "OwnerNode", UINT64, 4); +CONSTRUCT_TYPED_SFIELD(sfBaseFee, "BaseFee", UINT64, 5); +CONSTRUCT_TYPED_SFIELD(sfExchangeRate, "ExchangeRate", UINT64, 6); +CONSTRUCT_TYPED_SFIELD(sfLowNode, "LowNode", UINT64, 7); +CONSTRUCT_TYPED_SFIELD(sfHighNode, "HighNode", UINT64, 8); +CONSTRUCT_TYPED_SFIELD(sfDestinationNode, "DestinationNode", UINT64, 9); +CONSTRUCT_TYPED_SFIELD(sfCookie, "Cookie", UINT64, 10); +CONSTRUCT_TYPED_SFIELD(sfServerVersion, "ServerVersion", UINT64, 11); // 128-bit -SF_U128 const sfEmailHash(access, STI_HASH128, 1, "EmailHash"); +CONSTRUCT_TYPED_SFIELD(sfEmailHash, "EmailHash", HASH128, 1); // 160-bit (common) -SF_U160 const sfTakerPaysCurrency(access, STI_HASH160, 1, "TakerPaysCurrency"); -SF_U160 const sfTakerPaysIssuer(access, STI_HASH160, 2, "TakerPaysIssuer"); -SF_U160 const sfTakerGetsCurrency(access, STI_HASH160, 3, "TakerGetsCurrency"); -SF_U160 const sfTakerGetsIssuer(access, STI_HASH160, 4, "TakerGetsIssuer"); +CONSTRUCT_TYPED_SFIELD(sfTakerPaysCurrency, "TakerPaysCurrency", HASH160, 1); +CONSTRUCT_TYPED_SFIELD(sfTakerPaysIssuer, "TakerPaysIssuer", HASH160, 2); +CONSTRUCT_TYPED_SFIELD(sfTakerGetsCurrency, "TakerGetsCurrency", HASH160, 3); +CONSTRUCT_TYPED_SFIELD(sfTakerGetsIssuer, "TakerGetsIssuer", HASH160, 4); // 256-bit (common) -SF_U256 const sfLedgerHash(access, STI_HASH256, 1, "LedgerHash"); -SF_U256 const sfParentHash(access, STI_HASH256, 2, "ParentHash"); -SF_U256 const sfTransactionHash(access, STI_HASH256, 3, "TransactionHash"); -SF_U256 const sfAccountHash(access, STI_HASH256, 4, "AccountHash"); -SF_U256 const sfPreviousTxnID( - access, - STI_HASH256, - 5, - "PreviousTxnID", - SField::sMD_DeleteFinal); -SF_U256 const sfLedgerIndex(access, STI_HASH256, 6, "LedgerIndex"); -SF_U256 const sfWalletLocator(access, STI_HASH256, 7, "WalletLocator"); -SF_U256 const - sfRootIndex(access, STI_HASH256, 8, "RootIndex", SField::sMD_Always); -SF_U256 const sfAccountTxnID(access, STI_HASH256, 9, "AccountTxnID"); +CONSTRUCT_TYPED_SFIELD(sfLedgerHash, "LedgerHash", HASH256, 1); +CONSTRUCT_TYPED_SFIELD(sfParentHash, "ParentHash", HASH256, 2); +CONSTRUCT_TYPED_SFIELD(sfTransactionHash, "TransactionHash", HASH256, 3); +CONSTRUCT_TYPED_SFIELD(sfAccountHash, "AccountHash", HASH256, 4); +CONSTRUCT_TYPED_SFIELD(sfPreviousTxnID, "PreviousTxnID", HASH256, 5, SField::sMD_DeleteFinal); +CONSTRUCT_TYPED_SFIELD(sfLedgerIndex, "LedgerIndex", HASH256, 6); +CONSTRUCT_TYPED_SFIELD(sfWalletLocator, "WalletLocator", HASH256, 7); +CONSTRUCT_TYPED_SFIELD(sfRootIndex, "RootIndex", HASH256, 8, SField::sMD_Always); +CONSTRUCT_TYPED_SFIELD(sfAccountTxnID, "AccountTxnID", HASH256, 9); // 256-bit (uncommon) -SF_U256 const sfBookDirectory(access, STI_HASH256, 16, "BookDirectory"); -SF_U256 const sfInvoiceID(access, STI_HASH256, 17, "InvoiceID"); -SF_U256 const sfNickname(access, STI_HASH256, 18, "Nickname"); -SF_U256 const sfAmendment(access, STI_HASH256, 19, "Amendment"); -// 20 is currently unused -SF_U256 const sfDigest(access, STI_HASH256, 21, "Digest"); -SF_U256 const sfPayChannel(access, STI_HASH256, 22, "Channel"); -SF_U256 const sfConsensusHash(access, STI_HASH256, 23, "ConsensusHash"); -SF_U256 const sfCheckID(access, STI_HASH256, 24, "CheckID"); -SF_U256 const sfValidatedHash(access, STI_HASH256, 25, "ValidatedHash"); +CONSTRUCT_TYPED_SFIELD(sfBookDirectory, "BookDirectory", HASH256, 16); +CONSTRUCT_TYPED_SFIELD(sfInvoiceID, "InvoiceID", HASH256, 17); +CONSTRUCT_TYPED_SFIELD(sfNickname, "Nickname", HASH256, 18); +CONSTRUCT_TYPED_SFIELD(sfAmendment, "Amendment", HASH256, 19); +// 20 is currently unused +CONSTRUCT_TYPED_SFIELD(sfDigest, "Digest", HASH256, 21); +CONSTRUCT_TYPED_SFIELD(sfChannel, "Channel", HASH256, 22); +CONSTRUCT_TYPED_SFIELD(sfConsensusHash, "ConsensusHash", HASH256, 23); +CONSTRUCT_TYPED_SFIELD(sfCheckID, "CheckID", HASH256, 24); +CONSTRUCT_TYPED_SFIELD(sfValidatedHash, "ValidatedHash", HASH256, 25); // currency amount (common) -SF_Amount const sfAmount(access, STI_AMOUNT, 1, "Amount"); -SF_Amount const sfBalance(access, STI_AMOUNT, 2, "Balance"); -SF_Amount const sfLimitAmount(access, STI_AMOUNT, 3, "LimitAmount"); -SF_Amount const sfTakerPays(access, STI_AMOUNT, 4, "TakerPays"); -SF_Amount const sfTakerGets(access, STI_AMOUNT, 5, "TakerGets"); -SF_Amount const sfLowLimit(access, STI_AMOUNT, 6, "LowLimit"); -SF_Amount const sfHighLimit(access, STI_AMOUNT, 7, "HighLimit"); -SF_Amount const sfFee(access, STI_AMOUNT, 8, "Fee"); -SF_Amount const sfSendMax(access, STI_AMOUNT, 9, "SendMax"); -SF_Amount const sfDeliverMin(access, STI_AMOUNT, 10, "DeliverMin"); +CONSTRUCT_TYPED_SFIELD(sfAmount, "Amount", AMOUNT, 1); +CONSTRUCT_TYPED_SFIELD(sfBalance, "Balance", AMOUNT, 2); +CONSTRUCT_TYPED_SFIELD(sfLimitAmount, "LimitAmount", AMOUNT, 3); +CONSTRUCT_TYPED_SFIELD(sfTakerPays, "TakerPays", AMOUNT, 4); +CONSTRUCT_TYPED_SFIELD(sfTakerGets, "TakerGets", AMOUNT, 5); +CONSTRUCT_TYPED_SFIELD(sfLowLimit, "LowLimit", AMOUNT, 6); +CONSTRUCT_TYPED_SFIELD(sfHighLimit, "HighLimit", AMOUNT, 7); +CONSTRUCT_TYPED_SFIELD(sfFee, "Fee", AMOUNT, 8); +CONSTRUCT_TYPED_SFIELD(sfSendMax, "SendMax", AMOUNT, 9); +CONSTRUCT_TYPED_SFIELD(sfDeliverMin, "DeliverMin", AMOUNT, 10); // currency amount (uncommon) -SF_Amount const sfMinimumOffer(access, STI_AMOUNT, 16, "MinimumOffer"); -SF_Amount const sfRippleEscrow(access, STI_AMOUNT, 17, "RippleEscrow"); -SF_Amount const sfDeliveredAmount(access, STI_AMOUNT, 18, "DeliveredAmount"); +CONSTRUCT_TYPED_SFIELD(sfMinimumOffer, "MinimumOffer", AMOUNT, 16); +CONSTRUCT_TYPED_SFIELD(sfRippleEscrow, "RippleEscrow", AMOUNT, 17); +CONSTRUCT_TYPED_SFIELD(sfDeliveredAmount, "DeliveredAmount", AMOUNT, 18); // variable length (common) -SF_Blob const sfPublicKey(access, STI_VL, 1, "PublicKey"); -SF_Blob const sfMessageKey(access, STI_VL, 2, "MessageKey"); -SF_Blob const sfSigningPubKey(access, STI_VL, 3, "SigningPubKey"); -SF_Blob const sfTxnSignature( - access, - STI_VL, - 4, - "TxnSignature", - SField::sMD_Default, - SField::notSigning); -SF_Blob const sfSignature( - access, - STI_VL, - 6, - "Signature", - SField::sMD_Default, - SField::notSigning); -SF_Blob const sfDomain(access, STI_VL, 7, "Domain"); -SF_Blob const sfFundCode(access, STI_VL, 8, "FundCode"); -SF_Blob const sfRemoveCode(access, STI_VL, 9, "RemoveCode"); -SF_Blob const sfExpireCode(access, STI_VL, 10, "ExpireCode"); -SF_Blob const sfCreateCode(access, STI_VL, 11, "CreateCode"); -SF_Blob const sfMemoType(access, STI_VL, 12, "MemoType"); -SF_Blob const sfMemoData(access, STI_VL, 13, "MemoData"); -SF_Blob const sfMemoFormat(access, STI_VL, 14, "MemoFormat"); +CONSTRUCT_TYPED_SFIELD(sfPublicKey, "PublicKey", VL, 1); +CONSTRUCT_TYPED_SFIELD(sfMessageKey, "MessageKey", VL, 2); +CONSTRUCT_TYPED_SFIELD(sfSigningPubKey, "SigningPubKey", VL, 3); +CONSTRUCT_TYPED_SFIELD(sfTxnSignature, "TxnSignature", VL, 4, SField::sMD_Default, SField::notSigning); +// Was 5 used and then obsoleted? +CONSTRUCT_TYPED_SFIELD(sfSignature, "Signature", VL, 6, SField::sMD_Default, SField::notSigning); +CONSTRUCT_TYPED_SFIELD(sfDomain, "Domain", VL, 7); +CONSTRUCT_TYPED_SFIELD(sfFundCode, "FundCode", VL, 8); +CONSTRUCT_TYPED_SFIELD(sfRemoveCode, "RemoveCode", VL, 9); +CONSTRUCT_TYPED_SFIELD(sfExpireCode, "ExpireCode", VL, 10); +CONSTRUCT_TYPED_SFIELD(sfCreateCode, "CreateCode", VL, 11); +CONSTRUCT_TYPED_SFIELD(sfMemoType, "MemoType", VL, 12); +CONSTRUCT_TYPED_SFIELD(sfMemoData, "MemoData", VL, 13); +CONSTRUCT_TYPED_SFIELD(sfMemoFormat, "MemoFormat", VL, 14); // variable length (uncommon) -SF_Blob const sfFulfillment(access, STI_VL, 16, "Fulfillment"); -SF_Blob const sfCondition(access, STI_VL, 17, "Condition"); -SF_Blob const sfMasterSignature( - access, - STI_VL, - 18, - "MasterSignature", - SField::sMD_Default, - SField::notSigning); -SF_Blob const sfUNLModifyValidator(access, STI_VL, 19, "UNLModifyValidator"); -SF_Blob const sfValidatorToDisable(access, STI_VL, 20, "ValidatorToDisable"); -SF_Blob const sfValidatorToReEnable(access, STI_VL, 21, "ValidatorToReEnable"); +CONSTRUCT_TYPED_SFIELD(sfFulfillment, "Fulfillment", VL, 16); +CONSTRUCT_TYPED_SFIELD(sfCondition, "Condition", VL, 17); +CONSTRUCT_TYPED_SFIELD(sfMasterSignature, "MasterSignature", VL, 18, SField::sMD_Default, SField::notSigning); +CONSTRUCT_TYPED_SFIELD(sfUNLModifyValidator, "UNLModifyValidator", VL, 19); +CONSTRUCT_TYPED_SFIELD(sfValidatorToDisable, "ValidatorToDisable", VL, 20); +CONSTRUCT_TYPED_SFIELD(sfValidatorToReEnable, "ValidatorToReEnable", VL, 21); // account -SF_Account const sfAccount(access, STI_ACCOUNT, 1, "Account"); -SF_Account const sfOwner(access, STI_ACCOUNT, 2, "Owner"); -SF_Account const sfDestination(access, STI_ACCOUNT, 3, "Destination"); -SF_Account const sfIssuer(access, STI_ACCOUNT, 4, "Issuer"); -SF_Account const sfAuthorize(access, STI_ACCOUNT, 5, "Authorize"); -SF_Account const sfUnauthorize(access, STI_ACCOUNT, 6, "Unauthorize"); -// 7 is currently unused -SF_Account const sfRegularKey(access, STI_ACCOUNT, 8, "RegularKey"); - -// path set -SField const sfPaths(access, STI_PATHSET, 1, "Paths"); +CONSTRUCT_TYPED_SFIELD(sfAccount, "Account", ACCOUNT, 1); +CONSTRUCT_TYPED_SFIELD(sfOwner, "Owner", ACCOUNT, 2); +CONSTRUCT_TYPED_SFIELD(sfDestination, "Destination", ACCOUNT, 3); +CONSTRUCT_TYPED_SFIELD(sfIssuer, "Issuer", ACCOUNT, 4); +CONSTRUCT_TYPED_SFIELD(sfAuthorize, "Authorize", ACCOUNT, 5); +CONSTRUCT_TYPED_SFIELD(sfUnauthorize, "Unauthorize", ACCOUNT, 6); +// 7 is currently unused +CONSTRUCT_TYPED_SFIELD(sfRegularKey, "RegularKey", ACCOUNT, 8); // vector of 256-bit -SF_Vec256 const - sfIndexes(access, STI_VECTOR256, 1, "Indexes", SField::sMD_Never); -SF_Vec256 const sfHashes(access, STI_VECTOR256, 2, "Hashes"); -SF_Vec256 const sfAmendments(access, STI_VECTOR256, 3, "Amendments"); +CONSTRUCT_TYPED_SFIELD(sfIndexes, "Indexes", VECTOR256, 1, SField::sMD_Never); +CONSTRUCT_TYPED_SFIELD(sfHashes, "Hashes", VECTOR256, 2); +CONSTRUCT_TYPED_SFIELD(sfAmendments, "Amendments", VECTOR256, 3); + +// path set +CONSTRUCT_UNTYPED_SFIELD(sfPaths, "Paths", PATHSET, 1); // inner object // OBJECT/1 is reserved for end of object -SField const - sfTransactionMetaData(access, STI_OBJECT, 2, "TransactionMetaData"); -SField const sfCreatedNode(access, STI_OBJECT, 3, "CreatedNode"); -SField const sfDeletedNode(access, STI_OBJECT, 4, "DeletedNode"); -SField const sfModifiedNode(access, STI_OBJECT, 5, "ModifiedNode"); -SField const sfPreviousFields(access, STI_OBJECT, 6, "PreviousFields"); -SField const sfFinalFields(access, STI_OBJECT, 7, "FinalFields"); -SField const sfNewFields(access, STI_OBJECT, 8, "NewFields"); -SField const sfTemplateEntry(access, STI_OBJECT, 9, "TemplateEntry"); -SField const sfMemo(access, STI_OBJECT, 10, "Memo"); -SField const sfSignerEntry(access, STI_OBJECT, 11, "SignerEntry"); +CONSTRUCT_UNTYPED_SFIELD(sfTransactionMetaData, "TransactionMetaData", OBJECT, 2); +CONSTRUCT_UNTYPED_SFIELD(sfCreatedNode, "CreatedNode", OBJECT, 3); +CONSTRUCT_UNTYPED_SFIELD(sfDeletedNode, "DeletedNode", OBJECT, 4); +CONSTRUCT_UNTYPED_SFIELD(sfModifiedNode, "ModifiedNode", OBJECT, 5); +CONSTRUCT_UNTYPED_SFIELD(sfPreviousFields, "PreviousFields", OBJECT, 6); +CONSTRUCT_UNTYPED_SFIELD(sfFinalFields, "FinalFields", OBJECT, 7); +CONSTRUCT_UNTYPED_SFIELD(sfNewFields, "NewFields", OBJECT, 8); +CONSTRUCT_UNTYPED_SFIELD(sfTemplateEntry, "TemplateEntry", OBJECT, 9); +CONSTRUCT_UNTYPED_SFIELD(sfMemo, "Memo", OBJECT, 10); +CONSTRUCT_UNTYPED_SFIELD(sfSignerEntry, "SignerEntry", OBJECT, 11); // inner object (uncommon) -SField const sfSigner(access, STI_OBJECT, 16, "Signer"); -// 17 has not been used yet... -SField const sfMajority(access, STI_OBJECT, 18, "Majority"); -SField const sfDisabledValidator(access, STI_OBJECT, 19, "DisabledValidator"); +CONSTRUCT_UNTYPED_SFIELD(sfSigner, "Signer", OBJECT, 16); +// 17 has not been used yet +CONSTRUCT_UNTYPED_SFIELD(sfMajority, "Majority", OBJECT, 18); +CONSTRUCT_UNTYPED_SFIELD(sfDisabledValidator, "DisabledValidator", OBJECT, 19); // array of objects -// ARRAY/1 is reserved for end of array -// SField const sfSigningAccounts (access, STI_ARRAY, 2, "SigningAccounts"); // -// Never been used. -SField const sfSigners( - access, - STI_ARRAY, - 3, - "Signers", - SField::sMD_Default, - SField::notSigning); -SField const sfSignerEntries(access, STI_ARRAY, 4, "SignerEntries"); -SField const sfTemplate(access, STI_ARRAY, 5, "Template"); -SField const sfNecessary(access, STI_ARRAY, 6, "Necessary"); -SField const sfSufficient(access, STI_ARRAY, 7, "Sufficient"); -SField const sfAffectedNodes(access, STI_ARRAY, 8, "AffectedNodes"); -SField const sfMemos(access, STI_ARRAY, 9, "Memos"); +// ARRAY/1 is reserved for end of array +// 2 has never been used +CONSTRUCT_UNTYPED_SFIELD(sfSigners, "Signers", ARRAY, 3, SField::sMD_Default, SField::notSigning); +CONSTRUCT_UNTYPED_SFIELD(sfSignerEntries, "SignerEntries", ARRAY, 4); +CONSTRUCT_UNTYPED_SFIELD(sfTemplate, "Template", ARRAY, 5); +CONSTRUCT_UNTYPED_SFIELD(sfNecessary, "Necessary", ARRAY, 6); +CONSTRUCT_UNTYPED_SFIELD(sfSufficient, "Sufficient", ARRAY, 7); +CONSTRUCT_UNTYPED_SFIELD(sfAffectedNodes, "AffectedNodes", ARRAY, 8); +CONSTRUCT_UNTYPED_SFIELD(sfMemos, "Memos", ARRAY, 9); // array of objects (uncommon) -SField const sfMajorities(access, STI_ARRAY, 16, "Majorities"); -SField const sfDisabledValidators(access, STI_ARRAY, 17, "DisabledValidators"); +CONSTRUCT_UNTYPED_SFIELD(sfMajorities, "Majorities", ARRAY, 16); +CONSTRUCT_UNTYPED_SFIELD(sfDisabledValidators, "DisabledValidators", ARRAY, 17); + +// clang-format on + +#undef CONSTRUCT_TYPED_SFIELD +#undef CONSTRUCT_UNTYPED_SFIELD + +#pragma pop_macro("CONSTRUCT_TYPED_SFIELD") +#pragma pop_macro("CONSTRUCT_UNTYPED_SFIELD") SField::SField( private_access_tag_t, diff --git a/src/ripple/protocol/impl/STBase.cpp b/src/ripple/protocol/impl/STBase.cpp index 2c7d4b4f71a..f447b1fe236 100644 --- a/src/ripple/protocol/impl/STBase.cpp +++ b/src/ripple/protocol/impl/STBase.cpp @@ -36,16 +36,6 @@ STBase::STBase(SField const& n) : fName(&n) STBase& STBase::operator=(const STBase& t) { - if ((t.fName != fName) && fName->isUseful() && t.fName->isUseful()) - { - // VFALCO We shouldn't be logging at this low a level - /* - WriteLog ((t.getSType () == STI_AMOUNT) ? lsTRACE : lsWARNING, STBase) - // This is common for amounts - << "Caution: " << t.fName->getName () << " not replacing " << - fName->getName (); - */ - } if (!fName->isUseful()) fName = t.fName; return *this; @@ -110,12 +100,7 @@ bool STBase::isEquivalent(const STBase& t) const { assert(getSType() == STI_NOTPRESENT); - if (t.getSType() == STI_NOTPRESENT) - return true; - // VFALCO We shouldn't be logging at this low a level - // WriteLog (lsDEBUG, STBase) << "notEquiv " << getFullText() << " not - // STI_NOTPRESENT"; - return false; + return t.getSType() == STI_NOTPRESENT; } bool diff --git a/src/ripple/protocol/impl/STInteger.cpp b/src/ripple/protocol/impl/STInteger.cpp index dc580388d6a..2b154e369e4 100644 --- a/src/ripple/protocol/impl/STInteger.cpp +++ b/src/ripple/protocol/impl/STInteger.cpp @@ -25,6 +25,7 @@ #include #include #include +#include namespace ripple { @@ -56,7 +57,7 @@ STUInt8::getText() const << "Unknown result code in metadata: " << value_; } - return beast::lexicalCastThrow(value_); + return std::to_string(value_); } template <> @@ -113,7 +114,7 @@ STUInt16::getText() const return item->getName(); } - return beast::lexicalCastThrow(value_); + return std::to_string(value_); } template <> @@ -159,7 +160,7 @@ template <> std::string STUInt32::getText() const { - return beast::lexicalCastThrow(value_); + return std::to_string(value_); } template <> @@ -187,13 +188,17 @@ template <> std::string STUInt64::getText() const { - return beast::lexicalCastThrow(value_); + return std::to_string(value_); } template <> Json::Value STUInt64::getJson(JsonOptions) const { - return strHex(value_); + std::string str(16, 0); + auto ret = std::to_chars(str.data(), str.data() + str.size(), value_, 16); + assert(ret.ec == std::errc()); + str.resize(std::distance(str.data(), ret.ptr)); + return str; } } // namespace ripple diff --git a/src/ripple/protocol/impl/STLedgerEntry.cpp b/src/ripple/protocol/impl/STLedgerEntry.cpp index 6211e3e7f03..9c0ac3f511b 100644 --- a/src/ripple/protocol/impl/STLedgerEntry.cpp +++ b/src/ripple/protocol/impl/STLedgerEntry.cpp @@ -25,22 +25,29 @@ #include #include #include +#include namespace ripple { STLedgerEntry::STLedgerEntry(Keylet const& k) : STObject(sfLedgerEntry), key_(k.key), type_(k.type) { - if (!(0u <= type_ && - type_ <= std::min( - std::numeric_limits::max(), - std::numeric_limits< - std::underlying_type_t>::max()))) + // The on-ledger representation of a key type is a 16-bit unsigned integer + // but the LedgerEntryType enum has a larger range (including negative + // values), so catch obviously wrong values: + constexpr auto const minValidValue = + static_cast(std::numeric_limits::min()); + + constexpr auto const maxValidValue = + static_cast(std::numeric_limits::max()); + + if (type_ < minValidValue || type_ > maxValidValue) Throw("invalid ledger entry type: out of range"); + auto const format = LedgerFormats::getInstance().findByType(type_); if (format == nullptr) - Throw("invalid ledger entry type"); + Throw("unknown ledger entry type"); set(format->getSOTemplate()); diff --git a/src/ripple/protocol/impl/STParsedJSON.cpp b/src/ripple/protocol/impl/STParsedJSON.cpp index 7baad35a9f6..da5e53b3320 100644 --- a/src/ripple/protocol/impl/STParsedJSON.cpp +++ b/src/ripple/protocol/impl/STParsedJSON.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include namespace ripple { @@ -403,8 +404,17 @@ parseLeaf( { if (value.isString()) { - ret = detail::make_stvar( - field, uintFromHex(value.asString())); + auto const str = value.asString(); + + std::uint64_t val; + + auto [p, ec] = std::from_chars( + str.data(), str.data() + str.size(), val, 16); + + if (ec != std::errc() || (p != str.data() + str.size())) + Throw("invalid data"); + + ret = detail::make_stvar(field, val); } else if (value.isInt()) { @@ -430,71 +440,77 @@ parseLeaf( break; - case STI_HASH128: - try + case STI_HASH128: { + if (!value.isString()) { - if (value.isString()) - { - ret = - detail::make_stvar(field, value.asString()); - } - else + error = bad_type(json_name, fieldName); + return ret; + } + + uint128 num; + + if (auto const s = value.asString(); !num.parseHex(s)) + { + if (!s.empty()) { - error = bad_type(json_name, fieldName); + error = invalid_data(json_name, fieldName); return ret; } + + num.zero(); } - catch (std::exception const&) + + ret = detail::make_stvar(field, num); + break; + } + + case STI_HASH160: { + if (!value.isString()) { - error = invalid_data(json_name, fieldName); + error = bad_type(json_name, fieldName); return ret; } - break; + uint160 num; - case STI_HASH160: - try + if (auto const s = value.asString(); !num.parseHex(s)) { - if (value.isString()) - { - ret = - detail::make_stvar(field, value.asString()); - } - else + if (!s.empty()) { - error = bad_type(json_name, fieldName); + error = invalid_data(json_name, fieldName); return ret; } + + num.zero(); } - catch (std::exception const&) + + ret = detail::make_stvar(field, num); + break; + } + + case STI_HASH256: { + if (!value.isString()) { - error = invalid_data(json_name, fieldName); + error = bad_type(json_name, fieldName); return ret; } - break; + uint256 num; - case STI_HASH256: - try + if (auto const s = value.asString(); !num.parseHex(s)) { - if (value.isString()) - { - ret = - detail::make_stvar(field, value.asString()); - } - else + if (!s.empty()) { - error = bad_type(json_name, fieldName); + error = invalid_data(json_name, fieldName); return ret; } - } - catch (std::exception const&) - { - error = invalid_data(json_name, fieldName); - return ret; + + num.zero(); } + ret = detail::make_stvar(field, num); break; + } case STI_VL: if (!value.isString()) @@ -550,7 +566,8 @@ parseLeaf( for (Json::UInt i = 0; value.isValidIndex(i); ++i) { uint256 s; - s.SetHex(value[i].asString()); + if (!s.parseHex(value[i].asString())) + Throw("invalid data"); tail.push_back(s); } ret = detail::make_stvar(std::move(tail)); @@ -623,7 +640,7 @@ parseLeaf( // If we have what looks like a 160-bit hex value, // we set it, otherwise, we assume it's an AccountID - if (!uAccount.SetHexExact(account.asString())) + if (!uAccount.parseHex(account.asString())) { auto const a = parseBase58(account.asString()); @@ -649,7 +666,7 @@ parseLeaf( hasCurrency = true; - if (!uCurrency.SetHexExact(currency.asString())) + if (!uCurrency.parseHex(currency.asString())) { if (!to_currency( uCurrency, currency.asString())) @@ -670,7 +687,7 @@ parseLeaf( return ret; } - if (!uIssuer.SetHexExact(issuer.asString())) + if (!uIssuer.parseHex(issuer.asString())) { auto const a = parseBase58(issuer.asString()); @@ -711,14 +728,14 @@ parseLeaf( try { - // VFALCO This needs careful auditing - auto const account = parseHexOrBase58(strValue); - if (!account) - { - error = invalid_data(json_name, fieldName); - return ret; - } - ret = detail::make_stvar(field, *account); + if (AccountID account; account.parseHex(strValue)) + return detail::make_stvar(field, account); + + if (auto result = parseBase58(strValue)) + return detail::make_stvar(field, *result); + + error = invalid_data(json_name, fieldName); + return ret; } catch (std::exception const&) { diff --git a/src/ripple/protocol/impl/STPathSet.cpp b/src/ripple/protocol/impl/STPathSet.cpp index 2825c7cfbce..141fad329ce 100644 --- a/src/ripple/protocol/impl/STPathSet.cpp +++ b/src/ripple/protocol/impl/STPathSet.cpp @@ -156,7 +156,6 @@ Json::Value STPath::getJson(JsonOptions) const auto const iType = it.getNodeType(); elem[jss::type] = iType; - elem[jss::type_hex] = strHex(iType); if (iType & STPathElement::typeAccount) elem[jss::account] = to_string(it.getAccountID()); diff --git a/src/ripple/protocol/impl/STTx.cpp b/src/ripple/protocol/impl/STTx.cpp index 14bffddea55..6e1b86f1fb2 100644 --- a/src/ripple/protocol/impl/STTx.cpp +++ b/src/ripple/protocol/impl/STTx.cpp @@ -259,7 +259,7 @@ STTx::getMetaSQL( { static boost::format bfTrans( "('%s', '%s', '%s', '%d', '%d', '%c', %s, %s)"); - std::string rTxn = sqlEscape(rawTxn.peekData()); + std::string rTxn = sqlBlobLiteral(rawTxn.peekData()); auto format = TxFormats::getInstance().findByType(tx_type_); assert(format != nullptr); diff --git a/src/ripple/protocol/impl/Seed.cpp b/src/ripple/protocol/impl/Seed.cpp index 02c1ad2e3ed..f38556c2ecb 100644 --- a/src/ripple/protocol/impl/Seed.cpp +++ b/src/ripple/protocol/impl/Seed.cpp @@ -104,7 +104,7 @@ parseGenericSeed(std::string const& str) { uint128 seed; - if (seed.SetHexExact(str)) + if (seed.parseHex(str)) return Seed{Slice(seed.data(), seed.size())}; } diff --git a/src/ripple/protocol/impl/Sign.cpp b/src/ripple/protocol/impl/Sign.cpp index 3e667097af3..b6313827f91 100644 --- a/src/ripple/protocol/impl/Sign.cpp +++ b/src/ripple/protocol/impl/Sign.cpp @@ -27,7 +27,7 @@ sign( HashPrefix const& prefix, KeyType type, SecretKey const& sk, - SF_Blob const& sigField) + SF_VL const& sigField) { Serializer ss; ss.add32(prefix); @@ -40,7 +40,7 @@ verify( STObject const& st, HashPrefix const& prefix, PublicKey const& pk, - SF_Blob const& sigField) + SF_VL const& sigField) { auto const sig = get(st, sigField); if (!sig) diff --git a/src/ripple/protocol/impl/TxFormats.cpp b/src/ripple/protocol/impl/TxFormats.cpp index dd94c09c1a0..ff3f7f507a9 100644 --- a/src/ripple/protocol/impl/TxFormats.cpp +++ b/src/ripple/protocol/impl/TxFormats.cpp @@ -205,7 +205,7 @@ TxFormats::TxFormats() add(jss::PaymentChannelFund, ttPAYCHAN_FUND, { - {sfPayChannel, soeREQUIRED}, + {sfChannel, soeREQUIRED}, {sfAmount, soeREQUIRED}, {sfExpiration, soeOPTIONAL}, {sfTicketSequence, soeOPTIONAL}, @@ -215,7 +215,7 @@ TxFormats::TxFormats() add(jss::PaymentChannelClaim, ttPAYCHAN_CLAIM, { - {sfPayChannel, soeREQUIRED}, + {sfChannel, soeREQUIRED}, {sfAmount, soeOPTIONAL}, {sfBalance, soeOPTIONAL}, {sfSignature, soeOPTIONAL}, diff --git a/src/ripple/protocol/impl/UintTypes.cpp b/src/ripple/protocol/impl/UintTypes.cpp index 698555b9a06..39e25a81369 100644 --- a/src/ripple/protocol/impl/UintTypes.cpp +++ b/src/ripple/protocol/impl/UintTypes.cpp @@ -55,8 +55,11 @@ to_string(Currency const& currency) if (currency == noCurrency()) return "1"; - static Currency const sIsoBits( - from_hex_text("FFFFFFFFFFFFFFFFFFFFFFFF000000FFFFFFFFFF")); + static Currency const sIsoBits = []() { + Currency c; + (void)c.parseHex("FFFFFFFFFFFFFFFFFFFFFFFF000000FFFFFFFFFF"); + return c; + }(); if ((currency & sIsoBits).isZero()) { @@ -105,7 +108,7 @@ to_currency(Currency& currency, std::string const& code) return true; } - return currency.SetHexExact(code); + return currency.parseHex(code); } Currency diff --git a/src/ripple/protocol/jss.h b/src/ripple/protocol/jss.h index 216fdfe0f33..49feddd9d67 100644 --- a/src/ripple/protocol/jss.h +++ b/src/ripple/protocol/jss.h @@ -459,7 +459,6 @@ JSS(role); // out: Ping.cpp JSS(rpc); JSS(rt_accounts); // in: Subscribe, Unsubscribe JSS(running_duration_us); -JSS(sanity); // out: PeerImp JSS(search_depth); // in: RipplePathFind JSS(searched_all); // out: Tx JSS(secret); // in: TransactionSign, @@ -520,6 +519,7 @@ JSS(ticket_count); // out: AccountInfo JSS(ticket_seq); // in: LedgerEntry JSS(time); JSS(timeouts); // out: InboundLedger +JSS(track); // out: PeerImp JSS(traffic); // out: Overlay JSS(total); // out: counters JSS(totalCoins); // out: LedgerToJson diff --git a/src/ripple/protocol/tokens.h b/src/ripple/protocol/tokens.h index 1da8ca99345..d507c2542e3 100644 --- a/src/ripple/protocol/tokens.h +++ b/src/ripple/protocol/tokens.h @@ -45,14 +45,6 @@ template boost::optional parseBase58(TokenType type, std::string const& s); -template -boost::optional -parseHex(std::string const& s); - -template -boost::optional -parseHexOrBase58(std::string const& s); - /** Encode data in Base58Check format using XRPL alphabet For details on the format see diff --git a/src/ripple/rpc/handlers/AccountChannels.cpp b/src/ripple/rpc/handlers/AccountChannels.cpp index b1c85cc80a9..d222e4c72cd 100644 --- a/src/ripple/rpc/handlers/AccountChannels.cpp +++ b/src/ripple/rpc/handlers/AccountChannels.cpp @@ -121,7 +121,7 @@ doAccountChannels(RPC::JsonContext& context) if (!marker.isString()) return RPC::expected_field_error(jss::marker, "string"); - if (!startAfter.SetHex(marker.asString())) + if (!startAfter.parseHex(marker.asString())) { return rpcError(rpcINVALID_PARAMS); } diff --git a/src/ripple/rpc/handlers/AccountLines.cpp b/src/ripple/rpc/handlers/AccountLines.cpp index 947b0a3be06..2e1dac1d71d 100644 --- a/src/ripple/rpc/handlers/AccountLines.cpp +++ b/src/ripple/rpc/handlers/AccountLines.cpp @@ -139,7 +139,9 @@ doAccountLines(RPC::JsonContext& context) if (!marker.isString()) return RPC::expected_field_error(jss::marker, "string"); - startAfter.SetHex(marker.asString()); + if (!startAfter.parseHex(marker.asString())) + return rpcError(rpcINVALID_PARAMS); + auto const sleLine = ledger->read({ltRIPPLE_STATE, startAfter}); if (!sleLine) diff --git a/src/ripple/rpc/handlers/AccountObjects.cpp b/src/ripple/rpc/handlers/AccountObjects.cpp index 464031e93d1..1783c89da98 100644 --- a/src/ripple/rpc/handlers/AccountObjects.cpp +++ b/src/ripple/rpc/handlers/AccountObjects.cpp @@ -134,13 +134,13 @@ doAccountObjects(RPC::JsonContext& context) if (!std::getline(ss, s, ',')) return RPC::invalid_field_error(jss::marker); - if (!dirIndex.SetHex(s)) + if (!dirIndex.parseHex(s)) return RPC::invalid_field_error(jss::marker); if (!std::getline(ss, s, ',')) return RPC::invalid_field_error(jss::marker); - if (!entryIndex.SetHex(s)) + if (!entryIndex.parseHex(s)) return RPC::invalid_field_error(jss::marker); } diff --git a/src/ripple/rpc/handlers/AccountOffers.cpp b/src/ripple/rpc/handlers/AccountOffers.cpp index 9836417293b..063d0874378 100644 --- a/src/ripple/rpc/handlers/AccountOffers.cpp +++ b/src/ripple/rpc/handlers/AccountOffers.cpp @@ -101,7 +101,9 @@ doAccountOffers(RPC::JsonContext& context) if (!marker.isString()) return RPC::expected_field_error(jss::marker, "string"); - startAfter.SetHex(marker.asString()); + if (!startAfter.parseHex(marker.asString())) + return rpcError(rpcINVALID_PARAMS); + auto const sleOffer = ledger->read({ltOFFER, startAfter}); if (!sleOffer || accountID != sleOffer->getAccountID(sfAccount)) diff --git a/src/ripple/rpc/handlers/AccountTx.cpp b/src/ripple/rpc/handlers/AccountTx.cpp index ffcbe145fbb..104f5553573 100644 --- a/src/ripple/rpc/handlers/AccountTx.cpp +++ b/src/ripple/rpc/handlers/AccountTx.cpp @@ -165,7 +165,7 @@ parseLedgerArgs(Json::Value const& params) } LedgerHash hash; - if (!hash.SetHex(hashValue.asString())) + if (!hash.parseHex(hashValue.asString())) { RPC::Status status{rpcINVALID_PARAMS, "ledgerHashMalformed"}; status.inject(response); diff --git a/src/ripple/rpc/handlers/CanDelete.cpp b/src/ripple/rpc/handlers/CanDelete.cpp index d302a2f1b21..56015375a26 100644 --- a/src/ripple/rpc/handlers/CanDelete.cpp +++ b/src/ripple/rpc/handlers/CanDelete.cpp @@ -71,13 +71,9 @@ doCanDelete(RPC::JsonContext& context) if (!canDeleteSeq) return RPC::make_error(rpcNOT_READY); } - else if ( - canDeleteStr.size() == 64 && - canDeleteStr.find_first_not_of("0123456789abcdef") == - std::string::npos) + else if (uint256 lh; lh.parseHex(canDeleteStr)) { - auto ledger = context.ledgerMaster.getLedgerByHash( - from_hex_text(canDeleteStr)); + auto ledger = context.ledgerMaster.getLedgerByHash(lh); if (!ledger) return RPC::make_error(rpcLGR_NOT_FOUND, "ledgerNotFound"); diff --git a/src/ripple/rpc/handlers/Feature1.cpp b/src/ripple/rpc/handlers/Feature1.cpp index 47a284f309e..086d6acd26a 100644 --- a/src/ripple/rpc/handlers/Feature1.cpp +++ b/src/ripple/rpc/handlers/Feature1.cpp @@ -60,8 +60,7 @@ doFeature(RPC::JsonContext& context) auto feature = table.find(context.params[jss::feature].asString()); - if (!feature && - !feature.SetHexExact(context.params[jss::feature].asString())) + if (!feature && !feature.parseHex(context.params[jss::feature].asString())) return rpcError(rpcBAD_FEATURE); if (context.params.isMember(jss::vetoed)) diff --git a/src/ripple/rpc/handlers/LedgerData.cpp b/src/ripple/rpc/handlers/LedgerData.cpp index 9bad1485473..4cb0b4d4663 100644 --- a/src/ripple/rpc/handlers/LedgerData.cpp +++ b/src/ripple/rpc/handlers/LedgerData.cpp @@ -55,7 +55,7 @@ doLedgerData(RPC::JsonContext& context) if (isMarker) { Json::Value const& jMarker = params[jss::marker]; - if (!(jMarker.isString() && key.SetHex(jMarker.asString()))) + if (!(jMarker.isString() && key.parseHex(jMarker.asString()))) return RPC::expected_field_error(jss::marker, "valid"); } diff --git a/src/ripple/rpc/handlers/LedgerEntry.cpp b/src/ripple/rpc/handlers/LedgerEntry.cpp index 6d0fbbdb037..933c61f23fa 100644 --- a/src/ripple/rpc/handlers/LedgerEntry.cpp +++ b/src/ripple/rpc/handlers/LedgerEntry.cpp @@ -50,7 +50,11 @@ doLedgerEntry(RPC::JsonContext& context) if (context.params.isMember(jss::index)) { - uNodeIndex.SetHex(context.params[jss::index].asString()); + if (!uNodeIndex.parseHex(context.params[jss::index].asString())) + { + uNodeIndex = beast::zero; + jvResult[jss::error] = "malformedRequest"; + } } else if (context.params.isMember(jss::account_root)) { @@ -65,7 +69,12 @@ doLedgerEntry(RPC::JsonContext& context) else if (context.params.isMember(jss::check)) { expectedType = ltCHECK; - uNodeIndex.SetHex(context.params[jss::check].asString()); + + if (!uNodeIndex.parseHex(context.params[jss::check].asString())) + { + uNodeIndex = beast::zero; + jvResult[jss::error] = "malformedRequest"; + } } else if (context.params.isMember(jss::deposit_preauth)) { @@ -74,7 +83,7 @@ doLedgerEntry(RPC::JsonContext& context) if (!context.params[jss::deposit_preauth].isObject()) { if (!context.params[jss::deposit_preauth].isString() || - !uNodeIndex.SetHex( + !uNodeIndex.parseHex( context.params[jss::deposit_preauth].asString())) { uNodeIndex = beast::zero; @@ -115,7 +124,11 @@ doLedgerEntry(RPC::JsonContext& context) } else if (!context.params[jss::directory].isObject()) { - uNodeIndex.SetHex(context.params[jss::directory].asString()); + if (!uNodeIndex.parseHex(context.params[jss::directory].asString())) + { + uNodeIndex = beast::zero; + jvResult[jss::error] = "malformedRequest"; + } } else if ( context.params[jss::directory].isMember(jss::sub_index) && @@ -132,18 +145,22 @@ doLedgerEntry(RPC::JsonContext& context) if (context.params[jss::directory].isMember(jss::dir_root)) { + uint256 uDirRoot; + if (context.params[jss::directory].isMember(jss::owner)) { // May not specify both dir_root and owner. jvResult[jss::error] = "malformedRequest"; } + else if (!uDirRoot.parseHex( + context.params[jss::directory][jss::dir_root] + .asString())) + { + uNodeIndex = beast::zero; + jvResult[jss::error] = "malformedRequest"; + } else { - uint256 uDirRoot; - uDirRoot.SetHex( - context.params[jss::directory][jss::dir_root] - .asString()); - uNodeIndex = keylet::page(uDirRoot, uSubIndex).key; } } @@ -173,7 +190,11 @@ doLedgerEntry(RPC::JsonContext& context) expectedType = ltESCROW; if (!context.params[jss::escrow].isObject()) { - uNodeIndex.SetHex(context.params[jss::escrow].asString()); + if (!uNodeIndex.parseHex(context.params[jss::escrow].asString())) + { + uNodeIndex = beast::zero; + jvResult[jss::error] = "malformedRequest"; + } } else if ( !context.params[jss::escrow].isMember(jss::owner) || @@ -200,7 +221,11 @@ doLedgerEntry(RPC::JsonContext& context) expectedType = ltOFFER; if (!context.params[jss::offer].isObject()) { - uNodeIndex.SetHex(context.params[jss::offer].asString()); + if (!uNodeIndex.parseHex(context.params[jss::offer].asString())) + { + uNodeIndex = beast::zero; + jvResult[jss::error] = "malformedRequest"; + } } else if ( !context.params[jss::offer].isMember(jss::account) || @@ -225,7 +250,13 @@ doLedgerEntry(RPC::JsonContext& context) else if (context.params.isMember(jss::payment_channel)) { expectedType = ltPAYCHAN; - uNodeIndex.SetHex(context.params[jss::payment_channel].asString()); + + if (!uNodeIndex.parseHex( + context.params[jss::payment_channel].asString())) + { + uNodeIndex = beast::zero; + jvResult[jss::error] = "malformedRequest"; + } } else if (context.params.isMember(jss::ripple_state)) { @@ -271,7 +302,11 @@ doLedgerEntry(RPC::JsonContext& context) expectedType = ltTICKET; if (!context.params[jss::ticket].isObject()) { - uNodeIndex.SetHex(context.params[jss::ticket].asString()); + if (!uNodeIndex.parseHex(context.params[jss::ticket].asString())) + { + uNodeIndex = beast::zero; + jvResult[jss::error] = "malformedRequest"; + } } else if ( !context.params[jss::ticket].isMember(jss::account) || diff --git a/src/ripple/rpc/handlers/LedgerRequest.cpp b/src/ripple/rpc/handlers/LedgerRequest.cpp index 5d2d11b00aa..dfb9d099ff3 100644 --- a/src/ripple/rpc/handlers/LedgerRequest.cpp +++ b/src/ripple/rpc/handlers/LedgerRequest.cpp @@ -55,7 +55,7 @@ doLedgerRequest(RPC::JsonContext& context) if (hasHash) { auto const& jsonHash = context.params[jss::ledger_hash]; - if (!jsonHash.isString() || !ledgerHash.SetHex(jsonHash.asString())) + if (!jsonHash.isString() || !ledgerHash.parseHex(jsonHash.asString())) return RPC::invalid_field_error(jss::ledger_hash); } else diff --git a/src/ripple/rpc/handlers/PayChanClaim.cpp b/src/ripple/rpc/handlers/PayChanClaim.cpp index ddc09f8aab6..9c46893ddfa 100644 --- a/src/ripple/rpc/handlers/PayChanClaim.cpp +++ b/src/ripple/rpc/handlers/PayChanClaim.cpp @@ -60,7 +60,7 @@ doChannelAuthorize(RPC::JsonContext& context) return result; uint256 channelId; - if (!channelId.SetHexExact(params[jss::channel_id].asString())) + if (!channelId.parseHex(params[jss::channel_id].asString())) return rpcError(rpcCHANNEL_MALFORMED); boost::optional const optDrops = @@ -122,7 +122,7 @@ doChannelVerify(RPC::JsonContext& context) } uint256 channelId; - if (!channelId.SetHexExact(params[jss::channel_id].asString())) + if (!channelId.parseHex(params[jss::channel_id].asString())) return rpcError(rpcCHANNEL_MALFORMED); boost::optional const optDrops = diff --git a/src/ripple/rpc/handlers/Peers.cpp b/src/ripple/rpc/handlers/Peers.cpp index 0a7d424f8e4..8c94d9c3bd9 100644 --- a/src/ripple/rpc/handlers/Peers.cpp +++ b/src/ripple/rpc/handlers/Peers.cpp @@ -32,17 +32,33 @@ doPeers(RPC::JsonContext& context) { Json::Value jvResult(Json::objectValue); + jvResult[jss::peers] = context.app.overlay().json(); + + // Legacy support + if (context.apiVersion == 1) { - jvResult[jss::peers] = context.app.overlay().json(); + for (auto& p : jvResult[jss::peers]) + { + if (p.isMember(jss::track)) + { + auto const s = p[jss::track].asString(); + + if (s == "diverged") + p["sanity"] = "insane"; + else if (s == "unknown") + p["sanity"] = "unknown"; + } + } + } - auto const now = context.app.timeKeeper().now(); - auto const self = context.app.nodeIdentity().first; + auto const now = context.app.timeKeeper().now(); + auto const self = context.app.nodeIdentity().first; - Json::Value& cluster = (jvResult[jss::cluster] = Json::objectValue); - std::uint32_t ref = context.app.getFeeTrack().getLoadBase(); + Json::Value& cluster = (jvResult[jss::cluster] = Json::objectValue); + std::uint32_t ref = context.app.getFeeTrack().getLoadBase(); - context.app.cluster().for_each([&cluster, now, ref, &self]( - ClusterNode const& node) { + context.app.cluster().for_each( + [&cluster, now, ref, &self](ClusterNode const& node) { if (node.identity() == self) return; @@ -60,7 +76,6 @@ doPeers(RPC::JsonContext& context) ? 0 : (now - node.getReportTime()).count(); }); - } return jvResult; } diff --git a/src/ripple/rpc/handlers/TransactionEntry.cpp b/src/ripple/rpc/handlers/TransactionEntry.cpp index 3731b60637b..83c4d18a236 100644 --- a/src/ripple/rpc/handlers/TransactionEntry.cpp +++ b/src/ripple/rpc/handlers/TransactionEntry.cpp @@ -57,7 +57,11 @@ doTransactionEntry(RPC::JsonContext& context) uint256 uTransID; // XXX Relying on trusted WSS client. Would be better to have a strict // routine, returning success or failure. - uTransID.SetHex(context.params[jss::tx_hash].asString()); + if (!uTransID.parseHex(context.params[jss::tx_hash].asString())) + { + jvResult[jss::error] = "malformedRequest"; + return jvResult; + } auto [sttx, stobj] = lpLedger->txRead(uTransID); if (!sttx) diff --git a/src/ripple/rpc/handlers/Tx.cpp b/src/ripple/rpc/handlers/Tx.cpp index e5ab4b8dd3f..613128a47a4 100644 --- a/src/ripple/rpc/handlers/Tx.cpp +++ b/src/ripple/rpc/handlers/Tx.cpp @@ -37,20 +37,6 @@ namespace ripple { // transaction: // } -static bool -isHexTxID(std::string const& txid) -{ - if (txid.size() != 64) - return false; - - auto const ret = - std::find_if(txid.begin(), txid.end(), [](std::string::value_type c) { - return !std::isxdigit(static_cast(c)); - }); - - return (ret == txid.end()); -} - static bool isValidated(LedgerMaster& ledgerMaster, std::uint32_t seq, uint256 const& hash) { @@ -63,24 +49,6 @@ isValidated(LedgerMaster& ledgerMaster, std::uint32_t seq, uint256 const& hash) return ledgerMaster.getHashBySeq(seq) == hash; } -bool -getMetaHex(Ledger const& ledger, uint256 const& transID, std::string& hex) -{ - SHAMapTreeNode::TNType type; - auto const item = ledger.txMap().peekItem(transID, type); - - if (!item) - return false; - - if (type != SHAMapTreeNode::tnTRANSACTION_MD) - return false; - - SerialIter it(item->slice()); - it.getVL(); // skip transaction - hex = strHex(makeSlice(it.getVL())); - return true; -} - struct TxResult { Transaction::pointer txn; @@ -326,12 +294,10 @@ doTxJson(RPC::JsonContext& context) if (!context.params.isMember(jss::transaction)) return rpcError(rpcINVALID_PARAMS); - std::string txHash = context.params[jss::transaction].asString(); - if (!isHexTxID(txHash)) - return rpcError(rpcNOT_IMPL); - TxArgs args; - args.hash = from_hex_text(txHash); + + if (!args.hash.parseHex(context.params[jss::transaction].asString())) + return rpcError(rpcNOT_IMPL); args.binary = context.params.isMember(jss::binary) && context.params[jss::binary].asBool(); diff --git a/src/ripple/rpc/impl/GRPCHelpers.cpp b/src/ripple/rpc/impl/GRPCHelpers.cpp index f4a5ffbbcda..ed50fd2d441 100644 --- a/src/ripple/rpc/impl/GRPCHelpers.cpp +++ b/src/ripple/rpc/impl/GRPCHelpers.cpp @@ -56,7 +56,7 @@ void populateProtoVLasString( T const& getProto, STObject const& from, - SF_Blob const& field) + SF_VL const& field) { if (from.isFieldPresent(field)) { @@ -71,7 +71,7 @@ void populateProtoVec256( T const& getProto, STObject const& from, - SF_Vec256 const& field) + SF_VECTOR256 const& field) { if (from.isFieldPresent(field)) { @@ -89,7 +89,7 @@ void populateProtoAccount( T const& getProto, STObject const& from, - SF_Account const& field) + SF_ACCOUNT const& field) { if (from.isFieldPresent(field)) { @@ -103,7 +103,7 @@ void populateProtoAmount( T const& getProto, STObject const& from, - SF_Amount const& field) + SF_AMOUNT const& field) { if (from.isFieldPresent(field)) { @@ -117,7 +117,7 @@ void populateProtoCurrency( T const& getProto, STObject const& from, - SF_U160 const& field) + SF_HASH160 const& field) { if (from.isFieldPresent(field)) { @@ -364,7 +364,7 @@ void populateChannel(T& to, STObject const& from) { populateProtoPrimitive( - [&to]() { return to.mutable_channel(); }, from, sfPayChannel); + [&to]() { return to.mutable_channel(); }, from, sfChannel); } template diff --git a/src/ripple/rpc/impl/RPCHelpers.cpp b/src/ripple/rpc/impl/RPCHelpers.cpp index 6a415477a77..4aac4e6a53c 100644 --- a/src/ripple/rpc/impl/RPCHelpers.cpp +++ b/src/ripple/rpc/impl/RPCHelpers.cpp @@ -221,7 +221,7 @@ ledgerFromRequest(T& ledger, JsonContext& context) return {rpcINVALID_PARAMS, "ledgerHashNotString"}; uint256 ledgerHash; - if (!ledgerHash.SetHex(hashValue.asString())) + if (!ledgerHash.parseHex(hashValue.asString())) return {rpcINVALID_PARAMS, "ledgerHashMalformed"}; return getLedger(ledger, ledgerHash, context); } @@ -646,7 +646,7 @@ getSeedFromRPC(Json::Value const& params, Json::Value& error) { uint128 s; - if (s.SetHexExact(fieldContents)) + if (s.parseHex(fieldContents)) seed.emplace(Slice(s.data(), s.size())); } diff --git a/src/ripple/shamap/README.md b/src/ripple/shamap/README.md index a04a63c5f52..ef2d22024bd 100644 --- a/src/ripple/shamap/README.md +++ b/src/ripple/shamap/README.md @@ -17,14 +17,14 @@ or account state can be used to navigate the trie. A `SHAMap` is a trie with two node types: 1. SHAMapInnerNode -2. SHAMapTreeNode +2. SHAMapLeafNode -Both of these nodes directly inherit from SHAMapAbstractNode which holds data +Both of these nodes directly inherit from SHAMapTreeNode which holds data common to both of the node types. All non-leaf nodes have type SHAMapInnerNode. -All leaf nodes have type SHAMapTreeNode. +All leaf nodes have type SHAMapLeafNode. The root node is always a SHAMapInnerNode. @@ -184,16 +184,16 @@ by 4 bits, and then packed in sequence into a `uint256` (such that the longest path possible has 256 / 4 = 64 steps). The high 4 bits of the first byte identify which child of the root is chosen, the lower 4 bits of the first byte identify the child of that node, and so on. The `SHAMapNodeID` identifying the -root node has an ID of 0 and a depth of 0. See `SHAMapNodeID::selectBranch` for -details of how a `SHAMapNodeID` selects a "branch" (child) by indexing into its -path with its depth. +root node has an ID of 0 and a depth of 0. See `selectBranch` for details of +how we use a `SHAMapNodeID` to select a "branch" (child) by indexing into a +path at a given depth. While the current node is an inner node, traversing down the trie from the root continues, unless the path indicates a child that does not exist. And in this case, `nullptr` is returned to indicate no leaf node along the given path exists. Otherwise a leaf node is found and a (non-owning) pointer to it is returned. At each step, if a stack is requested, a -`pair, SHAMapNodeID>` is pushed onto the stack. +`pair, SHAMapNodeID>` is pushed onto the stack. When a child node is found by `selectBranch`, the traversal to that node consists of two steps: @@ -220,11 +220,11 @@ is this case stands for 'No Throw'. The `fetchNodeNT()` method goes through three phases: - 1. By calling `getCache()` we attempt to locate the missing node in the + 1. By calling `cacheLookup()` we attempt to locate the missing node in the TreeNodeCache. The TreeNodeCache is a cache of immutable SHAMapTreeNodes that are shared across all `SHAMap`s. - Any SHAMapTreeNode that is immutable has a sequence number of zero + Any SHAMapLeafNode that is immutable has a sequence number of zero (sharable). When a mutable `SHAMap` is created then its SHAMapTreeNodes are given non-zero sequence numbers (unsharable). But all nodes in the TreeNodeCache are immutable, so if one is found here, its sequence number @@ -250,17 +250,17 @@ the `SHAMap`, node `TreeNodeCache` or database, then we don't create duplicates by favoring the copy already in the `TreeNodeCache`. By using `canonicalize()` we manage a thread race condition where two different -threads might both recognize the lack of a SHAMapTreeNode at the same time +threads might both recognize the lack of a SHAMapLeafNode at the same time (during a fetch). If they both attempt to insert the node into the `SHAMap`, then `canonicalize` makes sure that the first node in wins and the slower thread receives back a pointer to the node inserted by the faster thread. Recall that these two `SHAMap`s will share the same `TreeNodeCache`. -## TreeNodeCache ## +## `TreeNodeCache` ## The `TreeNodeCache` is a `std::unordered_map` keyed on the hash of the -`SHAMap` node. The stored type consists of `shared_ptr`, -`weak_ptr`, and a time point indicating the most recent +`SHAMap` node. The stored type consists of `shared_ptr`, +`weak_ptr`, and a time point indicating the most recent access of this node in the cache. The time point is based on `std::chrono::steady_clock`. @@ -271,7 +271,7 @@ and logging, and a target age for the contained nodes. When the target age for a node is exceeded, and there are no more references to the node, the node is removed from the `TreeNodeCache`. -## FullBelowCache ## +## `FullBelowCache` ## This cache remembers which trie keys have all of their children resident in a `SHAMap`. This optimizes the process of acquiring a complete trie. This is used @@ -284,66 +284,56 @@ nodes, and thus that subtree does not need to be walked. These nodes are stored in the FullBelowCache. Subsequent walks check the FullBelowCache first when encountering a node, and ignore that subtree if found. -## SHAMapAbstractNode ## +## `SHAMapTreeNode` ## -This is a base class for the two concrete node types. It holds the following -common data: +This is an abstract base class for the concrete node types. It holds the +following common data: -1. A node type, one of: - a. error - b. inner - c. transaction with no metadata - d. transaction with metadata - e. account state -2. A hash -3. A sequence number +1. A hash +2. An identifier used to perform copy-on-write operations -## SHAMapInnerNode ## +### `SHAMapInnerNode` ### -SHAMapInnerNode publicly inherits directly from SHAMapAbstractNode. It holds +`SHAMapInnerNode` publicly inherits directly from `SHAMapTreeNode`. It holds the following data: 1. Up to 16 child nodes, each held with a shared_ptr. 2. A hash for each child. -3. A 16-bit bitset with a 1 bit set for each child that exists. -4. Flag to aid online delete and consistency with data on disk. +3. A bitset to indicate which of the 16 children exist. +4. An identifier used to determine whether the map below this node is + fully populated -## SHAMapTreeNode ## +### `SHAMapLeafNode` ### -SHAMapTreeNode publicly inherits directly from SHAMapAbstractNode. It holds the +`SHAMapLeafNode` is an abstract class which publicly inherits directly from +`SHAMapTreeNode`. It isIt holds the following data: 1. A shared_ptr to a const SHAMapItem. -## SHAMapItem ## +#### `SHAMapAccountStateLeafNode` #### -This holds the following data: +`SHAMapAccountStateLeafNode` is a class which publicly inherits directly from +`SHAMapLeafNode`. It is used to represent entries (i.e. account objects, escrow +objects, trust lines, etc.) in a state map. -1. uint256. The hash of the data. -2. vector. The data (transactions, account info). +#### `SHAMapTxLeafNode` #### -## SHAMap Improvements ## +`SHAMapTxLeafNode` is a class which publicly inherits directly from +`SHAMapLeafNode`. It is used to represent transactions in a state map. -Here's a simple one: the SHAMapTreeNode::mAccessSeq member is currently not used -and could be removed. +#### `SHAMapTxPlusMetaLeafNode` #### -Here's a more important change. The trie structure is currently embedded in the -SHAMapTreeNodes themselves. It doesn't have to be that way, and that should be -fixed. +`SHAMapTxPlusMetaLeafNode` is a class which publicly inherits directly from +`SHAMapLeafNode`. It is used to represent transactions along with metadata +associated with this transaction in a state map. -When we navigate the trie (say, like `SHAMap::walkTo()`) we currently ask each -node for information that we could determine locally. We know the depth because -we know how many nodes we have traversed. We know the ID that we need because -that's how we're steering. So we don't need to store the ID in the node. The -next refactor should remove all calls to `SHAMapTreeNode::GetID()`. +## SHAMapItem ## -Then we can remove the NodeID member from SHAMapTreeNode. +This holds the following data: -Then we can change the `SHAMap::mTNBtID` member to be `mTNByHash`. +1. uint256. The hash of the data. +2. vector. The data (transactions, account info). -An additional possible refactor would be to have a base type, SHAMapTreeNode, -and derive from that InnerNode and LeafNode types. That would remove some -storage (the array of 16 hashes) from the LeafNodes. That refactor would also -have the effect of simplifying methods like `isLeaf()` and `hasItem()`. diff --git a/src/ripple/shamap/SHAMap.h b/src/ripple/shamap/SHAMap.h index 86adbd05470..38466450eb9 100644 --- a/src/ripple/shamap/SHAMap.h +++ b/src/ripple/shamap/SHAMap.h @@ -27,33 +27,47 @@ #include #include #include +#include #include +#include #include -#include -#include #include #include -#include -#include -#include #include #include #include namespace ripple { +class SHAMapNodeID; +class SHAMapSyncFilter; + +/** Describes the current state of a given SHAMap */ enum class SHAMapState { - Modifying = 0, // Objects can be added and removed (like an open ledger) - Immutable = 1, // Map cannot be changed (like a closed ledger) - Synching = 2, // Map's hash is locked in, valid nodes can be added (like a - // peer's closing ledger) - Floating = 3, // Map is free to change hash (like a synching open ledger) - Invalid = - 4, // Map is known not to be valid (usually synching a corrupt ledger) -}; + /** The map is in flux and objects can be added and removed. + + Example: map underlying the open ledger. + */ + Modifying = 0, + + /** The map is set in stone and cannot be changed. + + Example: a map underlying a given closed ledger. + */ + Immutable = 1, + + /** The map's hash is fixed but valid nodes may be missing and can be added. -/** Function object which handles missing nodes. */ -using MissingNodeHandler = std::function; + Example: a map that's syncing a given peer's closing ledger. + */ + Synching = 2, + + /** The map is known to not be valid. + + Example: usually synching a corrupt ledger. + */ + Invalid = 3, +}; /** A SHAMap is both a radix tree with a fan-out of 16 and a Merkle tree. @@ -64,11 +78,11 @@ using MissingNodeHandler = std::function; 2. A node with only one child is merged with that child (the "merge property") - These properties in a significantly smaller memory footprint for a radix - tree. + These properties result in a significantly smaller memory footprint for + a radix tree. - And a fan-out of 16 means that each node in the tree has at most 16 - children. See https://en.wikipedia.org/wiki/Radix_tree + A fan-out of 16 means that each node in the tree has at most 16 + children. See https://en.wikipedia.org/wiki/Radix_tree A Merkle tree is a tree where each non-leaf node is labelled with the hash of the combined labels of its children nodes. @@ -83,21 +97,31 @@ class SHAMap private: Family& f_; beast::Journal journal_; - std::uint32_t seq_; - std::uint32_t ledgerSeq_ = 0; // sequence number of ledger this is part of - std::shared_ptr root_; + + /** ID to distinguish this map for all others we're sharing nodes with. */ + std::uint32_t cowid_ = 1; + + /** The sequence of the ledger that this map references, if any. */ + std::uint32_t ledgerSeq_ = 0; + + std::shared_ptr root_; mutable SHAMapState state_; - SHAMapType type_; - bool backed_ = true; // Map is backed by the database - bool full_ = false; // Map is believed complete in database + SHAMapType const type_; + bool backed_ = true; // Map is backed by the database + mutable bool full_ = false; // Map is believed complete in database public: + /** Each non-leaf node has 16 children (the 'radix tree' part of the map) */ + static inline constexpr unsigned int branchFactor = 16; + + /** The depth of the hash map: data is only present in the leaves */ + static inline constexpr unsigned int leafDepth = 64; + using DeltaItem = std::pair< std::shared_ptr, std::shared_ptr>; using Delta = std::map; - ~SHAMap(); SHAMap(SHAMap const&) = delete; SHAMap& operator=(SHAMap const&) = delete; @@ -107,6 +131,8 @@ class SHAMap SHAMap(SHAMapType t, uint256 const& hash, Family& f); + ~SHAMap() = default; + Family const& family() const { @@ -153,26 +179,26 @@ class SHAMap fetchRoot(SHAMapHash const& hash, SHAMapSyncFilter* filter); // normal hash access functions + + /** Does the tree have an item with the given ID? */ bool hasItem(uint256 const& id) const; + bool delItem(uint256 const& id); + bool - addItem(SHAMapItem&& i, bool isTransaction, bool hasMeta); + addItem(SHAMapNodeType type, SHAMapItem&& i); + SHAMapHash getHash() const; // save a copy if you have a temporary anyway bool - updateGiveItem( - std::shared_ptr, - bool isTransaction, - bool hasMeta); + updateGiveItem(SHAMapNodeType type, std::shared_ptr); + bool - addGiveItem( - std::shared_ptr, - bool isTransaction, - bool hasMeta); + addGiveItem(SHAMapNodeType type, std::shared_ptr item); // Save a copy if you need to extend the life // of the SHAMapItem beyond this SHAMap @@ -180,8 +206,6 @@ class SHAMap peekItem(uint256 const& id) const; std::shared_ptr const& peekItem(uint256 const& id, SHAMapHash& hash) const; - std::shared_ptr const& - peekItem(uint256 const& id, SHAMapTreeNode::TNType& type) const; // traverse functions const_iterator @@ -193,7 +217,7 @@ class SHAMap If function returns false, visitNodes exits. */ void - visitNodes(std::function const& function) const; + visitNodes(std::function const& function) const; /** Visit every node in this SHAMap that is not present in the specified SHAMap @@ -204,7 +228,7 @@ class SHAMap void visitDifferences( SHAMap const* have, - std::function) const; + std::function) const; /** Visit every leaf node in this SHAMap @@ -232,16 +256,16 @@ class SHAMap bool getNodeFat( - SHAMapNodeID node, + SHAMapNodeID const& wanted, std::vector& nodeIDs, - std::vector& rawNode, + std::vector& rawNodes, bool fatLeaves, std::uint32_t depth) const; - bool - getRootNode(Serializer& s, SHANodeFormat format) const; - std::vector - getNeededHashes(int max, SHAMapSyncFilter* filter); + /** Serializes the root in a format appropriate for sending over the wire */ + void + serializeRoot(Serializer& s) const; + SHAMapAddNode addRootNode( SHAMapHash const& hash, @@ -270,26 +294,21 @@ class SHAMap bool compare(SHAMap const& otherMap, Delta& differences, int maxCount) const; + /** Convert any modified nodes to shared. */ int - flushDirty(NodeObjectType t, std::uint32_t seq); + unshare(); + + /** Flush modified nodes to the nodestore and convert them to shared. */ + int + flushDirty(NodeObjectType t); + void walkMap(std::vector& missingNodes, int maxMissing) const; bool deepCompare(SHAMap& other) const; // Intended for debug/test only - using fetchPackEntry_t = std::pair; - - void - getFetchPack( - SHAMap const* have, - bool includeLeaves, - int max, - std::function) const; - void setUnbacked(); - int - unshare(); void dump(bool withHashes = false) const; @@ -297,29 +316,29 @@ class SHAMap invariants() const; private: - using SharedPtrNodeStack = std::stack< - std::pair, SHAMapNodeID>>; + using SharedPtrNodeStack = + std::stack, SHAMapNodeID>>; using DeltaRef = std::pair< std::shared_ptr const&, std::shared_ptr const&>; // tree node cache operations - std::shared_ptr - getCache(SHAMapHash const& hash) const; + std::shared_ptr + cacheLookup(SHAMapHash const& hash) const; void - canonicalize(SHAMapHash const& hash, std::shared_ptr&) + canonicalize(SHAMapHash const& hash, std::shared_ptr&) const; // database operations - std::shared_ptr + std::shared_ptr fetchNodeFromDB(SHAMapHash const& hash) const; - std::shared_ptr + std::shared_ptr fetchNodeNT(SHAMapHash const& hash) const; - std::shared_ptr + std::shared_ptr fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter* filter) const; - std::shared_ptr + std::shared_ptr fetchNode(SHAMapHash const& hash) const; - std::shared_ptr + std::shared_ptr checkFilter(SHAMapHash const& hash, SHAMapSyncFilter* filter) const; /** Update hashes up to the root */ @@ -327,16 +346,16 @@ class SHAMap dirtyUp( SharedPtrNodeStack& stack, uint256 const& target, - std::shared_ptr terminal); + std::shared_ptr terminal); /** Walk towards the specified id, returning the node. Caller must check if the return is nullptr, and if not, if the node->peekItem()->key() == id */ - SHAMapTreeNode* + SHAMapLeafNode* walkTowardsKey(uint256 const& id, SharedPtrNodeStack* stack = nullptr) const; /** Return nullptr if key not found */ - SHAMapTreeNode* + SHAMapLeafNode* findKey(uint256 const& id) const; /** Unshare the node, allowing it to be modified */ @@ -350,38 +369,35 @@ class SHAMap preFlushNode(std::shared_ptr node) const; /** write and canonicalize modified node */ - std::shared_ptr - writeNode( - NodeObjectType t, - std::uint32_t seq, - std::shared_ptr node) const; + std::shared_ptr + writeNode(NodeObjectType t, std::shared_ptr node) const; - SHAMapTreeNode* + SHAMapLeafNode* firstBelow( - std::shared_ptr, + std::shared_ptr, SharedPtrNodeStack& stack, int branch = 0) const; // Simple descent // Get a child of the specified node - SHAMapAbstractNode* + SHAMapTreeNode* descend(SHAMapInnerNode*, int branch) const; - SHAMapAbstractNode* + SHAMapTreeNode* descendThrow(SHAMapInnerNode*, int branch) const; - std::shared_ptr + std::shared_ptr descend(std::shared_ptr const&, int branch) const; - std::shared_ptr + std::shared_ptr descendThrow(std::shared_ptr const&, int branch) const; // Descend with filter - SHAMapAbstractNode* + SHAMapTreeNode* descendAsync( SHAMapInnerNode* parent, int branch, SHAMapSyncFilter* filter, bool& pending) const; - std::pair + std::pair descend( SHAMapInnerNode* parent, SHAMapNodeID const& parentID, @@ -390,31 +406,31 @@ class SHAMap // Non-storing // Does not hook the returned node to its parent - std::shared_ptr + std::shared_ptr descendNoStore(std::shared_ptr const&, int branch) const; /** If there is only one leaf below this node, get its contents */ std::shared_ptr const& - onlyBelow(SHAMapAbstractNode*) const; + onlyBelow(SHAMapTreeNode*) const; bool hasInnerNode(SHAMapNodeID const& nodeID, SHAMapHash const& hash) const; bool hasLeafNode(uint256 const& tag, SHAMapHash const& hash) const; - SHAMapTreeNode const* + SHAMapLeafNode const* peekFirstItem(SharedPtrNodeStack& stack) const; - SHAMapTreeNode const* + SHAMapLeafNode const* peekNextItem(uint256 const& id, SharedPtrNodeStack& stack) const; bool walkBranch( - SHAMapAbstractNode* node, + SHAMapTreeNode* node, std::shared_ptr const& otherMapItem, bool isFirstMap, Delta& differences, int& maxCount) const; int - walkSubTree(bool doWrite, NodeObjectType t, std::uint32_t seq); + walkSubTree(bool doWrite, NodeObjectType t); // Structure to track information about call to // getMissingNodes while it's in progress @@ -502,8 +518,7 @@ SHAMap::setImmutable() inline bool SHAMap::isSynching() const { - return (state_ == SHAMapState::Floating) || - (state_ == SHAMapState::Synching); + return state_ == SHAMapState::Synching; } inline void diff --git a/src/ripple/shamap/SHAMapAccountStateLeafNode.h b/src/ripple/shamap/SHAMapAccountStateLeafNode.h new file mode 100644 index 00000000000..1d1711d4941 --- /dev/null +++ b/src/ripple/shamap/SHAMapAccountStateLeafNode.h @@ -0,0 +1,93 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_SHAMAP_SHAMAPACCOUNTSTATELEAFNODE_H_INCLUDED +#define RIPPLE_SHAMAP_SHAMAPACCOUNTSTATELEAFNODE_H_INCLUDED + +#include +#include +#include +#include +#include +#include + +namespace ripple { + +/** A leaf node for a state object. */ +class SHAMapAccountStateLeafNode final + : public SHAMapLeafNode, + public CountedObject +{ +public: + SHAMapAccountStateLeafNode( + std::shared_ptr item, + std::uint32_t cowid) + : SHAMapLeafNode(std::move(item), cowid) + { + updateHash(); + } + + SHAMapAccountStateLeafNode( + std::shared_ptr item, + std::uint32_t cowid, + SHAMapHash const& hash) + : SHAMapLeafNode(std::move(item), cowid, hash) + { + } + + std::shared_ptr + clone(std::uint32_t cowid) const final override + { + return std::make_shared( + item_, cowid, hash_); + } + + SHAMapNodeType + getType() const final override + { + return SHAMapNodeType::tnACCOUNT_STATE; + } + + void + updateHash() final override + { + hash_ = SHAMapHash{sha512Half( + HashPrefix::leafNode, makeSlice(item_->peekData()), item_->key())}; + } + + void + serializeForWire(Serializer& s) const final override + { + s.addRaw(item_->peekData()); + s.addBitString(item_->key()); + s.add8(wireTypeAccountState); + } + + void + serializeWithPrefix(Serializer& s) const final override + { + s.add32(HashPrefix::leafNode); + s.addRaw(item_->peekData()); + s.addBitString(item_->key()); + } +}; + +} // namespace ripple + +#endif diff --git a/src/ripple/shamap/SHAMapInnerNode.h b/src/ripple/shamap/SHAMapInnerNode.h new file mode 100644 index 00000000000..d7788411364 --- /dev/null +++ b/src/ripple/shamap/SHAMapInnerNode.h @@ -0,0 +1,155 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_SHAMAP_SHAMAPINNERNODE_H_INCLUDED +#define RIPPLE_SHAMAP_SHAMAPINNERNODE_H_INCLUDED + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace ripple { + +class SHAMapInnerNode final : public SHAMapTreeNode, + public CountedObject +{ + std::array mHashes; + std::shared_ptr mChildren[16]; + int mIsBranch = 0; + std::uint32_t mFullBelowGen = 0; + + static std::mutex childLock; + +public: + SHAMapInnerNode(std::uint32_t cowid); + + std::shared_ptr + clone(std::uint32_t cowid) const override; + + SHAMapNodeType + getType() const override + { + return SHAMapNodeType::tnINNER; + } + + bool + isLeaf() const override + { + return false; + } + + bool + isInner() const override + { + return true; + } + + bool + isEmpty() const; + bool + isEmptyBranch(int m) const; + int + getBranchCount() const; + SHAMapHash const& + getChildHash(int m) const; + + void + setChild(int m, std::shared_ptr const& child); + void + shareChild(int m, std::shared_ptr const& child); + SHAMapTreeNode* + getChildPointer(int branch); + std::shared_ptr + getChild(int branch); + virtual std::shared_ptr + canonicalizeChild(int branch, std::shared_ptr node); + + // sync functions + bool + isFullBelow(std::uint32_t generation) const; + void + setFullBelowGen(std::uint32_t gen); + + void + updateHash() override; + + /** Recalculate the hash of all children and this node. */ + void + updateHashDeep(); + + void + serializeForWire(Serializer&) const override; + + void + serializeWithPrefix(Serializer&) const override; + + std::string + getString(SHAMapNodeID const&) const override; + + void + invariants(bool is_root = false) const override; + + static std::shared_ptr + makeFullInner(Slice data, SHAMapHash const& hash, bool hashValid); + + static std::shared_ptr + makeCompressedInner(Slice data); +}; + +inline SHAMapInnerNode::SHAMapInnerNode(std::uint32_t cowid) + : SHAMapTreeNode(cowid) +{ +} + +inline bool +SHAMapInnerNode::isEmptyBranch(int m) const +{ + return (mIsBranch & (1 << m)) == 0; +} + +inline SHAMapHash const& +SHAMapInnerNode::getChildHash(int m) const +{ + assert(m >= 0 && m < 16); + return mHashes[m]; +} + +inline bool +SHAMapInnerNode::isFullBelow(std::uint32_t generation) const +{ + return mFullBelowGen == generation; +} + +inline void +SHAMapInnerNode::setFullBelowGen(std::uint32_t gen) +{ + mFullBelowGen = gen; +} + +} // namespace ripple +#endif diff --git a/src/ripple/shamap/SHAMapItem.h b/src/ripple/shamap/SHAMapItem.h index 01561683e34..6e0664b0a43 100644 --- a/src/ripple/shamap/SHAMapItem.h +++ b/src/ripple/shamap/SHAMapItem.h @@ -21,6 +21,7 @@ #define RIPPLE_SHAMAP_SHAMAPITEM_H_INCLUDED #include +#include #include #include #include @@ -31,7 +32,7 @@ namespace ripple { // an item stored in a SHAMap -class SHAMapItem +class SHAMapItem : public CountedObject { private: uint256 tag_; diff --git a/src/ripple/shamap/SHAMapLeafNode.h b/src/ripple/shamap/SHAMapLeafNode.h new file mode 100644 index 00000000000..776aca76db8 --- /dev/null +++ b/src/ripple/shamap/SHAMapLeafNode.h @@ -0,0 +1,82 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_SHAMAP_SHAMAPLEAFNODE_H_INCLUDED +#define RIPPLE_SHAMAP_SHAMAPLEAFNODE_H_INCLUDED + +#include +#include +#include + +#include +#include + +namespace ripple { + +class SHAMapLeafNode : public SHAMapTreeNode +{ +protected: + std::shared_ptr item_; + + SHAMapLeafNode(std::shared_ptr item, std::uint32_t cowid); + SHAMapLeafNode( + std::shared_ptr item, + std::uint32_t cowid, + SHAMapHash const& hash); + +public: + SHAMapLeafNode(const SHAMapLeafNode&) = delete; + SHAMapLeafNode& + operator=(const SHAMapLeafNode&) = delete; + + bool + isLeaf() const final override + { + return true; + } + + bool + isInner() const final override + { + return false; + } + + void + invariants(bool is_root = false) const final override; + +public: + std::shared_ptr const& + peekItem() const; + + /** Set the item that this node points to and update the node's hash. + + @param i the new item + @return false if the change was, effectively, a noop (that is, if the + hash was unchanged); true otherwise. + */ + bool + setItem(std::shared_ptr i); + + std::string + getString(SHAMapNodeID const&) const final override; +}; + +} // namespace ripple + +#endif diff --git a/src/ripple/shamap/SHAMapNodeID.h b/src/ripple/shamap/SHAMapNodeID.h index a0041eececc..e89f63ce7df 100644 --- a/src/ripple/shamap/SHAMapNodeID.h +++ b/src/ripple/shamap/SHAMapNodeID.h @@ -20,160 +20,132 @@ #ifndef RIPPLE_SHAMAP_SHAMAPNODEID_H_INCLUDED #define RIPPLE_SHAMAP_SHAMAPNODEID_H_INCLUDED +#include #include -#include -#include +#include #include #include #include namespace ripple { -// Identifies a node in a half-SHA512 (256 bit) hash map -class SHAMapNodeID +/** Identifies a node inside a SHAMap */ +class SHAMapNodeID : public CountedObject { private: - uint256 mNodeID; - int mDepth; + uint256 id_; + unsigned int depth_ = 0; public: - SHAMapNodeID(); - SHAMapNodeID(int depth, uint256 const& hash); - SHAMapNodeID(void const* ptr, int len); + SHAMapNodeID() = default; + SHAMapNodeID(SHAMapNodeID const& other) = default; + SHAMapNodeID(unsigned int depth, uint256 const& hash); - bool - isValid() const; - bool - isRoot() const; - - // Convert to/from wire format (256-bit nodeID, 1-byte depth) - void - addIDRaw(Serializer& s) const; - std::string - getRawString() const; - - bool - operator==(const SHAMapNodeID& n) const; - bool - operator!=(const SHAMapNodeID& n) const; + SHAMapNodeID& + operator=(SHAMapNodeID const& other) = default; bool - operator<(const SHAMapNodeID& n) const; - bool - operator>(const SHAMapNodeID& n) const; - bool - operator<=(const SHAMapNodeID& n) const; - bool - operator>=(const SHAMapNodeID& n) const; + isRoot() const + { + return depth_ == 0; + } + // Get the wire format (256-bit nodeID, 1-byte depth) std::string - getString() const; - void - dump(beast::Journal journal) const; + getRawString() const; - // Only used by SHAMap and SHAMapTreeNode + unsigned int + getDepth() const + { + return depth_; + } uint256 const& - getNodeID() const; - SHAMapNodeID - getChildNodeID(int m) const; - int - selectBranch(uint256 const& hash) const; - int - getDepth() const; - bool - has_common_prefix(SHAMapNodeID const& other) const; + getNodeID() const + { + return id_; + } -private: - static uint256 const& - Masks(int depth); - - friend std::ostream& - operator<<(std::ostream& out, SHAMapNodeID const& node); - -private: // Currently unused SHAMapNodeID - getParentNodeID() const; -}; + getChildNodeID(unsigned int m) const; -//------------------------------------------------------------------------------ - -inline SHAMapNodeID::SHAMapNodeID() : mDepth(0) -{ -} - -inline int -SHAMapNodeID::getDepth() const -{ - return mDepth; -} - -inline uint256 const& -SHAMapNodeID::getNodeID() const -{ - return mNodeID; -} + // FIXME-C++20: use spaceship and operator synthesis + /** Comparison operators */ + bool + operator<(SHAMapNodeID const& n) const + { + return std::tie(depth_, id_) < std::tie(n.depth_, n.id_); + } -inline bool -SHAMapNodeID::isValid() const -{ - return (mDepth >= 0) && (mDepth <= 64); -} + bool + operator>(SHAMapNodeID const& n) const + { + return n < *this; + } -inline bool -SHAMapNodeID::isRoot() const -{ - return mDepth == 0; -} + bool + operator<=(SHAMapNodeID const& n) const + { + return !(n < *this); + } -inline SHAMapNodeID -SHAMapNodeID::getParentNodeID() const -{ - assert(mDepth); - return SHAMapNodeID(mDepth - 1, mNodeID & Masks(mDepth - 1)); -} + bool + operator>=(SHAMapNodeID const& n) const + { + return !(*this < n); + } -inline bool -SHAMapNodeID::operator<(const SHAMapNodeID& n) const -{ - return std::tie(mDepth, mNodeID) < std::tie(n.mDepth, n.mNodeID); -} + bool + operator==(SHAMapNodeID const& n) const + { + return (depth_ == n.depth_) && (id_ == n.id_); + } -inline bool -SHAMapNodeID::operator>(const SHAMapNodeID& n) const -{ - return n < *this; -} + bool + operator!=(SHAMapNodeID const& n) const + { + return !(*this == n); + } +}; -inline bool -SHAMapNodeID::operator<=(const SHAMapNodeID& n) const +inline std::string +to_string(SHAMapNodeID const& node) { - return !(n < *this); -} + if (node.isRoot()) + return "NodeID(root)"; -inline bool -SHAMapNodeID::operator>=(const SHAMapNodeID& n) const -{ - return !(*this < n); + return "NodeID(" + std::to_string(node.getDepth()) + "," + + to_string(node.getNodeID()) + ")"; } -inline bool -SHAMapNodeID::operator==(const SHAMapNodeID& n) const +inline std::ostream& +operator<<(std::ostream& out, SHAMapNodeID const& node) { - return (mDepth == n.mDepth) && (mNodeID == n.mNodeID); + return out << to_string(node); } -inline bool -SHAMapNodeID::operator!=(const SHAMapNodeID& n) const +/** Return an object representing a serialized SHAMap Node ID + * + * @param s A string of bytes + * @param data a non-null pointer to a buffer of @param size bytes. + * @param size the size, in bytes, of the buffer pointed to by @param data. + * @return A seated optional if the buffer contained a serialized SHAMap + * node ID and an unseated optional otherwise. + */ +/** @{ */ +[[nodiscard]] std::optional +deserializeSHAMapNodeID(void const* data, std::size_t size); + +[[nodiscard]] inline std::optional +deserializeSHAMapNodeID(std::string const& s) { - return !(*this == n); + return deserializeSHAMapNodeID(s.data(), s.size()); } +/** @} */ -inline std::ostream& -operator<<(std::ostream& out, SHAMapNodeID const& node) -{ - return out << node.getString(); -} +/** Returns the branch that would contain the given hash */ +[[nodiscard]] unsigned int +selectBranch(SHAMapNodeID const& id, uint256 const& hash); } // namespace ripple diff --git a/src/ripple/shamap/SHAMapSyncFilter.h b/src/ripple/shamap/SHAMapSyncFilter.h index 86a6451b61b..7ab649f8ad5 100644 --- a/src/ripple/shamap/SHAMapSyncFilter.h +++ b/src/ripple/shamap/SHAMapSyncFilter.h @@ -43,7 +43,7 @@ class SHAMapSyncFilter SHAMapHash const& nodeHash, std::uint32_t ledgerSeq, Blob&& nodeData, - SHAMapTreeNode::TNType type) const = 0; + SHAMapNodeType type) const = 0; virtual boost::optional getNode(SHAMapHash const& nodeHash) const = 0; diff --git a/src/ripple/shamap/SHAMapTreeNode.h b/src/ripple/shamap/SHAMapTreeNode.h index f453d7cd86f..7423444a921 100644 --- a/src/ripple/shamap/SHAMapTreeNode.h +++ b/src/ripple/shamap/SHAMapTreeNode.h @@ -20,6 +20,7 @@ #ifndef RIPPLE_SHAMAP_SHAMAPTREENODE_H_INCLUDED #define RIPPLE_SHAMAP_SHAMAPTREENODE_H_INCLUDED +#include #include #include #include @@ -32,14 +33,17 @@ namespace ripple { -enum SHANodeFormat { - snfPREFIX = 1, // Form that hashes to its official hash - snfWIRE = 2, // Compressed form used on the wire - snfHASH = 3, // just the hash -}; +// These are wire-protocol identifiers used during serialization to encode the +// type of a node. They should not be arbitrarily be changed. +static constexpr unsigned char const wireTypeTransaction = 0; +static constexpr unsigned char const wireTypeAccountState = 1; +static constexpr unsigned char const wireTypeInner = 2; +static constexpr unsigned char const wireTypeCompressedInner = 3; +static constexpr unsigned char const wireTypeTransactionWithMeta = 4; // A SHAMapHash is the hash of a node in a SHAMap, and also the // type of the hash of the entire SHAMap. + class SHAMapHash { uint256 hash_; @@ -119,311 +123,150 @@ operator!=(SHAMapHash const& x, SHAMapHash const& y) return !(x == y); } -class SHAMapAbstractNode -{ -public: - enum TNType { - tnERROR = 0, - tnINNER = 1, - tnTRANSACTION_NM = 2, // transaction, no metadata - tnTRANSACTION_MD = 3, // transaction, with metadata - tnACCOUNT_STATE = 4 - }; - -protected: - TNType mType; - SHAMapHash mHash; - std::uint32_t mSeq; - -protected: - virtual ~SHAMapAbstractNode() = 0; - SHAMapAbstractNode(SHAMapAbstractNode const&) = delete; - SHAMapAbstractNode& - operator=(SHAMapAbstractNode const&) = delete; - - SHAMapAbstractNode(TNType type, std::uint32_t seq); - SHAMapAbstractNode(TNType type, std::uint32_t seq, SHAMapHash const& hash); - -public: - std::uint32_t - getSeq() const; - void - setSeq(std::uint32_t s); - SHAMapHash const& - getNodeHash() const; - TNType - getType() const; - bool - isLeaf() const; - bool - isInner() const; - bool - isValid() const; - bool - isInBounds(SHAMapNodeID const& id) const; - - virtual bool - updateHash() = 0; - virtual void - addRaw(Serializer&, SHANodeFormat format) const = 0; - virtual std::string - getString(SHAMapNodeID const&) const; - virtual std::shared_ptr - clone(std::uint32_t seq) const = 0; - virtual uint256 const& - key() const = 0; - virtual void - invariants(bool is_root = false) const = 0; - - static std::shared_ptr - makeFromPrefix(Slice rawNode, SHAMapHash const& hash); - - static std::shared_ptr - makeFromWire(Slice rawNode); - -private: - static std::shared_ptr - makeTransaction( - Slice data, - std::uint32_t seq, - SHAMapHash const& hash, - bool hashValid); - - static std::shared_ptr - makeAccountState( - Slice data, - std::uint32_t seq, - SHAMapHash const& hash, - bool hashValid); - - static std::shared_ptr - makeTransactionWithMeta( - Slice data, - std::uint32_t seq, - SHAMapHash const& hash, - bool hashValid); +enum class SHAMapNodeType { + tnINNER = 1, + tnTRANSACTION_NM = 2, // transaction, no metadata + tnTRANSACTION_MD = 3, // transaction, with metadata + tnACCOUNT_STATE = 4 }; -class SHAMapInnerNode : public SHAMapAbstractNode +class SHAMapTreeNode { - std::array mHashes; - std::shared_ptr mChildren[16]; - int mIsBranch = 0; - std::uint32_t mFullBelowGen = 0; - - static std::mutex childLock; +protected: + SHAMapHash hash_; -public: - SHAMapInnerNode(std::uint32_t seq); - std::shared_ptr - clone(std::uint32_t seq) const override; + /** Determines the owning SHAMap, if any. Used for copy-on-write semantics. - bool - isEmpty() const; - bool - isEmptyBranch(int m) const; - int - getBranchCount() const; - SHAMapHash const& - getChildHash(int m) const; + If this value is 0, the node is not dirty and does not need to be + flushed. It is eligible for sharing and may be included multiple + SHAMap instances. + */ + std::uint32_t cowid_; - void - setChild(int m, std::shared_ptr const& child); - void - shareChild(int m, std::shared_ptr const& child); - SHAMapAbstractNode* - getChildPointer(int branch); - std::shared_ptr - getChild(int branch); - virtual std::shared_ptr - canonicalizeChild(int branch, std::shared_ptr node); - - // sync functions - bool - isFullBelow(std::uint32_t generation) const; - void - setFullBelowGen(std::uint32_t gen); - - bool - updateHash() override; - void - updateHashDeep(); - void - addRaw(Serializer&, SHANodeFormat format) const override; - std::string - getString(SHAMapNodeID const&) const override; - uint256 const& - key() const override; - void - invariants(bool is_root = false) const override; +protected: + SHAMapTreeNode(SHAMapTreeNode const&) = delete; + SHAMapTreeNode& + operator=(SHAMapTreeNode const&) = delete; - static std::shared_ptr - makeFullInner( - Slice data, - std::uint32_t seq, - SHAMapHash const& hash, - bool hashValid); + /** Construct a node - static std::shared_ptr - makeCompressedInner(Slice data, std::uint32_t seq); -}; + @param cowid The identifier of a SHAMap. For more, see #cowid_ + @param hash The hash associated with this node, if any. + */ + /** @{ */ + explicit SHAMapTreeNode(std::uint32_t cowid) noexcept : cowid_(cowid) + { + } -// SHAMapTreeNode represents a leaf, and may eventually be renamed to reflect -// that. -class SHAMapTreeNode : public SHAMapAbstractNode -{ -private: - std::shared_ptr mItem; + explicit SHAMapTreeNode( + std::uint32_t cowid, + SHAMapHash const& hash) noexcept + : hash_(hash), cowid_(cowid) + { + } + /** @} */ public: - SHAMapTreeNode(const SHAMapTreeNode&) = delete; - SHAMapTreeNode& - operator=(const SHAMapTreeNode&) = delete; - - SHAMapTreeNode( - std::shared_ptr item, - TNType type, - std::uint32_t seq); - SHAMapTreeNode( - std::shared_ptr item, - TNType type, - std::uint32_t seq, - SHAMapHash const& hash); - std::shared_ptr - clone(std::uint32_t seq) const override; + virtual ~SHAMapTreeNode() noexcept = default; - void - addRaw(Serializer&, SHANodeFormat format) const override; - uint256 const& - key() const override; - void - invariants(bool is_root = false) const override; + /** \defgroup SHAMap Copy-on-Write Support -public: // public only to SHAMap - // item node function - bool - hasItem() const; - std::shared_ptr const& - peekItem() const; - bool - setItem(std::shared_ptr i, TNType type); + By nature, a node may appear in multiple SHAMap instances. Rather than + actually duplicating these nodes, SHAMap opts to be memory efficient + and uses copy-on-write semantics for nodes. - std::string - getString(SHAMapNodeID const&) const override; - bool - updateHash() override; -}; + Only nodes that are not modified and don't need to be flushed back can + be shared. Once a node needs to be changed, it must first be copied and + the copy must marked as not shareable. -// SHAMapAbstractNode + Note that just because a node may not be *owned* by a given SHAMap + instance does not mean that the node is NOT a part of any SHAMap. It + only means that the node is not owned exclusively by any one SHAMap. -inline SHAMapAbstractNode::SHAMapAbstractNode(TNType type, std::uint32_t seq) - : mType(type), mSeq(seq) -{ -} + For more on copy-on-write, check out: + https://en.wikipedia.org/wiki/Copy-on-write + */ + /** @{ */ + /** Returns the SHAMap that owns this node. -inline SHAMapAbstractNode::SHAMapAbstractNode( - TNType type, - std::uint32_t seq, - SHAMapHash const& hash) - : mType(type), mHash(hash), mSeq(seq) -{ -} + @return the ID of the SHAMap that owns this node, or 0 if the node + is not owned by any SHAMap and is a candidate for sharing. + */ + std::uint32_t + cowid() const + { + return cowid_; + } -inline std::uint32_t -SHAMapAbstractNode::getSeq() const -{ - return mSeq; -} + /** If this node is shared with another map, mark it as no longer shared. -inline void -SHAMapAbstractNode::setSeq(std::uint32_t s) -{ - mSeq = s; -} + Only nodes that are not modified and do not need to be flushed back + should be marked as unshared. + */ + void + unshare() + { + cowid_ = 0; + } -inline SHAMapHash const& -SHAMapAbstractNode::getNodeHash() const -{ - return mHash; -} + /** Make a copy of this node, setting the owner. */ + virtual std::shared_ptr + clone(std::uint32_t cowid) const = 0; + /** @} */ -inline SHAMapAbstractNode::TNType -SHAMapAbstractNode::getType() const -{ - return mType; -} + /** Recalculate the hash of this node. */ + virtual void + updateHash() = 0; -inline bool -SHAMapAbstractNode::isLeaf() const -{ - return (mType == tnTRANSACTION_NM) || (mType == tnTRANSACTION_MD) || - (mType == tnACCOUNT_STATE); -} + /** Return the hash of this node. */ + SHAMapHash const& + getHash() const + { + return hash_; + } -inline bool -SHAMapAbstractNode::isInner() const -{ - return mType == tnINNER; -} + /** Determines the type of node. */ + virtual SHAMapNodeType + getType() const = 0; -inline bool -SHAMapAbstractNode::isValid() const -{ - return mType != tnERROR; -} + /** Determines if this is a leaf node. */ + virtual bool + isLeaf() const = 0; -inline bool -SHAMapAbstractNode::isInBounds(SHAMapNodeID const& id) const -{ - // Nodes at depth 64 must be leaves - return (!isInner() || (id.getDepth() < 64)); -} + /** Determines if this is an inner node. */ + virtual bool + isInner() const = 0; -// SHAMapInnerNode + /** Serialize the node in a format appropriate for sending over the wire */ + virtual void + serializeForWire(Serializer&) const = 0; -inline SHAMapInnerNode::SHAMapInnerNode(std::uint32_t seq) - : SHAMapAbstractNode(tnINNER, seq) -{ -} + /** Serialize the node in a format appropriate for hashing */ + virtual void + serializeWithPrefix(Serializer&) const = 0; -inline bool -SHAMapInnerNode::isEmptyBranch(int m) const -{ - return (mIsBranch & (1 << m)) == 0; -} + virtual std::string + getString(SHAMapNodeID const&) const; -inline SHAMapHash const& -SHAMapInnerNode::getChildHash(int m) const -{ - assert((m >= 0) && (m < 16) && (getType() == tnINNER)); - return mHashes[m]; -} + virtual void + invariants(bool is_root = false) const = 0; -inline bool -SHAMapInnerNode::isFullBelow(std::uint32_t generation) const -{ - return mFullBelowGen == generation; -} + static std::shared_ptr + makeFromPrefix(Slice rawNode, SHAMapHash const& hash); -inline void -SHAMapInnerNode::setFullBelowGen(std::uint32_t gen) -{ - mFullBelowGen = gen; -} + static std::shared_ptr + makeFromWire(Slice rawNode); -// SHAMapTreeNode +private: + static std::shared_ptr + makeTransaction(Slice data, SHAMapHash const& hash, bool hashValid); -inline bool -SHAMapTreeNode::hasItem() const -{ - return bool(mItem); -} + static std::shared_ptr + makeAccountState(Slice data, SHAMapHash const& hash, bool hashValid); -inline std::shared_ptr const& -SHAMapTreeNode::peekItem() const -{ - return mItem; -} + static std::shared_ptr + makeTransactionWithMeta(Slice data, SHAMapHash const& hash, bool hashValid); +}; } // namespace ripple diff --git a/src/ripple/shamap/SHAMapTxLeafNode.h b/src/ripple/shamap/SHAMapTxLeafNode.h new file mode 100644 index 00000000000..480318212db --- /dev/null +++ b/src/ripple/shamap/SHAMapTxLeafNode.h @@ -0,0 +1,89 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_SHAMAP_SHAMAPTXLEAFNODE_H_INCLUDED +#define RIPPLE_SHAMAP_SHAMAPTXLEAFNODE_H_INCLUDED + +#include +#include +#include +#include +#include +#include + +namespace ripple { + +/** A leaf node for a transaction. No metadata is included. */ +class SHAMapTxLeafNode final : public SHAMapLeafNode, + public CountedObject +{ +public: + SHAMapTxLeafNode( + std::shared_ptr item, + std::uint32_t cowid) + : SHAMapLeafNode(std::move(item), cowid) + { + updateHash(); + } + + SHAMapTxLeafNode( + std::shared_ptr item, + std::uint32_t cowid, + SHAMapHash const& hash) + : SHAMapLeafNode(std::move(item), cowid, hash) + { + } + + std::shared_ptr + clone(std::uint32_t cowid) const final override + { + return std::make_shared(item_, cowid, hash_); + } + + SHAMapNodeType + getType() const final override + { + return SHAMapNodeType::tnTRANSACTION_NM; + } + + void + updateHash() final override + { + hash_ = SHAMapHash{sha512Half( + HashPrefix::transactionID, makeSlice(item_->peekData()))}; + } + + void + serializeForWire(Serializer& s) const final override + { + s.addRaw(item_->peekData()); + s.add8(wireTypeTransaction); + } + + void + serializeWithPrefix(Serializer& s) const final override + { + s.add32(HashPrefix::transactionID); + s.addRaw(item_->peekData()); + } +}; + +} // namespace ripple + +#endif diff --git a/src/ripple/shamap/SHAMapTxPlusMetaLeafNode.h b/src/ripple/shamap/SHAMapTxPlusMetaLeafNode.h new file mode 100644 index 00000000000..ba919ce26f4 --- /dev/null +++ b/src/ripple/shamap/SHAMapTxPlusMetaLeafNode.h @@ -0,0 +1,92 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_SHAMAP_SHAMAPLEAFTXPLUSMETANODE_H_INCLUDED +#define RIPPLE_SHAMAP_SHAMAPLEAFTXPLUSMETANODE_H_INCLUDED + +#include +#include +#include +#include +#include +#include + +namespace ripple { + +/** A leaf node for a transaction and its associated metadata. */ +class SHAMapTxPlusMetaLeafNode final + : public SHAMapLeafNode, + public CountedObject +{ +public: + SHAMapTxPlusMetaLeafNode( + std::shared_ptr item, + std::uint32_t cowid) + : SHAMapLeafNode(std::move(item), cowid) + { + updateHash(); + } + + SHAMapTxPlusMetaLeafNode( + std::shared_ptr item, + std::uint32_t cowid, + SHAMapHash const& hash) + : SHAMapLeafNode(std::move(item), cowid, hash) + { + } + + std::shared_ptr + clone(std::uint32_t cowid) const override + { + return std::make_shared(item_, cowid, hash_); + } + + SHAMapNodeType + getType() const override + { + return SHAMapNodeType::tnTRANSACTION_MD; + } + + void + updateHash() final override + { + hash_ = SHAMapHash{sha512Half( + HashPrefix::txNode, makeSlice(item_->peekData()), item_->key())}; + } + + void + serializeForWire(Serializer& s) const final override + { + s.addRaw(item_->peekData()); + s.addBitString(item_->key()); + s.add8(wireTypeTransactionWithMeta); + } + + void + serializeWithPrefix(Serializer& s) const final override + { + s.add32(HashPrefix::txNode); + s.addRaw(item_->peekData()); + s.addBitString(item_->key()); + } +}; + +} // namespace ripple + +#endif diff --git a/src/ripple/shamap/TreeNodeCache.h b/src/ripple/shamap/TreeNodeCache.h index 9951db73dc2..f35c252f460 100644 --- a/src/ripple/shamap/TreeNodeCache.h +++ b/src/ripple/shamap/TreeNodeCache.h @@ -24,7 +24,7 @@ namespace ripple { -using TreeNodeCache = TaggedCache; +using TreeNodeCache = TaggedCache; } // namespace ripple diff --git a/src/ripple/shamap/impl/SHAMap.cpp b/src/ripple/shamap/impl/SHAMap.cpp index 19dff57c9c2..5e3b477b3d5 100644 --- a/src/ripple/shamap/impl/SHAMap.cpp +++ b/src/ripple/shamap/impl/SHAMap.cpp @@ -19,17 +19,41 @@ #include #include +#include +#include +#include +#include +#include namespace ripple { +[[nodiscard]] std::shared_ptr +makeTypedLeaf( + SHAMapNodeType type, + std::shared_ptr item, + std::uint32_t owner) +{ + if (type == SHAMapNodeType::tnTRANSACTION_NM) + return std::make_shared(std::move(item), owner); + + if (type == SHAMapNodeType::tnTRANSACTION_MD) + return std::make_shared( + std::move(item), owner); + + if (type == SHAMapNodeType::tnACCOUNT_STATE) + return std::make_shared( + std::move(item), owner); + + LogicError( + "Attempt to create leaf node of unknown type " + + std::to_string( + static_cast>(type))); +} + SHAMap::SHAMap(SHAMapType t, Family& f) - : f_(f) - , journal_(f.journal()) - , seq_(1) - , state_(SHAMapState::Modifying) - , type_(t) + : f_(f), journal_(f.journal()), state_(SHAMapState::Modifying), type_(t) { - root_ = std::make_shared(seq_); + root_ = std::make_shared(cowid_); } // The `hash` parameter is unused. It is part of the interface so it's clear @@ -37,18 +61,9 @@ SHAMap::SHAMap(SHAMapType t, Family& f) // known. The fact that the parameter is unused is an implementation detail that // should not change the interface. SHAMap::SHAMap(SHAMapType t, uint256 const& hash, Family& f) - : f_(f) - , journal_(f.journal()) - , seq_(1) - , state_(SHAMapState::Synching) - , type_(t) + : f_(f), journal_(f.journal()), state_(SHAMapState::Synching), type_(t) { - root_ = std::make_shared(seq_); -} - -SHAMap::~SHAMap() -{ - state_ = SHAMapState::Invalid; + root_ = std::make_shared(cowid_); } std::shared_ptr @@ -60,7 +75,7 @@ SHAMap::snapShot(bool isMutable) const if (!isMutable) newMap.state_ = SHAMapState::Immutable; - newMap.seq_ = seq_ + 1; + newMap.cowid_ = cowid_ + 1; newMap.ledgerSeq_ = ledgerSeq_; newMap.root_ = root_; newMap.backed_ = backed_; @@ -79,7 +94,7 @@ void SHAMap::dirtyUp( SharedPtrNodeStack& stack, uint256 const& target, - std::shared_ptr child) + std::shared_ptr child) { // walk the tree up from through the inner nodes to the root_ // update hashes and links @@ -89,7 +104,7 @@ SHAMap::dirtyUp( assert( (state_ != SHAMapState::Synching) && (state_ != SHAMapState::Immutable)); - assert(child && (child->getSeq() == seq_)); + assert(child && (child->cowid() == cowid_)); while (!stack.empty()) { @@ -99,7 +114,7 @@ SHAMap::dirtyUp( stack.pop(); assert(node != nullptr); - int branch = nodeID.selectBranch(target); + int branch = selectBranch(nodeID, target); assert(branch >= 0); node = unshareNode(std::move(node), nodeID); @@ -109,7 +124,7 @@ SHAMap::dirtyUp( } } -SHAMapTreeNode* +SHAMapLeafNode* SHAMap::walkTowardsKey(uint256 const& id, SharedPtrNodeStack* stack) const { assert(stack == nullptr || stack->empty()); @@ -122,7 +137,7 @@ SHAMap::walkTowardsKey(uint256 const& id, SharedPtrNodeStack* stack) const stack->push({inNode, nodeID}); auto const inner = std::static_pointer_cast(inNode); - auto const branch = nodeID.selectBranch(id); + auto const branch = selectBranch(nodeID, id); if (inner->isEmptyBranch(branch)) return nullptr; @@ -132,22 +147,22 @@ SHAMap::walkTowardsKey(uint256 const& id, SharedPtrNodeStack* stack) const if (stack != nullptr) stack->push({inNode, nodeID}); - return static_cast(inNode.get()); + return static_cast(inNode.get()); } -SHAMapTreeNode* +SHAMapLeafNode* SHAMap::findKey(uint256 const& id) const { - SHAMapTreeNode* leaf = walkTowardsKey(id); + SHAMapLeafNode* leaf = walkTowardsKey(id); if (leaf && leaf->peekItem()->key() != id) leaf = nullptr; return leaf; } -std::shared_ptr +std::shared_ptr SHAMap::fetchNodeFromDB(SHAMapHash const& hash) const { - std::shared_ptr node; + std::shared_ptr node; if (backed_) { @@ -156,7 +171,7 @@ SHAMap::fetchNodeFromDB(SHAMapHash const& hash) const { try { - node = SHAMapAbstractNode::makeFromPrefix( + node = SHAMapTreeNode::makeFromPrefix( makeSlice(nodeObject->getData()), hash); if (node) canonicalize(hash, node); @@ -164,13 +179,13 @@ SHAMap::fetchNodeFromDB(SHAMapHash const& hash) const catch (std::exception const&) { JLOG(journal_.warn()) << "Invalid DB node " << hash; - return std::shared_ptr(); + return std::shared_ptr(); } } else if (full_) { + full_ = false; f_.missingNode(ledgerSeq_); - const_cast(full_) = false; } } @@ -178,7 +193,7 @@ SHAMap::fetchNodeFromDB(SHAMapHash const& hash) const } // See if a sync filter has a node -std::shared_ptr +std::shared_ptr SHAMap::checkFilter(SHAMapHash const& hash, SHAMapSyncFilter* filter) const { if (auto nodeData = filter->getNode(hash)) @@ -186,7 +201,7 @@ SHAMap::checkFilter(SHAMapHash const& hash, SHAMapSyncFilter* filter) const try { auto node = - SHAMapAbstractNode::makeFromPrefix(makeSlice(*nodeData), hash); + SHAMapTreeNode::makeFromPrefix(makeSlice(*nodeData), hash); if (node) { filter->gotNode( @@ -211,10 +226,10 @@ SHAMap::checkFilter(SHAMapHash const& hash, SHAMapSyncFilter* filter) const // Get a node without throwing // Used on maps where missing nodes are expected -std::shared_ptr +std::shared_ptr SHAMap::fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter* filter) const { - std::shared_ptr node = getCache(hash); + auto node = cacheLookup(hash); if (node) return node; @@ -234,10 +249,10 @@ SHAMap::fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter* filter) const return node; } -std::shared_ptr +std::shared_ptr SHAMap::fetchNodeNT(SHAMapHash const& hash) const { - auto node = getCache(hash); + auto node = cacheLookup(hash); if (!node && backed_) node = fetchNodeFromDB(hash); @@ -246,7 +261,7 @@ SHAMap::fetchNodeNT(SHAMapHash const& hash) const } // Throw if the node is missing -std::shared_ptr +std::shared_ptr SHAMap::fetchNode(SHAMapHash const& hash) const { auto node = fetchNodeNT(hash); @@ -257,10 +272,10 @@ SHAMap::fetchNode(SHAMapHash const& hash) const return node; } -SHAMapAbstractNode* +SHAMapTreeNode* SHAMap::descendThrow(SHAMapInnerNode* parent, int branch) const { - SHAMapAbstractNode* ret = descend(parent, branch); + SHAMapTreeNode* ret = descend(parent, branch); if (!ret && !parent->isEmptyBranch(branch)) Throw(type_, parent->getChildHash(branch)); @@ -268,11 +283,11 @@ SHAMap::descendThrow(SHAMapInnerNode* parent, int branch) const return ret; } -std::shared_ptr +std::shared_ptr SHAMap::descendThrow(std::shared_ptr const& parent, int branch) const { - std::shared_ptr ret = descend(parent, branch); + std::shared_ptr ret = descend(parent, branch); if (!ret && !parent->isEmptyBranch(branch)) Throw(type_, parent->getChildHash(branch)); @@ -280,14 +295,14 @@ SHAMap::descendThrow(std::shared_ptr const& parent, int branch) return ret; } -SHAMapAbstractNode* +SHAMapTreeNode* SHAMap::descend(SHAMapInnerNode* parent, int branch) const { - SHAMapAbstractNode* ret = parent->getChildPointer(branch); + SHAMapTreeNode* ret = parent->getChildPointer(branch); if (ret || !backed_) return ret; - std::shared_ptr node = + std::shared_ptr node = fetchNodeNT(parent->getChildHash(branch)); if (!node) return nullptr; @@ -296,11 +311,11 @@ SHAMap::descend(SHAMapInnerNode* parent, int branch) const return node.get(); } -std::shared_ptr +std::shared_ptr SHAMap::descend(std::shared_ptr const& parent, int branch) const { - std::shared_ptr node = parent->getChild(branch); + std::shared_ptr node = parent->getChild(branch); if (node || !backed_) return node; @@ -314,18 +329,18 @@ SHAMap::descend(std::shared_ptr const& parent, int branch) // Gets the node that would be hooked to this branch, // but doesn't hook it up. -std::shared_ptr +std::shared_ptr SHAMap::descendNoStore( std::shared_ptr const& parent, int branch) const { - std::shared_ptr ret = parent->getChild(branch); + std::shared_ptr ret = parent->getChild(branch); if (!ret && backed_) ret = fetchNode(parent->getChildHash(branch)); return ret; } -std::pair +std::pair SHAMap::descend( SHAMapInnerNode* parent, SHAMapNodeID const& parentID, @@ -333,15 +348,15 @@ SHAMap::descend( SHAMapSyncFilter* filter) const { assert(parent->isInner()); - assert((branch >= 0) && (branch < 16)); + assert((branch >= 0) && (branch < branchFactor)); assert(!parent->isEmptyBranch(branch)); - SHAMapAbstractNode* child = parent->getChildPointer(branch); + SHAMapTreeNode* child = parent->getChildPointer(branch); if (!child) { auto const& childHash = parent->getChildHash(branch); - std::shared_ptr childNode = + std::shared_ptr childNode = fetchNodeNT(childHash, filter); if (childNode) @@ -354,7 +369,7 @@ SHAMap::descend( return std::make_pair(child, parentID.getChildNodeID(branch)); } -SHAMapAbstractNode* +SHAMapTreeNode* SHAMap::descendAsync( SHAMapInnerNode* parent, int branch, @@ -363,13 +378,13 @@ SHAMap::descendAsync( { pending = false; - SHAMapAbstractNode* ret = parent->getChildPointer(branch); + SHAMapTreeNode* ret = parent->getChildPointer(branch); if (ret) return ret; auto const& hash = parent->getChildHash(branch); - std::shared_ptr ptr = getCache(hash); + auto ptr = cacheLookup(hash); if (!ptr) { if (filter) @@ -386,8 +401,8 @@ SHAMap::descendAsync( if (!obj) return nullptr; - ptr = SHAMapAbstractNode::makeFromPrefix( - makeSlice(obj->getData()), hash); + ptr = + SHAMapTreeNode::makeFromPrefix(makeSlice(obj->getData()), hash); if (ptr && backed_) canonicalize(hash, ptr); } @@ -404,31 +419,29 @@ std::shared_ptr SHAMap::unshareNode(std::shared_ptr node, SHAMapNodeID const& nodeID) { // make sure the node is suitable for the intended operation (copy on write) - assert(node->isValid()); - assert(node->getSeq() <= seq_); - if (node->getSeq() != seq_) + assert(node->cowid() <= cowid_); + if (node->cowid() != cowid_) { // have a CoW assert(state_ != SHAMapState::Immutable); - node = std::static_pointer_cast(node->clone(seq_)); - assert(node->isValid()); + node = std::static_pointer_cast(node->clone(cowid_)); if (nodeID.isRoot()) root_ = node; } return node; } -SHAMapTreeNode* +SHAMapLeafNode* SHAMap::firstBelow( - std::shared_ptr node, + std::shared_ptr node, SharedPtrNodeStack& stack, int branch) const { // Return the first item at or below this node if (node->isLeaf()) { - auto n = std::static_pointer_cast(node); - stack.push({node, {64, n->peekItem()->key()}}); + auto n = std::static_pointer_cast(node); + stack.push({node, {leafDepth, n->peekItem()->key()}}); return n.get(); } auto inner = std::static_pointer_cast(node); @@ -436,7 +449,7 @@ SHAMap::firstBelow( stack.push({inner, SHAMapNodeID{}}); else stack.push({inner, stack.top().second.getChildNodeID(branch)}); - for (int i = 0; i < 16;) + for (int i = 0; i < branchFactor;) { if (!inner->isEmptyBranch(i)) { @@ -444,8 +457,8 @@ SHAMap::firstBelow( assert(!stack.empty()); if (node->isLeaf()) { - auto n = std::static_pointer_cast(node); - stack.push({n, {64, n->peekItem()->key()}}); + auto n = std::static_pointer_cast(node); + stack.push({n, {leafDepth, n->peekItem()->key()}}); return n.get(); } inner = std::static_pointer_cast(node); @@ -461,15 +474,15 @@ SHAMap::firstBelow( static const std::shared_ptr no_item; std::shared_ptr const& -SHAMap::onlyBelow(SHAMapAbstractNode* node) const +SHAMap::onlyBelow(SHAMapTreeNode* node) const { // If there is only one item below this node, return it while (!node->isLeaf()) { - SHAMapAbstractNode* nextNode = nullptr; + SHAMapTreeNode* nextNode = nullptr; auto inner = static_cast(node); - for (int i = 0; i < 16; ++i) + for (int i = 0; i < branchFactor; ++i) { if (!inner->isEmptyBranch(i)) { @@ -491,17 +504,16 @@ SHAMap::onlyBelow(SHAMapAbstractNode* node) const // An inner node must have at least one leaf // below it, unless it's the root_ - auto leaf = static_cast(node); - assert(leaf->hasItem() || (leaf == root_.get())); - + auto const leaf = static_cast(node); + assert(leaf->peekItem() || (leaf == root_.get())); return leaf->peekItem(); } -SHAMapTreeNode const* +SHAMapLeafNode const* SHAMap::peekFirstItem(SharedPtrNodeStack& stack) const { assert(stack.empty()); - SHAMapTreeNode* node = firstBelow(root_, stack); + SHAMapLeafNode* node = firstBelow(root_, stack); if (!node) { while (!stack.empty()) @@ -511,7 +523,7 @@ SHAMap::peekFirstItem(SharedPtrNodeStack& stack) const return node; } -SHAMapTreeNode const* +SHAMapLeafNode const* SHAMap::peekNextItem(uint256 const& id, SharedPtrNodeStack& stack) const { assert(!stack.empty()); @@ -522,7 +534,7 @@ SHAMap::peekNextItem(uint256 const& id, SharedPtrNodeStack& stack) const auto [node, nodeID] = stack.top(); assert(!node->isLeaf()); auto inner = std::static_pointer_cast(node); - for (auto i = nodeID.selectBranch(id) + 1; i < 16; ++i) + for (auto i = selectBranch(nodeID, id) + 1; i < branchFactor; ++i) { if (!inner->isEmptyBranch(i)) { @@ -543,7 +555,7 @@ SHAMap::peekNextItem(uint256 const& id, SharedPtrNodeStack& stack) const std::shared_ptr const& SHAMap::peekItem(uint256 const& id) const { - SHAMapTreeNode* leaf = findKey(id); + SHAMapLeafNode* leaf = findKey(id); if (!leaf) return no_item; @@ -551,27 +563,15 @@ SHAMap::peekItem(uint256 const& id) const return leaf->peekItem(); } -std::shared_ptr const& -SHAMap::peekItem(uint256 const& id, SHAMapTreeNode::TNType& type) const -{ - SHAMapTreeNode* leaf = findKey(id); - - if (!leaf) - return no_item; - - type = leaf->getType(); - return leaf->peekItem(); -} - std::shared_ptr const& SHAMap::peekItem(uint256 const& id, SHAMapHash& hash) const { - SHAMapTreeNode* leaf = findKey(id); + SHAMapLeafNode* leaf = findKey(id); if (!leaf) return no_item; - hash = leaf->getNodeHash(); + hash = leaf->getHash(); return leaf->peekItem(); } @@ -587,7 +587,7 @@ SHAMap::upper_bound(uint256 const& id) const auto [node, nodeID] = stack.top(); if (node->isLeaf()) { - auto leaf = static_cast(node.get()); + auto leaf = static_cast(node.get()); if (leaf->peekItem()->key() > id) return const_iterator( this, leaf->peekItem().get(), std::move(stack)); @@ -595,7 +595,8 @@ SHAMap::upper_bound(uint256 const& id) const else { auto inner = std::static_pointer_cast(node); - for (auto branch = nodeID.selectBranch(id) + 1; branch < 16; + for (auto branch = selectBranch(nodeID, id) + 1; + branch < branchFactor; ++branch) { if (!inner->isEmptyBranch(branch)) @@ -617,9 +618,7 @@ SHAMap::upper_bound(uint256 const& id) const bool SHAMap::hasItem(uint256 const& id) const { - // does the tree have an item with this ID - SHAMapTreeNode* leaf = findKey(id); - return (leaf != nullptr); + return (findKey(id) != nullptr); } bool @@ -634,17 +633,17 @@ SHAMap::delItem(uint256 const& id) if (stack.empty()) Throw(type_, id); - auto leaf = std::dynamic_pointer_cast(stack.top().first); + auto leaf = std::dynamic_pointer_cast(stack.top().first); stack.pop(); if (!leaf || (leaf->peekItem()->key() != id)) return false; - SHAMapTreeNode::TNType type = leaf->getType(); + SHAMapNodeType type = leaf->getType(); // What gets attached to the end of the chain // (For now, nothing, since we deleted the leaf) - std::shared_ptr prevNode; + std::shared_ptr prevNode; while (!stack.empty()) { @@ -654,7 +653,7 @@ SHAMap::delItem(uint256 const& id) stack.pop(); node = unshareNode(std::move(node), nodeID); - node->setChild(nodeID.selectBranch(id), prevNode); + node->setChild(selectBranch(nodeID, id), prevNode); if (!nodeID.isRoot()) { @@ -673,7 +672,7 @@ SHAMap::delItem(uint256 const& id) if (item) { - for (int i = 0; i < 16; ++i) + for (int i = 0; i < branchFactor; ++i) { if (!node->isEmptyBranch(i)) { @@ -681,8 +680,8 @@ SHAMap::delItem(uint256 const& id) break; } } - prevNode = std::make_shared( - item, type, node->getSeq()); + + prevNode = makeTypedLeaf(type, item, node->cowid()); } else { @@ -701,19 +700,13 @@ SHAMap::delItem(uint256 const& id) } bool -SHAMap::addGiveItem( - std::shared_ptr item, - bool isTransaction, - bool hasMeta) +SHAMap::addGiveItem(SHAMapNodeType type, std::shared_ptr item) { + assert(state_ != SHAMapState::Immutable); + assert(type != SHAMapNodeType::tnINNER); + // add the specified item, does not update uint256 tag = item->key(); - SHAMapTreeNode::TNType type = !isTransaction - ? SHAMapTreeNode::tnACCOUNT_STATE - : (hasMeta ? SHAMapTreeNode::tnTRANSACTION_MD - : SHAMapTreeNode::tnTRANSACTION_NM); - - assert(state_ != SHAMapState::Immutable); SharedPtrNodeStack stack; walkTowardsKey(tag, &stack); @@ -726,7 +719,7 @@ SHAMap::addGiveItem( if (node->isLeaf()) { - auto leaf = std::static_pointer_cast(node); + auto leaf = std::static_pointer_cast(node); if (leaf->peekItem()->key() == tag) return false; } @@ -735,48 +728,40 @@ SHAMap::addGiveItem( { // easy case, we end on an inner node auto inner = std::static_pointer_cast(node); - int branch = nodeID.selectBranch(tag); + int branch = selectBranch(nodeID, tag); assert(inner->isEmptyBranch(branch)); - auto newNode = - std::make_shared(std::move(item), type, seq_); + auto newNode = makeTypedLeaf(type, std::move(item), cowid_); inner->setChild(branch, newNode); } else { // this is a leaf node that has to be made an inner node holding two // items - auto leaf = std::static_pointer_cast(node); + auto leaf = std::static_pointer_cast(node); std::shared_ptr otherItem = leaf->peekItem(); assert(otherItem && (tag != otherItem->key())); - node = std::make_shared(node->getSeq()); + node = std::make_shared(node->cowid()); - int b1, b2; + unsigned int b1, b2; - while ((b1 = nodeID.selectBranch(tag)) == - (b2 = nodeID.selectBranch(otherItem->key()))) + while ((b1 = selectBranch(nodeID, tag)) == + (b2 = selectBranch(nodeID, otherItem->key()))) { stack.push({node, nodeID}); // we need a new inner node, since both go on same branch at this // level nodeID = nodeID.getChildNodeID(b1); - node = std::make_shared(seq_); + node = std::make_shared(cowid_); } // we can add the two leaf nodes here assert(node->isInner()); - std::shared_ptr newNode = - std::make_shared(std::move(item), type, seq_); - assert(newNode->isValid() && newNode->isLeaf()); - auto inner = std::static_pointer_cast(node); - inner->setChild(b1, newNode); - - newNode = - std::make_shared(std::move(otherItem), type, seq_); - assert(newNode->isValid() && newNode->isLeaf()); - inner->setChild(b2, newNode); + auto inner = static_cast(node.get()); + inner->setChild(b1, makeTypedLeaf(type, std::move(item), cowid_)); + inner->setChild(b2, makeTypedLeaf(type, std::move(otherItem), cowid_)); } dirtyUp(stack, tag, node); @@ -784,31 +769,27 @@ SHAMap::addGiveItem( } bool -SHAMap::addItem(SHAMapItem&& i, bool isTransaction, bool hasMetaData) +SHAMap::addItem(SHAMapNodeType type, SHAMapItem&& i) { - return addGiveItem( - std::make_shared(std::move(i)), - isTransaction, - hasMetaData); + return addGiveItem(type, std::make_shared(std::move(i))); } SHAMapHash SHAMap::getHash() const { - auto hash = root_->getNodeHash(); + auto hash = root_->getHash(); if (hash.isZero()) { const_cast(*this).unshare(); - hash = root_->getNodeHash(); + hash = root_->getHash(); } return hash; } bool SHAMap::updateGiveItem( - std::shared_ptr item, - bool isTransaction, - bool hasMeta) + SHAMapNodeType type, + std::shared_ptr item) { // can't change the tag but can change the hash uint256 tag = item->key(); @@ -821,7 +802,7 @@ SHAMap::updateGiveItem( if (stack.empty()) Throw(type_, tag); - auto node = std::dynamic_pointer_cast(stack.top().first); + auto node = std::dynamic_pointer_cast(stack.top().first); auto nodeID = stack.top().second; stack.pop(); @@ -831,26 +812,24 @@ SHAMap::updateGiveItem( return false; } - node = unshareNode(std::move(node), nodeID); - - if (!node->setItem( - std::move(item), - !isTransaction ? SHAMapTreeNode::tnACCOUNT_STATE - : (hasMeta ? SHAMapTreeNode::tnTRANSACTION_MD - : SHAMapTreeNode::tnTRANSACTION_NM))) + if (node->getType() != type) { - JLOG(journal_.trace()) << "SHAMap setItem, no change"; - return true; + JLOG(journal_.fatal()) << "SHAMap::setItem: cross-type change!"; + return false; } - dirtyUp(stack, tag, node); + node = unshareNode(std::move(node), nodeID); + + if (node->setItem(std::move(item))) + dirtyUp(stack, tag, node); + return true; } bool SHAMap::fetchRoot(SHAMapHash const& hash, SHAMapSyncFilter* filter) { - if (hash == root_->getNodeHash()) + if (hash == root_->getHash()) return true; if (auto stream = journal_.trace()) @@ -874,42 +853,37 @@ SHAMap::fetchRoot(SHAMapHash const& hash, SHAMapSyncFilter* filter) if (newRoot) { root_ = newRoot; - assert(root_->getNodeHash() == hash); + assert(root_->getHash() == hash); return true; } return false; } -// Replace a node with a shareable node. -// -// This code handles two cases: -// -// 1) An unshared, unshareable node needs to be made shareable -// so immutable SHAMap's can have references to it. -// -// 2) An unshareable node is shared. This happens when you make -// a mutable snapshot of a mutable SHAMap. -std::shared_ptr -SHAMap::writeNode( - NodeObjectType t, - std::uint32_t seq, - std::shared_ptr node) const +/** Replace a node with a shareable node. + + This code handles two cases: + + 1) An unshared, unshareable node needs to be made shareable + so immutable SHAMap's can have references to it. + 2) An unshareable node is shared. This happens when you make + a mutable snapshot of a mutable SHAMap. + + @note The node must have already been unshared by having the caller + first call SHAMapTreeNode::unshare(). + */ +std::shared_ptr +SHAMap::writeNode(NodeObjectType t, std::shared_ptr node) const { - // Node is ours, so we can just make it shareable - assert(node->getSeq() == seq_); + assert(node->cowid() == 0); assert(backed_); - node->setSeq(0); - canonicalize(node->getNodeHash(), node); + canonicalize(node->getHash(), node); Serializer s; - node->addRaw(s, snfPREFIX); + node->serializeWithPrefix(s); f_.db().store( - t, - std::move(s.modData()), - node->getNodeHash().as_uint256(), - ledgerSeq_); + t, std::move(s.modData()), node->getHash().as_uint256(), ledgerSeq_); return node; } @@ -922,13 +896,13 @@ SHAMap::preFlushNode(std::shared_ptr node) const { // A shared node should never need to be flushed // because that would imply someone modified it - assert(node->getSeq() != 0); + assert(node->cowid() != 0); - if (node->getSeq() != seq_) + if (node->cowid() != cowid_) { // Node is not uniquely ours, so unshare it before // possibly modifying it - node = std::static_pointer_cast(node->clone(seq_)); + node = std::static_pointer_cast(node->clone(cowid_)); } return node; } @@ -936,35 +910,36 @@ SHAMap::preFlushNode(std::shared_ptr node) const int SHAMap::unshare() { - // Don't share nodes wth parent map - return walkSubTree(false, hotUNKNOWN, 0); + // Don't share nodes with parent map + return walkSubTree(false, hotUNKNOWN); } -/** Convert all modified nodes to shared nodes */ -// If requested, write them to the node store int -SHAMap::flushDirty(NodeObjectType t, std::uint32_t seq) +SHAMap::flushDirty(NodeObjectType t) { - return walkSubTree(true, t, seq); + // We only write back if this map is backed. + return walkSubTree(backed_, t); } int -SHAMap::walkSubTree(bool doWrite, NodeObjectType t, std::uint32_t seq) +SHAMap::walkSubTree(bool doWrite, NodeObjectType t) { + assert(!doWrite || backed_); + int flushed = 0; - Serializer s; - if (!root_ || (root_->getSeq() == 0)) + if (!root_ || (root_->cowid() == 0)) return flushed; if (root_->isLeaf()) { // special case -- root_ is leaf root_ = preFlushNode(std::move(root_)); root_->updateHash(); - if (doWrite && backed_) - root_ = writeNode(t, seq, std::move(root_)); - else - root_->setSeq(0); + root_->unshare(); + + if (doWrite) + root_ = writeNode(t, std::move(root_)); + return 1; } @@ -988,7 +963,7 @@ SHAMap::walkSubTree(bool doWrite, NodeObjectType t, std::uint32_t seq) // We can't flush an inner node until we flush its children while (1) { - while (pos < 16) + while (pos < branchFactor) { if (node->isEmptyBranch(pos)) { @@ -1001,7 +976,7 @@ SHAMap::walkSubTree(bool doWrite, NodeObjectType t, std::uint32_t seq) int branch = pos; auto child = node->getChild(pos++); - if (child && (child->getSeq() != 0)) + if (child && (child->cowid() != 0)) { // This is a node that needs to be flushed @@ -1024,13 +999,12 @@ SHAMap::walkSubTree(bool doWrite, NodeObjectType t, std::uint32_t seq) // flush this leaf ++flushed; - assert(node->getSeq() == seq_); + assert(node->cowid() == cowid_); child->updateHash(); + child->unshare(); - if (doWrite && backed_) - child = writeNode(t, seq, std::move(child)); - else - child->setSeq(0); + if (doWrite) + child = writeNode(t, std::move(child)); node->shareChild(branch, child); } @@ -1042,11 +1016,11 @@ SHAMap::walkSubTree(bool doWrite, NodeObjectType t, std::uint32_t seq) node->updateHashDeep(); // This inner node can now be shared - if (doWrite && backed_) + node->unshare(); + + if (doWrite) node = std::static_pointer_cast( - writeNode(t, seq, std::move(node))); - else - node->setSeq(0); + writeNode(t, std::move(node))); ++flushed; @@ -1058,7 +1032,7 @@ SHAMap::walkSubTree(bool doWrite, NodeObjectType t, std::uint32_t seq) stack.pop(); // Hook this inner node to its parent - assert(parent->getSeq() == seq_); + assert(parent->cowid() == cowid_); parent->shareChild(pos, node); // Continue with parent's next child, if any @@ -1078,7 +1052,7 @@ SHAMap::dump(bool hash) const int leafCount = 0; JLOG(journal_.info()) << " MAP Contains"; - std::stack> stack; + std::stack> stack; stack.push({root_.get(), SHAMapNodeID()}); do @@ -1089,20 +1063,20 @@ SHAMap::dump(bool hash) const JLOG(journal_.info()) << node->getString(nodeID); if (hash) { - JLOG(journal_.info()) << "Hash: " << node->getNodeHash(); + JLOG(journal_.info()) << "Hash: " << node->getHash(); } if (node->isInner()) { auto inner = static_cast(node); - for (int i = 0; i < 16; ++i) + for (int i = 0; i < branchFactor; ++i) { if (!inner->isEmptyBranch(i)) { auto child = inner->getChildPointer(i); if (child) { - assert(child->getNodeHash() == inner->getChildHash(i)); + assert(child->getHash() == inner->getChildHash(i)); stack.push({child, nodeID.getChildNodeID(i)}); } } @@ -1115,22 +1089,22 @@ SHAMap::dump(bool hash) const JLOG(journal_.info()) << leafCount << " resident leaves"; } -std::shared_ptr -SHAMap::getCache(SHAMapHash const& hash) const +std::shared_ptr +SHAMap::cacheLookup(SHAMapHash const& hash) const { auto ret = f_.getTreeNodeCache(ledgerSeq_)->fetch(hash.as_uint256()); - assert(!ret || !ret->getSeq()); + assert(!ret || !ret->cowid()); return ret; } void SHAMap::canonicalize( SHAMapHash const& hash, - std::shared_ptr& node) const + std::shared_ptr& node) const { assert(backed_); - assert(node->getSeq() == 0); - assert(node->getNodeHash() == hash); + assert(node->cowid() == 0); + assert(node->getHash() == hash); f_.getTreeNodeCache(ledgerSeq_) ->canonicalize_replace_client(hash.as_uint256(), node); diff --git a/src/ripple/shamap/impl/SHAMapDelta.cpp b/src/ripple/shamap/impl/SHAMapDelta.cpp index 67a26dc5cb2..98efec2ce03 100644 --- a/src/ripple/shamap/impl/SHAMapDelta.cpp +++ b/src/ripple/shamap/impl/SHAMapDelta.cpp @@ -32,7 +32,7 @@ namespace ripple { bool SHAMap::walkBranch( - SHAMapAbstractNode* node, + SHAMapTreeNode* node, std::shared_ptr const& otherMapItem, bool isFirstMap, Delta& differences, @@ -40,7 +40,7 @@ SHAMap::walkBranch( { // Walk a branch of a SHAMap that's matched by an empty branch or single // item in the other map - std::stack> nodeStack; + std::stack> nodeStack; nodeStack.push(node); bool emptyBranch = !otherMapItem; @@ -61,7 +61,7 @@ SHAMap::walkBranch( else { // This is a leaf node, process its item - auto item = static_cast(node)->peekItem(); + auto item = static_cast(node)->peekItem(); if (emptyBranch || (item->key() != otherMapItem->key())) { @@ -133,7 +133,7 @@ SHAMap::compare(SHAMap const& otherMap, Delta& differences, int maxCount) const if (getHash() == otherMap.getHash()) return true; - using StackEntry = std::pair; + using StackEntry = std::pair; std::stack> nodeStack; // track nodes we've pushed @@ -152,8 +152,8 @@ SHAMap::compare(SHAMap const& otherMap, Delta& differences, int maxCount) const if (ourNode->isLeaf() && otherNode->isLeaf()) { // two leaves - auto ours = static_cast(ourNode); - auto other = static_cast(otherNode); + auto ours = static_cast(ourNode); + auto other = static_cast(otherNode); if (ours->peekItem()->key() == other->peekItem()->key()) { if (ours->peekItem()->peekData() != @@ -188,14 +188,14 @@ SHAMap::compare(SHAMap const& otherMap, Delta& differences, int maxCount) const else if (ourNode->isInner() && otherNode->isLeaf()) { auto ours = static_cast(ourNode); - auto other = static_cast(otherNode); + auto other = static_cast(otherNode); if (!walkBranch( ours, other->peekItem(), true, differences, maxCount)) return false; } else if (ourNode->isLeaf() && otherNode->isInner()) { - auto ours = static_cast(ourNode); + auto ours = static_cast(ourNode); auto other = static_cast(otherNode); if (!otherMap.walkBranch( other, ours->peekItem(), false, differences, maxCount)) @@ -211,7 +211,7 @@ SHAMap::compare(SHAMap const& otherMap, Delta& differences, int maxCount) const if (other->isEmptyBranch(i)) { // We have a branch, the other tree does not - SHAMapAbstractNode* iNode = descendThrow(ours, i); + SHAMapTreeNode* iNode = descendThrow(ours, i); if (!walkBranch( iNode, std::shared_ptr(), @@ -223,8 +223,7 @@ SHAMap::compare(SHAMap const& otherMap, Delta& differences, int maxCount) const else if (ours->isEmptyBranch(i)) { // The other tree has a branch, we do not - SHAMapAbstractNode* iNode = - otherMap.descendThrow(other, i); + SHAMapTreeNode* iNode = otherMap.descendThrow(other, i); if (!otherMap.walkBranch( iNode, std::shared_ptr(), @@ -267,7 +266,7 @@ SHAMap::walkMap(std::vector& missingNodes, int maxMissing) { if (!node->isEmptyBranch(i)) { - std::shared_ptr nextNode = + std::shared_ptr nextNode = descendNoStore(node, i); if (nextNode) diff --git a/src/ripple/shamap/impl/SHAMapInnerNode.cpp b/src/ripple/shamap/impl/SHAMapInnerNode.cpp new file mode 100644 index 00000000000..62258544af0 --- /dev/null +++ b/src/ripple/shamap/impl/SHAMapInnerNode.cpp @@ -0,0 +1,313 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace ripple { + +std::mutex SHAMapInnerNode::childLock; + +std::shared_ptr +SHAMapInnerNode::clone(std::uint32_t cowid) const +{ + auto p = std::make_shared(cowid); + p->hash_ = hash_; + p->mIsBranch = mIsBranch; + p->mFullBelowGen = mFullBelowGen; + p->mHashes = mHashes; + std::lock_guard lock(childLock); + for (int i = 0; i < 16; ++i) + p->mChildren[i] = mChildren[i]; + return p; +} + +std::shared_ptr +SHAMapInnerNode::makeFullInner( + Slice data, + SHAMapHash const& hash, + bool hashValid) +{ + if (data.size() != 512) + Throw("Invalid FI node"); + + auto ret = std::make_shared(0); + + Serializer s(data.data(), data.size()); + + for (int i = 0; i < 16; ++i) + { + s.getBitString(ret->mHashes[i].as_uint256(), i * 32); + + if (ret->mHashes[i].isNonZero()) + ret->mIsBranch |= (1 << i); + } + + if (hashValid) + ret->hash_ = hash; + else + ret->updateHash(); + return ret; +} + +std::shared_ptr +SHAMapInnerNode::makeCompressedInner(Slice data) +{ + Serializer s(data.data(), data.size()); + + int len = s.getLength(); + + auto ret = std::make_shared(0); + + for (int i = 0; i < (len / 33); ++i) + { + int pos; + + if (!s.get8(pos, 32 + (i * 33))) + Throw("short CI node"); + + if ((pos < 0) || (pos >= 16)) + Throw("invalid CI node"); + + s.getBitString(ret->mHashes[pos].as_uint256(), i * 33); + + if (ret->mHashes[pos].isNonZero()) + ret->mIsBranch |= (1 << pos); + } + + ret->updateHash(); + + return ret; +} + +void +SHAMapInnerNode::updateHash() +{ + uint256 nh; + if (mIsBranch != 0) + { + sha512_half_hasher h; + using beast::hash_append; + hash_append(h, HashPrefix::innerNode); + for (auto const& hh : mHashes) + hash_append(h, hh); + nh = static_cast(h); + } + hash_ = SHAMapHash{nh}; +} + +void +SHAMapInnerNode::updateHashDeep() +{ + for (auto pos = 0; pos < 16; ++pos) + { + if (mChildren[pos] != nullptr) + mHashes[pos] = mChildren[pos]->getHash(); + } + updateHash(); +} + +void +SHAMapInnerNode::serializeForWire(Serializer& s) const +{ + assert(!isEmpty()); + + // If the node is sparse, then only send non-empty branches: + if (getBranchCount() < 12) + { + // compressed node + for (int i = 0; i < mHashes.size(); ++i) + { + if (!isEmptyBranch(i)) + { + s.addBitString(mHashes[i].as_uint256()); + s.add8(i); + } + } + + s.add8(wireTypeCompressedInner); + } + else + { + for (auto const& hh : mHashes) + s.addBitString(hh.as_uint256()); + + s.add8(wireTypeInner); + } +} + +void +SHAMapInnerNode::serializeWithPrefix(Serializer& s) const +{ + assert(!isEmpty()); + + s.add32(HashPrefix::innerNode); + for (auto const& hh : mHashes) + s.addBitString(hh.as_uint256()); +} + +bool +SHAMapInnerNode::isEmpty() const +{ + return mIsBranch == 0; +} + +int +SHAMapInnerNode::getBranchCount() const +{ + int count = 0; + + for (int i = 0; i < 16; ++i) + if (!isEmptyBranch(i)) + ++count; + + return count; +} + +std::string +SHAMapInnerNode::getString(const SHAMapNodeID& id) const +{ + std::string ret = SHAMapTreeNode::getString(id); + for (int i = 0; i < mHashes.size(); ++i) + { + if (!isEmptyBranch(i)) + { + ret += "\n"; + ret += std::to_string(i); + ret += " = "; + ret += to_string(mHashes[i]); + } + } + return ret; +} + +// We are modifying an inner node +void +SHAMapInnerNode::setChild(int m, std::shared_ptr const& child) +{ + assert((m >= 0) && (m < 16)); + assert(cowid_ != 0); + assert(child.get() != this); + mHashes[m].zero(); + hash_.zero(); + if (child) + mIsBranch |= (1 << m); + else + mIsBranch &= ~(1 << m); + mChildren[m] = child; +} + +// finished modifying, now make shareable +void +SHAMapInnerNode::shareChild(int m, std::shared_ptr const& child) +{ + assert((m >= 0) && (m < 16)); + assert(cowid_ != 0); + assert(child); + assert(child.get() != this); + + mChildren[m] = child; +} + +SHAMapTreeNode* +SHAMapInnerNode::getChildPointer(int branch) +{ + assert(branch >= 0 && branch < 16); + + std::lock_guard lock(childLock); + return mChildren[branch].get(); +} + +std::shared_ptr +SHAMapInnerNode::getChild(int branch) +{ + assert(branch >= 0 && branch < 16); + + std::lock_guard lock(childLock); + return mChildren[branch]; +} + +std::shared_ptr +SHAMapInnerNode::canonicalizeChild( + int branch, + std::shared_ptr node) +{ + assert(branch >= 0 && branch < 16); + assert(node); + assert(node->getHash() == mHashes[branch]); + + std::lock_guard lock(childLock); + if (mChildren[branch]) + { + // There is already a node hooked up, return it + node = mChildren[branch]; + } + else + { + // Hook this node up + mChildren[branch] = node; + } + return node; +} + +void +SHAMapInnerNode::invariants(bool is_root) const +{ + unsigned count = 0; + for (int i = 0; i < 16; ++i) + { + if (mHashes[i].isNonZero()) + { + assert((mIsBranch & (1 << i)) != 0); + if (mChildren[i] != nullptr) + mChildren[i]->invariants(); + ++count; + } + else + { + assert((mIsBranch & (1 << i)) == 0); + } + } + if (!is_root) + { + assert(hash_.isNonZero()); + assert(count >= 1); + } + assert((count == 0) ? hash_.isZero() : hash_.isNonZero()); +} + +} // namespace ripple diff --git a/src/ripple/shamap/impl/SHAMapLeafNode.cpp b/src/ripple/shamap/impl/SHAMapLeafNode.cpp new file mode 100644 index 00000000000..4c2d82f8879 --- /dev/null +++ b/src/ripple/shamap/impl/SHAMapLeafNode.cpp @@ -0,0 +1,94 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include + +namespace ripple { + +SHAMapLeafNode::SHAMapLeafNode( + std::shared_ptr item, + std::uint32_t cowid) + : SHAMapTreeNode(cowid), item_(std::move(item)) +{ + assert(item_->peekData().size() >= 12); +} + +SHAMapLeafNode::SHAMapLeafNode( + std::shared_ptr item, + std::uint32_t cowid, + SHAMapHash const& hash) + : SHAMapTreeNode(cowid, hash), item_(std::move(item)) +{ + assert(item_->peekData().size() >= 12); +} + +std::shared_ptr const& +SHAMapLeafNode::peekItem() const +{ + return item_; +} + +bool +SHAMapLeafNode::setItem(std::shared_ptr i) +{ + assert(cowid_ != 0); + item_ = std::move(i); + + auto const oldHash = hash_; + + updateHash(); + + return (oldHash != hash_); +} + +std::string +SHAMapLeafNode::getString(const SHAMapNodeID& id) const +{ + std::string ret = SHAMapTreeNode::getString(id); + + auto const type = getType(); + + if (type == SHAMapNodeType::tnTRANSACTION_NM) + ret += ",txn\n"; + else if (type == SHAMapNodeType::tnTRANSACTION_MD) + ret += ",txn+md\n"; + else if (type == SHAMapNodeType::tnACCOUNT_STATE) + ret += ",as\n"; + else + ret += ",leaf\n"; + + ret += " Tag="; + ret += to_string(item_->key()); + ret += "\n Hash="; + ret += to_string(hash_); + ret += "/"; + ret += std::to_string(item_->size()); + return ret; +} + +void +SHAMapLeafNode::invariants(bool) const +{ + assert(hash_.isNonZero()); + assert(item_ != nullptr); +} + +} // namespace ripple diff --git a/src/ripple/shamap/impl/SHAMapNodeID.cpp b/src/ripple/shamap/impl/SHAMapNodeID.cpp index 5b14585943d..b239bb890a1 100644 --- a/src/ripple/shamap/impl/SHAMapNodeID.cpp +++ b/src/ripple/shamap/impl/SHAMapNodeID.cpp @@ -17,18 +17,17 @@ */ //============================================================================== -#include #include #include +#include +#include #include -#include #include -#include namespace ripple { -uint256 const& -SHAMapNodeID::Masks(int depth) +static uint256 const& +depthMask(unsigned int depth) { enum { mask_size = 65 }; @@ -49,106 +48,88 @@ SHAMapNodeID::Masks(int depth) entry[mask_size - 1] = selector; } }; + static masks_t const masks; return masks.entry[depth]; } // canonicalize the hash to a node ID for this depth -SHAMapNodeID::SHAMapNodeID(int depth, uint256 const& hash) - : mNodeID(hash), mDepth(depth) +SHAMapNodeID::SHAMapNodeID(unsigned int depth, uint256 const& hash) + : id_(hash), depth_(depth) { - assert((depth >= 0) && (depth < 65)); - assert(mNodeID == (mNodeID & Masks(depth))); -} - -SHAMapNodeID::SHAMapNodeID(void const* ptr, int len) -{ - if (len < 33) - mDepth = -1; - else - { - std::memcpy(mNodeID.begin(), ptr, 32); - mDepth = *(static_cast(ptr) + 32); - } -} - -std::string -SHAMapNodeID::getString() const -{ - if ((mDepth == 0) && (mNodeID.isZero())) - return "NodeID(root)"; - - return "NodeID(" + std::to_string(mDepth) + "," + to_string(mNodeID) + ")"; -} - -void -SHAMapNodeID::addIDRaw(Serializer& s) const -{ - s.addBitString(mNodeID); - s.add8(mDepth); + assert(depth <= SHAMap::leafDepth); + assert(id_ == (id_ & depthMask(depth))); } std::string SHAMapNodeID::getRawString() const { Serializer s(33); - addIDRaw(s); + s.addBitString(id_); + s.add8(depth_); return s.getString(); } -// This can be optimized to avoid the << if needed SHAMapNodeID -SHAMapNodeID::getChildNodeID(int m) const +SHAMapNodeID::getChildNodeID(unsigned int m) const +{ + assert(m < SHAMap::branchFactor); + + // A SHAMap has exactly 65 levels, so nodes must not exceed that + // depth; if they do, this breaks the invariant of never allowing + // the construction of a SHAMapNodeID at an invalid depth. We assert + // to catch this in debug builds. + // + // We throw (but never assert) if the node is at level 64, since + // entries at that depth are leaf nodes and have no children and even + // constructing a child node from them would break the above invariant. + assert(depth_ <= SHAMap::leafDepth); + + if (depth_ >= SHAMap::leafDepth) + Throw( + "Request for child node ID of " + to_string(*this)); + + if (id_ != (id_ & depthMask(depth_))) + Throw("Incorrect mask for " + to_string(*this)); + + SHAMapNodeID node{depth_ + 1, id_}; + node.id_.begin()[depth_ / 2] |= (depth_ & 1) ? m : (m << 4); + return node; +} + +[[nodiscard]] std::optional +deserializeSHAMapNodeID(void const* data, std::size_t size) { - assert((m >= 0) && (m < 16)); - assert(mDepth < 64); + std::optional ret; - uint256 child(mNodeID); - child.begin()[mDepth / 2] |= (mDepth & 1) ? m : (m << 4); + if (size == 33) + { + unsigned int depth = *(static_cast(data) + 32); + if (depth <= SHAMap::leafDepth) + { + auto const id = uint256::fromVoid(data); - return SHAMapNodeID(mDepth + 1, child); + if (id == (id & depthMask(depth))) + ret.emplace(depth, id); + } + } + + return ret; } -// Which branch would contain the specified hash -int -SHAMapNodeID::selectBranch(uint256 const& hash) const +[[nodiscard]] unsigned int +selectBranch(SHAMapNodeID const& id, uint256 const& hash) { - int branch = *(hash.begin() + (mDepth / 2)); + auto const depth = id.getDepth(); + auto branch = static_cast(*(hash.begin() + (depth / 2))); - if (mDepth & 1) + if (depth & 1) branch &= 0xf; else branch >>= 4; - assert((branch >= 0) && (branch < 16)); - + assert(branch < SHAMap::branchFactor); return branch; } -bool -SHAMapNodeID::has_common_prefix(SHAMapNodeID const& other) const -{ - assert(mDepth <= other.mDepth); - auto x = mNodeID.begin(); - auto y = other.mNodeID.begin(); - for (unsigned i = 0; i < mDepth / 2; ++i, ++x, ++y) - { - if (*x != *y) - return false; - } - if (mDepth & 1) - { - auto i = mDepth / 2; - return (*(mNodeID.begin() + i) & 0xF0) == - (*(other.mNodeID.begin() + i) & 0xF0); - } - return true; -} - -void -SHAMapNodeID::dump(beast::Journal journal) const -{ - JLOG(journal.debug()) << getString(); -} - } // namespace ripple diff --git a/src/ripple/shamap/impl/SHAMapSync.cpp b/src/ripple/shamap/impl/SHAMapSync.cpp index 19f3937985a..0b58641209d 100644 --- a/src/ripple/shamap/impl/SHAMapSync.cpp +++ b/src/ripple/shamap/impl/SHAMapSync.cpp @@ -18,8 +18,8 @@ //============================================================================== #include -#include #include +#include namespace ripple { @@ -28,20 +28,16 @@ SHAMap::visitLeaves( std::function const& item)> const& leafFunction) const { - visitNodes([&leafFunction](SHAMapAbstractNode& node) { + visitNodes([&leafFunction](SHAMapTreeNode& node) { if (!node.isInner()) - leafFunction(static_cast(node).peekItem()); + leafFunction(static_cast(node).peekItem()); return true; }); } void -SHAMap::visitNodes( - std::function const& function) const +SHAMap::visitNodes(std::function const& function) const { - // Visit every node in a SHAMap - assert(root_->isValid()); - if (!root_) return; @@ -63,7 +59,7 @@ SHAMap::visitNodes( uint256 childHash; if (!node->isEmptyBranch(pos)) { - std::shared_ptr child = + std::shared_ptr child = descendNoStore(node, pos); if (!function(*child)) return; @@ -104,26 +100,24 @@ SHAMap::visitNodes( void SHAMap::visitDifferences( SHAMap const* have, - std::function function) const + std::function function) const { // Visit every node in this SHAMap that is not present // in the specified SHAMap - assert(root_->isValid()); - if (!root_) return; - if (root_->getNodeHash().isZero()) + if (root_->getHash().isZero()) return; - if (have && (root_->getNodeHash() == have->root_->getNodeHash())) + if (have && (root_->getHash() == have->root_->getHash())) return; if (root_->isLeaf()) { - auto leaf = std::static_pointer_cast(root_); + auto leaf = std::static_pointer_cast(root_); if (!have || - !have->hasLeafNode(leaf->peekItem()->key(), leaf->getNodeHash())) + !have->hasLeafNode(leaf->peekItem()->key(), leaf->getHash())) function(*root_); return; } @@ -160,7 +154,7 @@ SHAMap::visitDifferences( else if ( !have || !have->hasLeafNode( - static_cast(next)->peekItem()->key(), + static_cast(next)->peekItem()->key(), childHash)) { if (!function(*next)) @@ -247,7 +241,7 @@ SHAMap::gmn_ProcessNodes(MissingNodes& mn, MissingNodes::StackEntry& se) if (backed_) { f_.getFullBelowCache(ledgerSeq_) - ->insert(node->getNodeHash().as_uint256()); + ->insert(node->getHash().as_uint256()); } } @@ -320,8 +314,7 @@ SHAMap::gmn_ProcessDeferredReads(MissingNodes& mn) std::vector> SHAMap::getMissingNodes(int max, SHAMapSyncFilter* filter) { - assert(root_->isValid()); - assert(root_->getNodeHash().isNonZero()); + assert(root_->getHash().isNonZero()); assert(max > 0); MissingNodes mn( @@ -429,23 +422,9 @@ SHAMap::getMissingNodes(int max, SHAMapSyncFilter* filter) return std::move(mn.missingNodes_); } -std::vector -SHAMap::getNeededHashes(int max, SHAMapSyncFilter* filter) -{ - auto ret = getMissingNodes(max, filter); - - std::vector hashes; - hashes.reserve(ret.size()); - - for (auto const& n : ret) - hashes.push_back(n.second); - - return hashes; -} - bool SHAMap::getNodeFat( - SHAMapNodeID wanted, + SHAMapNodeID const& wanted, std::vector& nodeIDs, std::vector& rawNodes, bool fatLeaves, @@ -459,7 +438,7 @@ SHAMap::getNodeFat( while (node && node->isInner() && (nodeID.getDepth() < wanted.getDepth())) { - int branch = nodeID.selectBranch(wanted.getNodeID()); + int branch = selectBranch(nodeID, wanted.getNodeID()); auto inner = static_cast(node); if (inner->isEmptyBranch(branch)) return false; @@ -482,7 +461,7 @@ SHAMap::getNodeFat( return false; } - std::stack> stack; + std::stack> stack; stack.emplace(node, nodeID, depth); while (!stack.empty()) @@ -493,7 +472,7 @@ SHAMap::getNodeFat( { // Add this node to the reply Serializer s; - node->addRaw(s, snfWIRE); + node->serializeForWire(s); nodeIDs.push_back(nodeID); rawNodes.push_back(std::move(s.modData())); } @@ -527,7 +506,7 @@ SHAMap::getNodeFat( { // Just include this node Serializer ns; - childNode->addRaw(ns, snfWIRE); + childNode->serializeForWire(ns); nodeIDs.push_back(childID); rawNodes.push_back(std::move(ns.modData())); } @@ -540,11 +519,10 @@ SHAMap::getNodeFat( return true; } -bool -SHAMap::getRootNode(Serializer& s, SHANodeFormat format) const +void +SHAMap::serializeRoot(Serializer& s) const { - root_->addRaw(s, format); - return true; + root_->serializeForWire(s); } SHAMapAddNode @@ -554,16 +532,16 @@ SHAMap::addRootNode( SHAMapSyncFilter* filter) { // we already have a root_ node - if (root_->getNodeHash().isNonZero()) + if (root_->getHash().isNonZero()) { JLOG(journal_.trace()) << "got root node, already have one"; - assert(root_->getNodeHash() == hash); + assert(root_->getHash() == hash); return SHAMapAddNode::duplicate(); } - assert(seq_ >= 1); - auto node = SHAMapAbstractNode::makeFromWire(rootNode); - if (!node || !node->isValid() || node->getNodeHash() != hash) + assert(cowid_ >= 1); + auto node = SHAMapTreeNode::makeFromWire(rootNode); + if (!node || node->getHash() != hash) return SHAMapAddNode::invalid(); if (backed_) @@ -577,10 +555,10 @@ SHAMap::addRootNode( if (filter) { Serializer s; - root_->addRaw(s, snfPREFIX); + root_->serializeWithPrefix(s); filter->gotNode( false, - root_->getNodeHash(), + root_->getHash(), ledgerSeq_, std::move(s.modData()), root_->getType()); @@ -595,7 +573,6 @@ SHAMap::addKnownNode( Slice const& rawNode, SHAMapSyncFilter* filter) { - // return value: true=okay, false=error assert(!node.isRoot()); if (!isSynching()) @@ -605,7 +582,7 @@ SHAMap::addKnownNode( } auto const generation = f_.getFullBelowCache(ledgerSeq_)->getGeneration(); - auto newNode = SHAMapAbstractNode::makeFromWire(rawNode); + auto newNode = SHAMapTreeNode::makeFromWire(rawNode); SHAMapNodeID iNodeID; auto iNode = root_.get(); @@ -613,7 +590,7 @@ SHAMap::addKnownNode( !static_cast(iNode)->isFullBelow(generation) && (iNodeID.getDepth() < node.getDepth())) { - int branch = iNodeID.selectBranch(node.getNodeID()); + int branch = selectBranch(iNodeID, node.getNodeID()); assert(branch >= 0); auto inner = static_cast(iNode); if (inner->isEmptyBranch(branch)) @@ -634,14 +611,17 @@ SHAMap::addKnownNode( if (iNode == nullptr) { - if (!newNode || !newNode->isValid() || - childHash != newNode->getNodeHash()) + if (!newNode || childHash != newNode->getHash()) { JLOG(journal_.warn()) << "Corrupt node received"; return SHAMapAddNode::invalid(); } - if (!newNode->isInBounds(iNodeID)) + // Inner nodes must be at a level strictly less than 64 + // but leaf nodes (while notionally at level 64) can be + // at any depth up to and including 64: + if ((iNodeID.getDepth() > leafDepth) || + (newNode->isInner() && iNodeID.getDepth() == leafDepth)) { // Map is provably invalid state_ = SHAMapState::Invalid; @@ -666,7 +646,7 @@ SHAMap::addKnownNode( if (filter) { Serializer s; - newNode->addRaw(s, snfPREFIX); + newNode->serializeWithPrefix(s); filter->gotNode( false, childHash, @@ -687,7 +667,7 @@ bool SHAMap::deepCompare(SHAMap& other) const { // Intended for debug/test only - std::stack> stack; + std::stack> stack; stack.push({root_.get(), other.root_.get()}); @@ -701,7 +681,7 @@ SHAMap::deepCompare(SHAMap& other) const JLOG(journal_.info()) << "unable to fetch node"; return false; } - else if (otherNode->getNodeHash() != node->getNodeHash()) + else if (otherNode->getHash() != node->getHash()) { JLOG(journal_.warn()) << "node hash mismatch"; return false; @@ -711,9 +691,9 @@ SHAMap::deepCompare(SHAMap& other) const { if (!otherNode->isLeaf()) return false; - auto& nodePeek = static_cast(node)->peekItem(); + auto& nodePeek = static_cast(node)->peekItem(); auto& otherNodePeek = - static_cast(otherNode)->peekItem(); + static_cast(otherNode)->peekItem(); if (nodePeek->key() != otherNodePeek->key()) return false; if (nodePeek->peekData() != otherNodePeek->peekData()) @@ -765,7 +745,7 @@ SHAMap::hasInnerNode( while (node->isInner() && (nodeID.getDepth() < targetNodeID.getDepth())) { - int branch = nodeID.selectBranch(targetNodeID.getNodeID()); + int branch = selectBranch(nodeID, targetNodeID.getNodeID()); auto inner = static_cast(node); if (inner->isEmptyBranch(branch)) return false; @@ -774,7 +754,7 @@ SHAMap::hasInnerNode( nodeID = nodeID.getChildNodeID(branch); } - return (node->isInner()) && (node->getNodeHash() == targetNodeHash); + return (node->isInner()) && (node->getHash() == targetNodeHash); } /** Does this map have this leaf node? @@ -786,11 +766,11 @@ SHAMap::hasLeafNode(uint256 const& tag, SHAMapHash const& targetNodeHash) const SHAMapNodeID nodeID; if (!node->isInner()) // only one leaf node in the tree - return node->getNodeHash() == targetNodeHash; + return node->getHash() == targetNodeHash; do { - int branch = nodeID.selectBranch(tag); + int branch = selectBranch(nodeID, tag); auto inner = static_cast(node); if (inner->isEmptyBranch(branch)) return false; // Dead end, node must not be here @@ -807,35 +787,4 @@ SHAMap::hasLeafNode(uint256 const& tag, SHAMapHash const& targetNodeHash) const // already } -/** -@param have A pointer to the map that the recipient already has (if any). -@param includeLeaves True if leaf nodes should be included. -@param max The maximum number of nodes to return. -@param func The functor to call for each node added to the FetchPack. - -Note: a caller should set includeLeaves to false for transaction trees. -There's no point in including the leaves of transaction trees. -*/ -void -SHAMap::getFetchPack( - SHAMap const* have, - bool includeLeaves, - int max, - std::function func) const -{ - visitDifferences( - have, [includeLeaves, &max, &func](SHAMapAbstractNode& smn) -> bool { - if (includeLeaves || smn.isInner()) - { - Serializer s; - smn.addRaw(s, snfPREFIX); - func(smn.getNodeHash(), s.peekData()); - - if (--max <= 0) - return false; - } - return true; - }); -} - } // namespace ripple diff --git a/src/ripple/shamap/impl/SHAMapTreeNode.cpp b/src/ripple/shamap/impl/SHAMapTreeNode.cpp index 147787ae929..09416ec2424 100644 --- a/src/ripple/shamap/impl/SHAMapTreeNode.cpp +++ b/src/ripple/shamap/impl/SHAMapTreeNode.cpp @@ -24,61 +24,21 @@ #include #include #include +#include +#include +#include #include +#include +#include #include #include namespace ripple { -std::mutex SHAMapInnerNode::childLock; - -SHAMapAbstractNode::~SHAMapAbstractNode() = default; - -std::shared_ptr -SHAMapInnerNode::clone(std::uint32_t seq) const -{ - auto p = std::make_shared(seq); - p->mHash = mHash; - p->mIsBranch = mIsBranch; - p->mFullBelowGen = mFullBelowGen; - p->mHashes = mHashes; - std::lock_guard lock(childLock); - for (int i = 0; i < 16; ++i) - p->mChildren[i] = mChildren[i]; - return p; -} - -std::shared_ptr -SHAMapTreeNode::clone(std::uint32_t seq) const -{ - return std::make_shared(mItem, mType, seq, mHash); -} - -SHAMapTreeNode::SHAMapTreeNode( - std::shared_ptr item, - TNType type, - std::uint32_t seq) - : SHAMapAbstractNode(type, seq), mItem(std::move(item)) -{ - assert(mItem->peekData().size() >= 12); - updateHash(); -} - -SHAMapTreeNode::SHAMapTreeNode( - std::shared_ptr item, - TNType type, - std::uint32_t seq, - SHAMapHash const& hash) - : SHAMapAbstractNode(type, seq, hash), mItem(std::move(item)) -{ - assert(mItem->peekData().size() >= 12); -} - -std::shared_ptr -SHAMapAbstractNode::makeTransaction( +std::shared_ptr +SHAMapTreeNode::makeTransaction( Slice data, - std::uint32_t seq, SHAMapHash const& hash, bool hashValid) { @@ -89,17 +49,14 @@ SHAMapAbstractNode::makeTransaction( sha512Half(HashPrefix::transactionID, data), s); if (hashValid) - return std::make_shared( - std::move(item), tnTRANSACTION_NM, seq, hash); + return std::make_shared(std::move(item), 0, hash); - return std::make_shared( - std::move(item), tnTRANSACTION_NM, seq); + return std::make_shared(std::move(item), 0); } -std::shared_ptr -SHAMapAbstractNode::makeTransactionWithMeta( +std::shared_ptr +SHAMapTreeNode::makeTransactionWithMeta( Slice data, - std::uint32_t seq, SHAMapHash const& hash, bool hashValid) { @@ -120,17 +77,15 @@ SHAMapAbstractNode::makeTransactionWithMeta( auto item = std::make_shared(tag, s.peekData()); if (hashValid) - return std::make_shared( - std::move(item), tnTRANSACTION_MD, seq, hash); + return std::make_shared( + std::move(item), 0, hash); - return std::make_shared( - std::move(item), tnTRANSACTION_MD, seq); + return std::make_shared(std::move(item), 0); } -std::shared_ptr -SHAMapAbstractNode::makeAccountState( +std::shared_ptr +SHAMapTreeNode::makeAccountState( Slice data, - std::uint32_t seq, SHAMapHash const& hash, bool hashValid) { @@ -154,74 +109,14 @@ SHAMapAbstractNode::makeAccountState( auto item = std::make_shared(tag, s.peekData()); if (hashValid) - return std::make_shared( - std::move(item), tnACCOUNT_STATE, seq, hash); + return std::make_shared( + std::move(item), 0, hash); - return std::make_shared( - std::move(item), tnACCOUNT_STATE, seq); -} - -std::shared_ptr -SHAMapInnerNode::makeFullInner( - Slice data, - std::uint32_t seq, - SHAMapHash const& hash, - bool hashValid) -{ - if (data.size() != 512) - Throw("Invalid FI node"); - - auto ret = std::make_shared(seq); - - Serializer s(data.data(), data.size()); - - for (int i = 0; i < 16; ++i) - { - s.getBitString(ret->mHashes[i].as_uint256(), i * 32); - - if (ret->mHashes[i].isNonZero()) - ret->mIsBranch |= (1 << i); - } - - if (hashValid) - ret->mHash = hash; - else - ret->updateHash(); - return ret; + return std::make_shared(std::move(item), 0); } -std::shared_ptr -SHAMapInnerNode::makeCompressedInner(Slice data, std::uint32_t seq) -{ - Serializer s(data.data(), data.size()); - - int len = s.getLength(); - - auto ret = std::make_shared(seq); - - for (int i = 0; i < (len / 33); ++i) - { - int pos; - - if (!s.get8(pos, 32 + (i * 33))) - Throw("short CI node"); - - if ((pos < 0) || (pos >= 16)) - Throw("invalid CI node"); - - s.getBitString(ret->mHashes[pos].as_uint256(), i * 33); - - if (ret->mHashes[pos].isNonZero()) - ret->mIsBranch |= (1 << pos); - } - - ret->updateHash(); - - return ret; -} - -std::shared_ptr -SHAMapAbstractNode::makeFromWire(Slice rawNode) +std::shared_ptr +SHAMapTreeNode::makeFromWire(Slice rawNode) { if (rawNode.empty()) return {}; @@ -233,29 +128,27 @@ SHAMapAbstractNode::makeFromWire(Slice rawNode) bool const hashValid = false; SHAMapHash const hash; - std::uint32_t const seq = 0; + if (type == wireTypeTransaction) + return makeTransaction(rawNode, hash, hashValid); - if (type == 0) - return makeTransaction(rawNode, seq, hash, hashValid); + if (type == wireTypeAccountState) + return makeAccountState(rawNode, hash, hashValid); - if (type == 1) - return makeAccountState(rawNode, seq, hash, hashValid); + if (type == wireTypeInner) + return SHAMapInnerNode::makeFullInner(rawNode, hash, hashValid); - if (type == 2) - return SHAMapInnerNode::makeFullInner(rawNode, seq, hash, hashValid); + if (type == wireTypeCompressedInner) + return SHAMapInnerNode::makeCompressedInner(rawNode); - if (type == 3) - return SHAMapInnerNode::makeCompressedInner(rawNode, seq); - - if (type == 4) - return makeTransactionWithMeta(rawNode, seq, hash, hashValid); + if (type == wireTypeTransactionWithMeta) + return makeTransactionWithMeta(rawNode, hash, hashValid); Throw( "wire: Unknown type (" + std::to_string(type) + ")"); } -std::shared_ptr -SHAMapAbstractNode::makeFromPrefix(Slice rawNode, SHAMapHash const& hash) +std::shared_ptr +SHAMapTreeNode::makeFromPrefix(Slice rawNode, SHAMapHash const& hash) { if (rawNode.size() < 4) Throw("prefix: short node"); @@ -271,19 +164,18 @@ SHAMapAbstractNode::makeFromPrefix(Slice rawNode, SHAMapHash const& hash) rawNode.remove_prefix(4); bool const hashValid = true; - std::uint32_t const seq = 0; if (type == HashPrefix::transactionID) - return makeTransaction(rawNode, seq, hash, hashValid); + return makeTransaction(rawNode, hash, hashValid); if (type == HashPrefix::leafNode) - return makeAccountState(rawNode, seq, hash, hashValid); + return makeAccountState(rawNode, hash, hashValid); if (type == HashPrefix::innerNode) - return SHAMapInnerNode::makeFullInner(rawNode, seq, hash, hashValid); + return SHAMapInnerNode::makeFullInner(rawNode, hash, hashValid); if (type == HashPrefix::txNode) - return makeTransactionWithMeta(rawNode, seq, hash, hashValid); + return makeTransactionWithMeta(rawNode, hash, hashValid); Throw( "prefix: unknown type (" + @@ -291,378 +183,10 @@ SHAMapAbstractNode::makeFromPrefix(Slice rawNode, SHAMapHash const& hash) ")"); } -bool -SHAMapInnerNode::updateHash() -{ - uint256 nh; - if (mIsBranch != 0) - { - sha512_half_hasher h; - using beast::hash_append; - hash_append(h, HashPrefix::innerNode); - for (auto const& hh : mHashes) - hash_append(h, hh); - nh = static_cast(h); - } - if (nh == mHash.as_uint256()) - return false; - mHash = SHAMapHash{nh}; - return true; -} - -void -SHAMapInnerNode::updateHashDeep() -{ - for (auto pos = 0; pos < 16; ++pos) - { - if (mChildren[pos] != nullptr) - mHashes[pos] = mChildren[pos]->getNodeHash(); - } - updateHash(); -} - -bool -SHAMapTreeNode::updateHash() -{ - uint256 nh; - if (mType == tnTRANSACTION_NM) - { - nh = - sha512Half(HashPrefix::transactionID, makeSlice(mItem->peekData())); - } - else if (mType == tnACCOUNT_STATE) - { - nh = sha512Half( - HashPrefix::leafNode, makeSlice(mItem->peekData()), mItem->key()); - } - else if (mType == tnTRANSACTION_MD) - { - nh = sha512Half( - HashPrefix::txNode, makeSlice(mItem->peekData()), mItem->key()); - } - else - assert(false); - - if (nh == mHash.as_uint256()) - return false; - - mHash = SHAMapHash{nh}; - return true; -} - -void -SHAMapInnerNode::addRaw(Serializer& s, SHANodeFormat format) const -{ - assert((format == snfPREFIX) || (format == snfWIRE) || (format == snfHASH)); - - if (mType == tnERROR) - Throw("invalid I node type"); - - if (format == snfHASH) - { - s.addBitString(mHash.as_uint256()); - } - else if (mType == tnINNER) - { - assert(!isEmpty()); - - if (format == snfPREFIX) - { - s.add32(HashPrefix::innerNode); - - for (auto const& hh : mHashes) - s.addBitString(hh.as_uint256()); - } - else // format == snfWIRE - { - if (getBranchCount() < 12) - { - // compressed node - for (int i = 0; i < mHashes.size(); ++i) - if (!isEmptyBranch(i)) - { - s.addBitString(mHashes[i].as_uint256()); - s.add8(i); - } - - s.add8(3); - } - else - { - for (auto const& hh : mHashes) - s.addBitString(hh.as_uint256()); - - s.add8(2); - } - } - } - else - assert(false); -} - -void -SHAMapTreeNode::addRaw(Serializer& s, SHANodeFormat format) const -{ - assert((format == snfPREFIX) || (format == snfWIRE) || (format == snfHASH)); - - if (mType == tnERROR) - Throw("invalid I node type"); - - if (format == snfHASH) - { - s.addBitString(mHash.as_uint256()); - } - else if (mType == tnACCOUNT_STATE) - { - if (format == snfPREFIX) - { - s.add32(HashPrefix::leafNode); - s.addRaw(mItem->peekData()); - s.addBitString(mItem->key()); - } - else - { - s.addRaw(mItem->peekData()); - s.addBitString(mItem->key()); - s.add8(1); - } - } - else if (mType == tnTRANSACTION_NM) - { - if (format == snfPREFIX) - { - s.add32(HashPrefix::transactionID); - s.addRaw(mItem->peekData()); - } - else - { - s.addRaw(mItem->peekData()); - s.add8(0); - } - } - else if (mType == tnTRANSACTION_MD) - { - if (format == snfPREFIX) - { - s.add32(HashPrefix::txNode); - s.addRaw(mItem->peekData()); - s.addBitString(mItem->key()); - } - else - { - s.addRaw(mItem->peekData()); - s.addBitString(mItem->key()); - s.add8(4); - } - } - else - assert(false); -} - -bool -SHAMapTreeNode::setItem(std::shared_ptr i, TNType type) -{ - mType = type; - mItem = std::move(i); - assert(isLeaf()); - assert(mSeq != 0); - return updateHash(); -} - -bool -SHAMapInnerNode::isEmpty() const -{ - return mIsBranch == 0; -} - -int -SHAMapInnerNode::getBranchCount() const -{ - assert(isInner()); - int count = 0; - - for (int i = 0; i < 16; ++i) - if (!isEmptyBranch(i)) - ++count; - - return count; -} - -std::string -SHAMapAbstractNode::getString(const SHAMapNodeID& id) const -{ - std::string ret = "NodeID("; - ret += beast::lexicalCastThrow(id.getDepth()); - ret += ","; - ret += to_string(id.getNodeID()); - ret += ")"; - return ret; -} - -std::string -SHAMapInnerNode::getString(const SHAMapNodeID& id) const -{ - std::string ret = SHAMapAbstractNode::getString(id); - for (int i = 0; i < mHashes.size(); ++i) - { - if (!isEmptyBranch(i)) - { - ret += "\nb"; - ret += beast::lexicalCastThrow(i); - ret += " = "; - ret += to_string(mHashes[i]); - } - } - return ret; -} - std::string SHAMapTreeNode::getString(const SHAMapNodeID& id) const { - std::string ret = SHAMapAbstractNode::getString(id); - if (mType == tnTRANSACTION_NM) - ret += ",txn\n"; - else if (mType == tnTRANSACTION_MD) - ret += ",txn+md\n"; - else if (mType == tnACCOUNT_STATE) - ret += ",as\n"; - else - ret += ",leaf\n"; - - ret += " Tag="; - ret += to_string(peekItem()->key()); - ret += "\n Hash="; - ret += to_string(mHash); - ret += "/"; - ret += beast::lexicalCast(mItem->size()); - return ret; -} - -// We are modifying an inner node -void -SHAMapInnerNode::setChild( - int m, - std::shared_ptr const& child) -{ - assert((m >= 0) && (m < 16)); - assert(mType == tnINNER); - assert(mSeq != 0); - assert(child.get() != this); - mHashes[m].zero(); - mHash.zero(); - if (child) - mIsBranch |= (1 << m); - else - mIsBranch &= ~(1 << m); - mChildren[m] = child; -} - -// finished modifying, now make shareable -void -SHAMapInnerNode::shareChild( - int m, - std::shared_ptr const& child) -{ - assert((m >= 0) && (m < 16)); - assert(mType == tnINNER); - assert(mSeq != 0); - assert(child); - assert(child.get() != this); - - mChildren[m] = child; -} - -SHAMapAbstractNode* -SHAMapInnerNode::getChildPointer(int branch) -{ - assert(branch >= 0 && branch < 16); - assert(isInner()); - - std::lock_guard lock(childLock); - return mChildren[branch].get(); -} - -std::shared_ptr -SHAMapInnerNode::getChild(int branch) -{ - assert(branch >= 0 && branch < 16); - assert(isInner()); - - std::lock_guard lock(childLock); - return mChildren[branch]; -} - -std::shared_ptr -SHAMapInnerNode::canonicalizeChild( - int branch, - std::shared_ptr node) -{ - assert(branch >= 0 && branch < 16); - assert(isInner()); - assert(node); - assert(node->getNodeHash() == mHashes[branch]); - - std::lock_guard lock(childLock); - if (mChildren[branch]) - { - // There is already a node hooked up, return it - node = mChildren[branch]; - } - else - { - // Hook this node up - mChildren[branch] = node; - } - return node; -} - -uint256 const& -SHAMapInnerNode::key() const -{ - Throw("SHAMapInnerNode::key() should never be called"); - static uint256 x; - return x; -} - -uint256 const& -SHAMapTreeNode::key() const -{ - return mItem->key(); -} - -void -SHAMapInnerNode::invariants(bool is_root) const -{ - assert(mType == tnINNER); - unsigned count = 0; - for (int i = 0; i < 16; ++i) - { - if (mHashes[i].isNonZero()) - { - assert((mIsBranch & (1 << i)) != 0); - if (mChildren[i] != nullptr) - mChildren[i]->invariants(); - ++count; - } - else - { - assert((mIsBranch & (1 << i)) == 0); - } - } - if (!is_root) - { - assert(mHash.isNonZero()); - assert(count >= 1); - } - assert((count == 0) ? mHash.isZero() : mHash.isNonZero()); -} - -void -SHAMapTreeNode::invariants(bool) const -{ - assert(mType >= tnTRANSACTION_NM); - assert(mHash.isNonZero()); - assert(mItem != nullptr); + return to_string(id); } } // namespace ripple diff --git a/src/test/app/AccountDelete_test.cpp b/src/test/app/AccountDelete_test.cpp index 71fff4523d6..09ebd714804 100644 --- a/src/test/app/AccountDelete_test.cpp +++ b/src/test/app/AccountDelete_test.cpp @@ -411,7 +411,7 @@ class AccountDelete_test : public beast::unit_test::suite jv[jss::TransactionType] = jss::PaymentChannelClaim; jv[jss::Flags] = tfClose; jv[jss::Account] = account.human(); - jv[sfPayChannel.jsonName] = to_string(payChanKeylet.key); + jv[sfChannel.jsonName] = to_string(payChanKeylet.key); jv[sfPublicKey.jsonName] = strHex(pk.slice()); return jv; }; @@ -512,7 +512,7 @@ class AccountDelete_test : public beast::unit_test::suite jv[jss::TransactionType] = jss::PaymentChannelClaim; jv[jss::Flags] = tfUniversal; jv[jss::Account] = alice.human(); - jv[sfPayChannel.jsonName] = to_string(payChanKey.key); + jv[sfChannel.jsonName] = to_string(payChanKey.key); jv[sfBalance.jsonName] = payChanXRP.value().getJson(JsonOptions::none); return jv; diff --git a/src/test/app/AccountTxPaging_test.cpp b/src/test/app/AccountTxPaging_test.cpp index 0169c5609c3..c164c86e7cf 100644 --- a/src/test/app/AccountTxPaging_test.cpp +++ b/src/test/app/AccountTxPaging_test.cpp @@ -686,7 +686,7 @@ class AccountTxPaging_test : public beast::unit_test::suite payChanFund[jss::TransactionType] = jss::PaymentChannelFund; payChanFund[jss::Flags] = tfUniversal; payChanFund[jss::Account] = alice.human(); - payChanFund[sfPayChannel.jsonName] = payChanIndex; + payChanFund[sfChannel.jsonName] = payChanIndex; payChanFund[jss::Amount] = XRP(200).value().getJson(JsonOptions::none); env(payChanFund, sig(alie)); @@ -699,7 +699,7 @@ class AccountTxPaging_test : public beast::unit_test::suite payChanClaim[jss::TransactionType] = jss::PaymentChannelClaim; payChanClaim[jss::Flags] = tfClose; payChanClaim[jss::Account] = gw.human(); - payChanClaim[sfPayChannel.jsonName] = payChanIndex; + payChanClaim[sfChannel.jsonName] = payChanIndex; payChanClaim[sfPublicKey.jsonName] = strHex(alice.pk().slice()); env(payChanClaim); env.close(); diff --git a/src/test/app/LedgerHistory_test.cpp b/src/test/app/LedgerHistory_test.cpp index 513905a6bf7..ba4faa9da05 100644 --- a/src/test/app/LedgerHistory_test.cpp +++ b/src/test/app/LedgerHistory_test.cpp @@ -72,8 +72,8 @@ class LedgerHistory_test : public beast::unit_test::suite res->updateSkipList(); { - res->stateMap().flushDirty(hotACCOUNT_NODE, res->info().seq); - res->txMap().flushDirty(hotTRANSACTION_NODE, res->info().seq); + res->stateMap().flushDirty(hotACCOUNT_NODE); + res->txMap().flushDirty(hotTRANSACTION_NODE); } res->unshare(); diff --git a/src/test/app/Manifest_test.cpp b/src/test/app/Manifest_test.cpp index 18460ce3689..8dbc92ed78a 100644 --- a/src/test/app/Manifest_test.cpp +++ b/src/test/app/Manifest_test.cpp @@ -602,14 +602,29 @@ class Manifest_test : public beast::unit_test::suite std::uint32_t sequence = 0; // public key with invalid type - auto const ret = strUnHex( - "9930E7FC9D56BB25D6893BA3F317AE5BCF33B3291BD63DB32654A313222F7FD02" - "0"); - auto const badKey = Slice{ret->data(), ret->size()}; - - // short public key - auto const retShort = strUnHex("0330"); - auto const shortKey = Slice{retShort->data(), retShort->size()}; + std::array const badKey{ + 0x99, 0x30, 0xE7, 0xFC, 0x9D, 0x56, 0xBB, 0x25, 0xD6, 0x89, 0x3B, + 0xA3, 0xF3, 0x17, 0xAE, 0x5B, 0xCF, 0x33, 0xB3, 0x29, 0x1B, 0xD6, + 0x3D, 0xB3, 0x26, 0x54, 0xA3, 0x13, 0x22, 0x2F, 0x7F, 0xD0, 0x20}; + + // Short public key: + std::array const shortKey{ + 0x03, + 0x30, + 0xE7, + 0xFC, + 0x9D, + 0x56, + 0xBB, + 0x25, + 0xD6, + 0x89, + 0x3B, + 0xA3, + 0xF3, + 0x17, + 0xAE, + 0x5B}; auto toString = [](STObject const& st) { Serializer s; @@ -749,13 +764,13 @@ class Manifest_test : public beast::unit_test::suite { // reject invalid public key type auto badSt = st; - badSt[sfPublicKey] = badKey; + badSt[sfPublicKey] = makeSlice(badKey); BEAST_EXPECT(!deserializeManifest(toString(badSt))); } { // reject short public key auto badSt = st; - badSt[sfPublicKey] = shortKey; + badSt[sfPublicKey] = makeSlice(shortKey); BEAST_EXPECT(!deserializeManifest(toString(badSt))); } { @@ -767,13 +782,13 @@ class Manifest_test : public beast::unit_test::suite { // reject invalid signing public key type auto badSt = st; - badSt[sfSigningPubKey] = badKey; + badSt[sfSigningPubKey] = makeSlice(badKey); BEAST_EXPECT(!deserializeManifest(toString(badSt))); } { // reject short signing public key auto badSt = st; - badSt[sfSigningPubKey] = shortKey; + badSt[sfSigningPubKey] = makeSlice(shortKey); BEAST_EXPECT(!deserializeManifest(toString(badSt))); } { diff --git a/src/test/app/Path_test.cpp b/src/test/app/Path_test.cpp index 17e15c95043..1889a790e2f 100644 --- a/src/test/app/Path_test.cpp +++ b/src/test/app/Path_test.cpp @@ -802,7 +802,7 @@ class Path_test : public beast::unit_test::suite "issuer" : "rPMh7Pi9ct699iZUTWaytJUoHcJ7cgyziK", "value" : "1000" }, - "HighNode" : "0000000000000000", + "HighNode" : "0", "HighQualityIn" : 2000, "HighQualityOut" : 1400000000, "LedgerEntryType" : "RippleState", @@ -811,7 +811,7 @@ class Path_test : public beast::unit_test::suite "issuer" : "rG1QQv2nh2gr7RCZ1P8YYcBUKCCN633jCn", "value" : "0" }, - "LowNode" : "0000000000000000" + "LowNode" : "0" })", jv); @@ -847,14 +847,14 @@ class Path_test : public beast::unit_test::suite "issuer" : "rPMh7Pi9ct699iZUTWaytJUoHcJ7cgyziK", "value" : "1000" }, - "HighNode" : "0000000000000000", + "HighNode" : "0", "LedgerEntryType" : "RippleState", "LowLimit" : { "currency" : "USD", "issuer" : "rG1QQv2nh2gr7RCZ1P8YYcBUKCCN633jCn", "value" : "1000" }, - "LowNode" : "0000000000000000" + "LowNode" : "0" })", jv); @@ -900,7 +900,7 @@ class Path_test : public beast::unit_test::suite "issuer" : "rPMh7Pi9ct699iZUTWaytJUoHcJ7cgyziK", "value" : "0" }, - "HighNode" : "0000000000000000", + "HighNode" : "0", "LedgerEntryType" : "RippleState", "LowLimit" : { @@ -908,7 +908,7 @@ class Path_test : public beast::unit_test::suite "issuer" : "rG1QQv2nh2gr7RCZ1P8YYcBUKCCN633jCn", "value" : "0" }, - "LowNode" : "0000000000000000" + "LowNode" : "0" })", jv); diff --git a/src/test/app/TrustAndBalance_test.cpp b/src/test/app/TrustAndBalance_test.cpp index 5e07f1110d8..72ab5eb43ea 100644 --- a/src/test/app/TrustAndBalance_test.cpp +++ b/src/test/app/TrustAndBalance_test.cpp @@ -454,30 +454,24 @@ class TrustAndBalance_test : public beast::unit_test::suite jvs[jss::streams].append("transactions"); BEAST_EXPECT(wsc->invoke("subscribe", jvs)[jss::status] == "success"); + char const* invoiceid = + "243F6A8885A308D313198A2E03707344A4093822299F31D0082EFA98EC4E6C89"; + Json::Value jv; auto tx = env.jt( pay(env.master, alice, XRP(10000)), - json(sfInvoiceID.fieldName, "DEADBEEF")); + json(sfInvoiceID.fieldName, invoiceid)); jv[jss::tx_blob] = strHex(tx.stx->getSerializer().slice()); auto jrr = wsc->invoke("submit", jv)[jss::result]; BEAST_EXPECT(jrr[jss::status] == "success"); - BEAST_EXPECT( - jrr[jss::tx_json][sfInvoiceID.fieldName] == - "0000000000000000" - "0000000000000000" - "0000000000000000" - "00000000DEADBEEF"); + BEAST_EXPECT(jrr[jss::tx_json][sfInvoiceID.fieldName] == invoiceid); env.close(); using namespace std::chrono_literals; - BEAST_EXPECT(wsc->findMsg(2s, [](auto const& jval) { + BEAST_EXPECT(wsc->findMsg(2s, [invoiceid](auto const& jval) { auto const& t = jval[jss::transaction]; return t[jss::TransactionType] == jss::Payment && - t[sfInvoiceID.fieldName] == - "0000000000000000" - "0000000000000000" - "0000000000000000" - "00000000DEADBEEF"; + t[sfInvoiceID.fieldName] == invoiceid; })); BEAST_EXPECT(wsc->invoke("unsubscribe", jv)[jss::status] == "success"); diff --git a/src/test/basics/base_uint_test.cpp b/src/test/basics/base_uint_test.cpp index ab327bc28bf..2ac8bbe8f6d 100644 --- a/src/test/basics/base_uint_test.cpp +++ b/src/test/basics/base_uint_test.cpp @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include @@ -34,9 +34,10 @@ namespace test { template struct nonhash { + static constexpr auto const endian = boost::endian::order::big; static constexpr std::size_t WIDTH = Bits / 8; + std::array data_; - static beast::endian const endian = beast::endian::big; nonhash() = default; @@ -62,6 +63,8 @@ struct base_uint_test : beast::unit_test::suite void run() override { + testcase("base_uint: general purpose tests"); + static_assert( !std::is_constructible>::value, ""); static_assert( @@ -151,59 +154,47 @@ struct base_uint_test : beast::unit_test::suite BEAST_EXPECT(uset.size() == 4); - // SetHex tests... - test96 fromHex; - BEAST_EXPECT(fromHex.SetHexExact(to_string(u))); - BEAST_EXPECT(fromHex == u); - fromHex = z; + test96 tmp; + BEAST_EXPECT(tmp.parseHex(to_string(u))); + BEAST_EXPECT(tmp == u); + tmp = z; // fails with extra char - BEAST_EXPECT(!fromHex.SetHexExact("A" + to_string(u))); - fromHex = z; + BEAST_EXPECT(!tmp.parseHex("A" + to_string(u))); + tmp = z; // fails with extra char at end - BEAST_EXPECT(!fromHex.SetHexExact(to_string(u) + "A")); - // NOTE: the value fromHex is actually correctly parsed - // in this case, but that is an implementation detail and - // not guaranteed, thus we don't check the value here. - fromHex = z; - - BEAST_EXPECT(fromHex.SetHex(to_string(u))); - BEAST_EXPECT(fromHex == u); - fromHex = z; - - // leading space/0x allowed if not strict - BEAST_EXPECT(fromHex.SetHex(" 0x" + to_string(u))); - BEAST_EXPECT(fromHex == u); - fromHex = z; - - // other leading chars also allowed (ignored) - BEAST_EXPECT(fromHex.SetHex("FEFEFE" + to_string(u))); - BEAST_EXPECT(fromHex == u); - fromHex = z; - - // invalid hex chars should fail (0 replaced with Z here) - BEAST_EXPECT(!fromHex.SetHex( - boost::algorithm::replace_all_copy(to_string(u), "0", "Z"))); - fromHex = z; - - BEAST_EXPECT(fromHex.SetHex(to_string(u), true)); - BEAST_EXPECT(fromHex == u); - fromHex = z; - - // strict mode fails with leading chars - BEAST_EXPECT(!fromHex.SetHex(" 0x" + to_string(u), true)); - fromHex = z; - - // SetHex ignores extra leading hexits, so the parsed value - // is still correct for the following case (strict or non-strict) - BEAST_EXPECT(fromHex.SetHex("DEAD" + to_string(u), true)); - BEAST_EXPECT(fromHex == u); - fromHex = z; - - BEAST_EXPECT(fromHex.SetHex("DEAD" + to_string(u), false)); - BEAST_EXPECT(fromHex == u); - fromHex = z; + BEAST_EXPECT(!tmp.parseHex(to_string(u) + "A")); + + // fails with a non-hex character at some point in the string: + tmp = z; + + for (std::size_t i = 0; i != 24; ++i) + { + std::string x = to_string(z); + x[i] = ('G' + (i % 10)); + BEAST_EXPECT(!tmp.parseHex(x)); + } + + // Walking 1s: + for (std::size_t i = 0; i != 24; ++i) + { + std::string s1 = "000000000000000000000000"; + s1[i] = '1'; + + BEAST_EXPECT(tmp.parseHex(s1)); + BEAST_EXPECT(to_string(tmp) == s1); + } + + // Walking 0s: + for (std::size_t i = 0; i != 24; ++i) + { + std::string s1 = "111111111111111111111111"; + s1[i] = '0'; + + BEAST_EXPECT(tmp.parseHex(s1)); + BEAST_EXPECT(to_string(tmp) == s1); + } } }; diff --git a/src/test/conditions/PreimageSha256_test.cpp b/src/test/conditions/PreimageSha256_test.cpp index d9df96226cc..cf4bc8c7c1e 100644 --- a/src/test/conditions/PreimageSha256_test.cpp +++ b/src/test/conditions/PreimageSha256_test.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -37,29 +38,9 @@ class PreimageSha256_test : public beast::unit_test::suite inline Buffer hexblob(std::string const& s) { - std::vector x; - x.reserve(s.size() / 2); - - auto iter = s.cbegin(); - - while (iter != s.cend()) - { - int cHigh = charUnHex(*iter++); - - if (cHigh < 0) - return {}; - - int cLow = charUnHex(*iter++); - - if (cLow < 0) - return {}; - - x.push_back( - static_cast(cHigh << 4) | - static_cast(cLow)); - } - - return {x.data(), x.size()}; + auto blob = strUnHex(s); + BEAST_EXPECT(blob); + return {blob->data(), blob->size()}; } void diff --git a/src/test/core/Config_test.cpp b/src/test/core/Config_test.cpp index 03282fd59de..b7a8a18355d 100644 --- a/src/test/core/Config_test.cpp +++ b/src/test/core/Config_test.cpp @@ -1065,6 +1065,80 @@ r.ripple.com 51235 } } + void + testOverlay() + { + testcase("overlay: unknown time"); + + auto testUnknown = + [](std::string value) -> std::optional { + try + { + Config c; + c.loadFromString("[overlay]\nmax_unknown_time=" + value); + return c.MAX_UNKNOWN_TIME; + } + catch (std::runtime_error&) + { + return {}; + } + }; + + // Failures + BEAST_EXPECT(!testUnknown("none")); + BEAST_EXPECT(!testUnknown("0.5")); + BEAST_EXPECT(!testUnknown("180 seconds")); + BEAST_EXPECT(!testUnknown("9 minutes")); + + // Below lower bound + BEAST_EXPECT(!testUnknown("299")); + + // In bounds + BEAST_EXPECT(testUnknown("300") == std::chrono::seconds{300}); + BEAST_EXPECT(testUnknown("301") == std::chrono::seconds{301}); + BEAST_EXPECT(testUnknown("1799") == std::chrono::seconds{1799}); + BEAST_EXPECT(testUnknown("1800") == std::chrono::seconds{1800}); + + // Above upper bound + BEAST_EXPECT(!testUnknown("1801")); + + testcase("overlay: diverged time"); + + // In bounds: + auto testDiverged = + [](std::string value) -> std::optional { + try + { + Config c; + c.loadFromString("[overlay]\nmax_diverged_time=" + value); + return c.MAX_DIVERGED_TIME; + } + catch (std::runtime_error&) + { + return {}; + } + }; + + // Failures + BEAST_EXPECT(!testDiverged("none")); + BEAST_EXPECT(!testDiverged("0.5")); + BEAST_EXPECT(!testDiverged("180 seconds")); + BEAST_EXPECT(!testDiverged("9 minutes")); + + // Below lower bound + BEAST_EXPECT(!testDiverged("0")); + BEAST_EXPECT(!testDiverged("59")); + + // In bounds + BEAST_EXPECT(testDiverged("60") == std::chrono::seconds{60}); + BEAST_EXPECT(testDiverged("61") == std::chrono::seconds{61}); + BEAST_EXPECT(testDiverged("899") == std::chrono::seconds{899}); + BEAST_EXPECT(testDiverged("900") == std::chrono::seconds{900}); + + // Above upper bound + BEAST_EXPECT(!testDiverged("901")); + } + void run() override { @@ -1079,6 +1153,7 @@ r.ripple.com 51235 testComments(); testGetters(); testAmendment(); + testOverlay(); } }; diff --git a/src/test/csf/Peer.h b/src/test/csf/Peer.h index a9454a1a110..fe99de7e535 100644 --- a/src/test/csf/Peer.h +++ b/src/test/csf/Peer.h @@ -707,7 +707,7 @@ struct Peer // Basic Sequence number router // A message that will be flooded across the network is tagged with a - // seqeuence number by the origin node in a BroadcastMesg. Receivers will + // sequence number by the origin node in a BroadcastMesg. Receivers will // ignore a message as stale if they've already processed a newer sequence // number, or will process and potentially relay the message along. // diff --git a/src/test/jtx/Env.h b/src/test/jtx/Env.h index f2934bb5002..57b68b11879 100644 --- a/src/test/jtx/Env.h +++ b/src/test/jtx/Env.h @@ -26,7 +26,6 @@ #include #include #include -#include // #include #include #include @@ -50,6 +49,7 @@ #include #include #include +#include #include #include #include diff --git a/src/test/jtx/impl/Env.cpp b/src/test/jtx/impl/Env.cpp index 855dfe7bbf0..a8c5aca49b3 100644 --- a/src/test/jtx/impl/Env.cpp +++ b/src/test/jtx/impl/Env.cpp @@ -327,7 +327,9 @@ Env::sign_and_submit(JTx const& jt, Json::Value params) params[jss::tx_json] = jt.jv; jr = client().invoke("submit", params); } - txid_.SetHex(jr[jss::result][jss::tx_json][jss::hash].asString()); + + if (!txid_.parseHex(jr[jss::result][jss::tx_json][jss::hash].asString())) + txid_.zero(); std::tie(ter_, didApply) = parseResult(jr); diff --git a/src/test/ledger/Directory_test.cpp b/src/test/ledger/Directory_test.cpp index 39d319ad9b2..183e82d9a70 100644 --- a/src/test/ledger/Directory_test.cpp +++ b/src/test/ledger/Directory_test.cpp @@ -326,11 +326,11 @@ struct Directory_test : public beast::unit_test::suite env.close(); uint256 base; - base.SetHex( + (void)base.parseHex( "fb71c9aa3310141da4b01d6c744a98286af2d72ab5448d5adc0910ca0c910880"); uint256 item; - item.SetHex( + (void)item.parseHex( "bad0f021aa3b2f6754a8fe82a5779730aa0bbbab82f17201ef24900efc2c7312"); { diff --git a/src/test/nodestore/DatabaseShard_test.cpp b/src/test/nodestore/DatabaseShard_test.cpp index c53568e6d87..2dcc4c67a15 100644 --- a/src/test/nodestore/DatabaseShard_test.cpp +++ b/src/test/nodestore/DatabaseShard_test.cpp @@ -283,15 +283,14 @@ class DatabaseShard_test : public TestBase } // Store the state map - auto visitAcc = [&](SHAMapAbstractNode& node) { + auto visitAcc = [&](SHAMapTreeNode const& node) { Serializer s; - node.addRaw(s, snfPREFIX); + node.serializeWithPrefix(s); db.store( - node.getType() == SHAMapAbstractNode::TNType::tnINNER - ? hotUNKNOWN - : hotACCOUNT_NODE, + node.getType() == SHAMapNodeType::tnINNER ? hotUNKNOWN + : hotACCOUNT_NODE, std::move(s.modData()), - node.getNodeHash().as_uint256(), + node.getHash().as_uint256(), ledger.info().seq); return true; }; @@ -311,15 +310,14 @@ class DatabaseShard_test : public TestBase } // Store the transaction map - auto visitTx = [&](SHAMapAbstractNode& node) { + auto visitTx = [&](SHAMapTreeNode& node) { Serializer s; - node.addRaw(s, snfPREFIX); + node.serializeWithPrefix(s); db.store( - node.getType() == SHAMapAbstractNode::TNType::tnINNER - ? hotUNKNOWN - : hotTRANSACTION_NODE, + node.getType() == SHAMapNodeType::tnINNER ? hotUNKNOWN + : hotTRANSACTION_NODE, std::move(s.modData()), - node.getNodeHash().as_uint256(), + node.getHash().as_uint256(), ledger.info().seq); return true; }; @@ -357,20 +355,19 @@ class DatabaseShard_test : public TestBase LedgerFill{*fetched, LedgerFill::full | LedgerFill::binary})); // walk shamap and validate each node - auto fcompAcc = [&](SHAMapAbstractNode& node) -> bool { + auto fcompAcc = [&](SHAMapTreeNode& node) -> bool { Serializer s; - node.addRaw(s, snfPREFIX); + node.serializeWithPrefix(s); auto nSrc{NodeObject::createObject( - node.getType() == SHAMapAbstractNode::TNType::tnINNER - ? hotUNKNOWN - : hotACCOUNT_NODE, + node.getType() == SHAMapNodeType::tnINNER ? hotUNKNOWN + : hotACCOUNT_NODE, std::move(s.modData()), - node.getNodeHash().as_uint256())}; + node.getHash().as_uint256())}; if (!BEAST_EXPECT(nSrc)) return false; auto nDst = db.fetchNodeObject( - node.getNodeHash().as_uint256(), ledger.info().seq); + node.getHash().as_uint256(), ledger.info().seq); if (!BEAST_EXPECT(nDst)) return false; @@ -381,20 +378,19 @@ class DatabaseShard_test : public TestBase if (ledger.stateMap().getHash().isNonZero()) ledger.stateMap().snapShot(false)->visitNodes(fcompAcc); - auto fcompTx = [&](SHAMapAbstractNode& node) -> bool { + auto fcompTx = [&](SHAMapTreeNode& node) -> bool { Serializer s; - node.addRaw(s, snfPREFIX); + node.serializeWithPrefix(s); auto nSrc{NodeObject::createObject( - node.getType() == SHAMapAbstractNode::TNType::tnINNER - ? hotUNKNOWN - : hotTRANSACTION_NODE, + node.getType() == SHAMapNodeType::tnINNER ? hotUNKNOWN + : hotTRANSACTION_NODE, std::move(s.modData()), - node.getNodeHash().as_uint256())}; + node.getHash().as_uint256())}; if (!BEAST_EXPECT(nSrc)) return false; auto nDst = db.fetchNodeObject( - node.getNodeHash().as_uint256(), ledger.info().seq); + node.getHash().as_uint256(), ledger.info().seq); if (!BEAST_EXPECT(nDst)) return false; diff --git a/src/test/overlay/compression_test.cpp b/src/test/overlay/compression_test.cpp index 965fc7d6d14..041659b3722 100644 --- a/src/test/overlay/compression_test.cpp +++ b/src/test/overlay/compression_test.cpp @@ -203,18 +203,11 @@ class compression_test : public beast::unit_test::suite env.trust(bob["USD"](fund), alice); env.close(); - auto toBinary = [](std::string const& text) { - std::string binary; - for (size_t i = 0; i < text.size(); ++i) - { - unsigned int c = charUnHex(text[i]); - c = c << 4; - ++i; - c = c | charUnHex(text[i]); - binary.push_back(c); - } - - return binary; + auto toBinary = [this](std::string const& text) { + auto blob = strUnHex(text); + BEAST_EXPECT(blob); + return std::string{ + reinterpret_cast(blob->data()), blob->size()}; }; std::string usdTxBlob = ""; @@ -249,7 +242,7 @@ class compression_test : public beast::unit_test::suite uint256 const hash(ripple::sha512Half(123456789)); getLedger->set_ledgerhash(hash.begin(), hash.size()); getLedger->set_ledgerseq(123456789); - ripple::SHAMapNodeID sha(hash.data(), hash.size()); + ripple::SHAMapNodeID sha(17, hash); getLedger->add_nodeids(sha.getRawString()); getLedger->set_requestcookie(123456789); getLedger->set_querytype(protocol::qtINDIRECT); @@ -309,7 +302,7 @@ class compression_test : public beast::unit_test::suite uint256 hash(ripple::sha512Half(i)); auto object = getObject->add_objects(); object->set_hash(hash.data(), hash.size()); - ripple::SHAMapNodeID sha(hash.data(), hash.size()); + ripple::SHAMapNodeID sha(i % 55, hash); object->set_nodeid(sha.getRawString()); object->set_index(""); object->set_data(""); diff --git a/src/test/peerfinder/PeerFinder_test.cpp b/src/test/peerfinder/PeerFinder_test.cpp index b57911d28b6..b0750e689f3 100644 --- a/src/test/peerfinder/PeerFinder_test.cpp +++ b/src/test/peerfinder/PeerFinder_test.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -156,11 +157,130 @@ class PeerFinder_test : public beast::unit_test::suite BEAST_EXPECT(n <= (seconds + 59) / 60); } + void + test_config() + { + // if peers_max is configured then peers_in_max and peers_out_max are + // ignored + auto run = [&](std::string const& test, + std::optional maxPeers, + std::optional maxIn, + std::optional maxOut, + std::uint16_t port, + std::uint16_t expectOut, + std::uint16_t expectIn, + std::uint16_t expectIpLimit) { + ripple::Config c; + + testcase(test); + + std::string toLoad = ""; + int max = 0; + if (maxPeers) + { + max = maxPeers.value(); + toLoad += "[peers_max]\n" + std::to_string(max) + "\n" + + "[peers_in_max]\n" + std::to_string(maxIn.value_or(0)) + + "\n" + "[peers_out_max]\n" + + std::to_string(maxOut.value_or(0)) + "\n"; + } + else if (maxIn && maxOut) + { + toLoad += "[peers_in_max]\n" + std::to_string(*maxIn) + "\n" + + "[peers_out_max]\n" + std::to_string(*maxOut) + "\n"; + } + + c.loadFromString(toLoad); + BEAST_EXPECT( + (c.PEERS_MAX == max && c.PEERS_IN_MAX == 0 && + c.PEERS_OUT_MAX == 0) || + (c.PEERS_IN_MAX == *maxIn && c.PEERS_OUT_MAX == *maxOut)); + + Config config = Config::makeConfig(c, port, false, 0); + + Counts counts; + counts.onConfig(config); + BEAST_EXPECT( + counts.out_max() == expectOut && + counts.inboundSlots() == expectIn && + config.ipLimit == expectIpLimit); + }; + + // if max_peers == 0 => maxPeers = 21, + // else if max_peers < 10 => maxPeers = 10 else maxPeers = max_peers + // expectOut => if legacy => max(0.15 * maxPeers, 10), + // if legacy && !wantIncoming => maxPeers else max_out_peers + // expectIn => if legacy && wantIncoming => maxPeers - outPeers + // else if !wantIncoming => 0 else max_in_peers + // ipLimit => if expectIn <= 21 => 2 else 2 + min(5, expectIn/21) + // ipLimit = max(1, min(ipLimit, expectIn/2)) + + // legacy test with max_peers + run("legacy no config", {}, {}, {}, 4000, 10, 11, 2); + run("legacy max_peers 0", 0, 100, 10, 4000, 10, 11, 2); + run("legacy max_peers 5", 5, 100, 10, 4000, 10, 0, 1); + run("legacy max_peers 20", 20, 100, 10, 4000, 10, 10, 2); + run("legacy max_peers 100", 100, 100, 10, 4000, 15, 85, 6); + run("legacy max_peers 20, private", 20, 100, 10, 0, 20, 0, 1); + + // test with max_in_peers and max_out_peers + run("new in 100/out 10", {}, 100, 10, 4000, 10, 100, 6); + run("new in 0/out 10", {}, 0, 10, 4000, 10, 0, 1); + run("new in 100/out 10, private", {}, 100, 10, 0, 10, 0, 6); + } + + void + test_invalid_config() + { + testcase("invalid config"); + + auto run = [&](std::string const& toLoad) { + ripple::Config c; + try + { + c.loadFromString(toLoad); + fail(); + } + catch (...) + { + pass(); + } + }; + run(R"rippleConfig( +[peers_in_max] +100 +)rippleConfig"); + run(R"rippleConfig( +[peers_out_max] +100 +)rippleConfig"); + run(R"rippleConfig( +[peers_in_max] +100 +[peers_out_max] +5 +)rippleConfig"); + run(R"rippleConfig( +[peers_in_max] +1001 +[peers_out_max] +10 +)rippleConfig"); + run(R"rippleConfig( +[peers_in_max] +10 +[peers_out_max] +1001 +)rippleConfig"); + } + void run() override { test_backoff1(); test_backoff2(); + test_config(); + test_invalid_config(); } }; diff --git a/src/test/protocol/STAmount_test.cpp b/src/test/protocol/STAmount_test.cpp index 10c4e1d4abe..bd3e96694fd 100644 --- a/src/test/protocol/STAmount_test.cpp +++ b/src/test/protocol/STAmount_test.cpp @@ -306,9 +306,6 @@ class STAmount_test : public beast::unit_test::suite const std::string cur = "015841551A748AD2C1F76FF6ECB0CCCD00000000"; unexpected(!to_currency(c, cur), "create custom currency"); unexpected(to_string(c) != cur, "check custom currency"); - unexpected( - c != Currency(from_hex_text(cur)), - "check custom currency"); } //-------------------------------------------------------------------------- diff --git a/src/test/protocol/digest_test.cpp b/src/test/protocol/digest_test.cpp deleted file mode 100644 index 7c82b5e7682..00000000000 --- a/src/test/protocol/digest_test.cpp +++ /dev/null @@ -1,160 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { - -class digest_test : public beast::unit_test::suite -{ - std::vector dataset1; - - template - void - test(char const* name) - { - using namespace std::chrono; - - // Prime the cache - for (int i = 0; i != 4; i++) - { - for (auto const& x : dataset1) - { - Hasher h; - h(x.data(), x.size()); - (void)static_cast(h); - } - } - - std::array results; - - for (auto& result : results) - { - auto const start = high_resolution_clock::now(); - - for (auto const& x : dataset1) - { - Hasher h; - h(x.data(), x.size()); - (void)static_cast(h); - } - - auto const d = high_resolution_clock::now() - start; - - result = d; - } - - log << " " << name << ":" << '\n'; - - auto const sum = - std::accumulate(results.begin(), results.end(), nanoseconds{0}); - { - auto s = duration_cast(sum); - auto ms = duration_cast(sum) - s; - log << " Total Time = " << s.count() << "." << ms.count() - << " seconds" << std::endl; - } - - auto const mean = sum / results.size(); - { - auto s = duration_cast(mean); - auto ms = duration_cast(mean) - s; - log << " Mean Time = " << s.count() << "." << ms.count() - << " seconds" << std::endl; - } - - std::vector diff(results.size()); - std::transform( - results.begin(), - results.end(), - diff.begin(), - [&mean](nanoseconds trial) { return (trial - mean).count(); }); - auto const sq_sum = - std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0); - { - nanoseconds const stddev{static_cast( - std::sqrt(sq_sum / results.size()))}; - auto s = duration_cast(stddev); - auto ms = duration_cast(stddev) - s; - log << " Std Dev = " << s.count() << "." << ms.count() - << " seconds" << std::endl; - } - } - -public: - digest_test() - { - beast::xor_shift_engine g(19207813); - std::array buf; - - for (int i = 0; i < 1000000; i++) - { - beast::rngfill(buf.data(), buf.size(), g); - dataset1.push_back(uint256{buf}); - } - } - - void - testSHA512() - { - testcase("SHA512"); - test("OpenSSL"); - test("Beast"); - pass(); - } - - void - testSHA256() - { - testcase("SHA256"); - test("OpenSSL"); - test("Beast"); - pass(); - } - - void - testRIPEMD160() - { - testcase("RIPEMD160"); - test("OpenSSL"); - test("Beast"); - pass(); - } - - void - run() override - { - testSHA512(); - testSHA256(); - testRIPEMD160(); - } -}; - -BEAST_DEFINE_TESTSUITE_MANUAL_PRIO(digest, ripple_data, ripple, 20); - -} // namespace ripple diff --git a/src/test/rpc/AccountObjects_test.cpp b/src/test/rpc/AccountObjects_test.cpp index 8f77b6ce9da..70c70e69775 100644 --- a/src/test/rpc/AccountObjects_test.cpp +++ b/src/test/rpc/AccountObjects_test.cpp @@ -34,10 +34,10 @@ static char const* bobs_account_objects[] = { R"json({ "Account" : "rPMh7Pi9ct699iZUTWaytJUoHcJ7cgyziK", "BookDirectory" : "50AD0A9E54D2B381288D535EB724E4275FFBF41580D28A925D038D7EA4C68000", - "BookNode" : "0000000000000000", + "BookNode" : "0", "Flags" : 65536, "LedgerEntryType" : "Offer", - "OwnerNode" : "0000000000000000", + "OwnerNode" : "0", "Sequence" : 6, "TakerGets" : { "currency" : "USD", @@ -59,14 +59,14 @@ static char const* bobs_account_objects[] = { "issuer" : "rPMh7Pi9ct699iZUTWaytJUoHcJ7cgyziK", "value" : "1000" }, - "HighNode" : "0000000000000000", + "HighNode" : "0", "LedgerEntryType" : "RippleState", "LowLimit" : { "currency" : "USD", "issuer" : "r9cZvwKU3zzuZK9JFovGg1JC5n7QiqNL8L", "value" : "0" }, - "LowNode" : "0000000000000000", + "LowNode" : "0", "index" : "D13183BCFFC9AAC9F96AEBB5F66E4A652AD1F5D10273AEB615478302BEBFD4A4" })json", R"json({ @@ -81,23 +81,23 @@ static char const* bobs_account_objects[] = { "issuer" : "rPMh7Pi9ct699iZUTWaytJUoHcJ7cgyziK", "value" : "1000" }, - "HighNode" : "0000000000000000", + "HighNode" : "0", "LedgerEntryType" : "RippleState", "LowLimit" : { "currency" : "USD", "issuer" : "r32rQHyesiTtdWFU7UJVtff4nCR5SHCbJW", "value" : "0" }, - "LowNode" : "0000000000000000", + "LowNode" : "0", "index" : "D89BC239086183EB9458C396E643795C1134963E6550E682A190A5F021766D43" })json", R"json({ "Account" : "rPMh7Pi9ct699iZUTWaytJUoHcJ7cgyziK", "BookDirectory" : "B025997A323F5C3E03DDF1334471F5984ABDE31C59D463525D038D7EA4C68000", - "BookNode" : "0000000000000000", + "BookNode" : "0", "Flags" : 65536, "LedgerEntryType" : "Offer", - "OwnerNode" : "0000000000000000", + "OwnerNode" : "0", "Sequence" : 7, "TakerGets" : { "currency" : "USD", @@ -287,9 +287,6 @@ class AccountObjects_test : public beast::unit_test::suite auto& aobj = resp[jss::result][jss::account_objects][i]; aobj.removeMember("PreviousTxnID"); aobj.removeMember("PreviousTxnLgrSeq"); - - if (aobj != bobj[i]) - std::cout << "Fail at " << i << ": " << aobj << std::endl; BEAST_EXPECT(aobj == bobj[i]); } } diff --git a/src/test/rpc/AccountSet_test.cpp b/src/test/rpc/AccountSet_test.cpp index b9745f3ecdd..a3fe45e774b 100644 --- a/src/test/rpc/AccountSet_test.cpp +++ b/src/test/rpc/AccountSet_test.cpp @@ -258,11 +258,11 @@ class AccountSet_test : public beast::unit_test::suite env.fund(XRP(10000), alice); auto jt = noop(alice); - uint256 somehash = from_hex_text( - "9633ec8af54f16b5286db1d7b519ef49eefc050c0c8ac4384f1d88acd1bfdf05"); - jt[sfWalletLocator.fieldName] = to_string(somehash); + std::string const locator = + "9633EC8AF54F16B5286DB1D7B519EF49EEFC050C0C8AC4384F1D88ACD1BFDF05"; + jt[sfWalletLocator.fieldName] = locator; env(jt); - BEAST_EXPECT((*env.le(alice))[sfWalletLocator] == somehash); + BEAST_EXPECT(to_string((*env.le(alice))[sfWalletLocator]) == locator); jt[sfWalletLocator.fieldName] = ""; env(jt); @@ -280,11 +280,10 @@ class AccountSet_test : public beast::unit_test::suite env.fund(XRP(10000), alice); auto jt = noop(alice); - uint128 somehash = - from_hex_text("fff680681c2f5e6095324e2e08838f221a72ab4f"); - jt[sfEmailHash.fieldName] = to_string(somehash); + std::string const mh("5F31A79367DC3137FADA860C05742EE6"); + jt[sfEmailHash.fieldName] = mh; env(jt); - BEAST_EXPECT((*env.le(alice))[sfEmailHash] == somehash); + BEAST_EXPECT(to_string((*env.le(alice))[sfEmailHash]) == mh); jt[sfEmailHash.fieldName] = ""; env(jt); diff --git a/src/test/rpc/AccountTx_test.cpp b/src/test/rpc/AccountTx_test.cpp index c6c168c60e8..dbcd791ff5b 100644 --- a/src/test/rpc/AccountTx_test.cpp +++ b/src/test/rpc/AccountTx_test.cpp @@ -368,7 +368,7 @@ class AccountTx_test : public beast::unit_test::suite payChanFund[jss::TransactionType] = jss::PaymentChannelFund; payChanFund[jss::Flags] = tfUniversal; payChanFund[jss::Account] = alice.human(); - payChanFund[sfPayChannel.jsonName] = payChanIndex; + payChanFund[sfChannel.jsonName] = payChanIndex; payChanFund[jss::Amount] = XRP(200).value().getJson(JsonOptions::none); env(payChanFund, sig(alie)); @@ -379,7 +379,7 @@ class AccountTx_test : public beast::unit_test::suite payChanClaim[jss::TransactionType] = jss::PaymentChannelClaim; payChanClaim[jss::Flags] = tfClose; payChanClaim[jss::Account] = gw.human(); - payChanClaim[sfPayChannel.jsonName] = payChanIndex; + payChanClaim[sfChannel.jsonName] = payChanIndex; payChanClaim[sfPublicKey.jsonName] = strHex(alice.pk().slice()); env(payChanClaim); env.close(); diff --git a/src/test/rpc/Book_test.cpp b/src/test/rpc/Book_test.cpp index 548f0b6eb8f..a2ebc21cb92 100644 --- a/src/test/rpc/Book_test.cpp +++ b/src/test/rpc/Book_test.cpp @@ -1054,10 +1054,10 @@ class Book_test : public beast::unit_test::suite BEAST_EXPECT( jrOffer[sfBookDirectory.fieldName] == getBookDir(env, XRP, USD.issue())); - BEAST_EXPECT(jrOffer[sfBookNode.fieldName] == "0000000000000000"); + BEAST_EXPECT(jrOffer[sfBookNode.fieldName] == "0"); BEAST_EXPECT(jrOffer[jss::Flags] == 0); BEAST_EXPECT(jrOffer[sfLedgerEntryType.fieldName] == jss::Offer); - BEAST_EXPECT(jrOffer[sfOwnerNode.fieldName] == "0000000000000000"); + BEAST_EXPECT(jrOffer[sfOwnerNode.fieldName] == "0"); BEAST_EXPECT(jrOffer[sfSequence.fieldName] == 5); BEAST_EXPECT( jrOffer[jss::TakerGets] == @@ -1110,10 +1110,10 @@ class Book_test : public beast::unit_test::suite BEAST_EXPECT( jrNextOffer[sfBookDirectory.fieldName] == getBookDir(env, XRP, USD.issue())); - BEAST_EXPECT(jrNextOffer[sfBookNode.fieldName] == "0000000000000000"); + BEAST_EXPECT(jrNextOffer[sfBookNode.fieldName] == "0"); BEAST_EXPECT(jrNextOffer[jss::Flags] == 0); BEAST_EXPECT(jrNextOffer[sfLedgerEntryType.fieldName] == jss::Offer); - BEAST_EXPECT(jrNextOffer[sfOwnerNode.fieldName] == "0000000000000000"); + BEAST_EXPECT(jrNextOffer[sfOwnerNode.fieldName] == "0"); BEAST_EXPECT(jrNextOffer[sfSequence.fieldName] == 5); BEAST_EXPECT( jrNextOffer[jss::TakerGets] == diff --git a/src/test/rpc/Feature_test.cpp b/src/test/rpc/Feature_test.cpp index 5773f756667..89b97639523 100644 --- a/src/test/rpc/Feature_test.cpp +++ b/src/test/rpc/Feature_test.cpp @@ -129,7 +129,7 @@ class Feature_test : public beast::unit_test::suite ++it) { uint256 id; - id.SetHexExact(it.key().asString().c_str()); + (void)id.parseHex(it.key().asString().c_str()); if (!BEAST_EXPECT((*it).isMember(jss::name))) return; bool expectEnabled = env.app().getAmendmentTable().isEnabled(id); diff --git a/src/test/rpc/LedgerRPC_test.cpp b/src/test/rpc/LedgerRPC_test.cpp index b58d2135904..d0070857d3f 100644 --- a/src/test/rpc/LedgerRPC_test.cpp +++ b/src/test/rpc/LedgerRPC_test.cpp @@ -249,10 +249,10 @@ class LedgerRPC_test : public beast::unit_test::suite Env env{*this, envconfig(no_admin)}; - env.close(); + // env.close(); Json::Value jvParams; - jvParams[jss::ledger_index] = 3u; + jvParams[jss::ledger_index] = 1u; jvParams[jss::full] = true; auto const jrr = env.rpc("json", "ledger", to_string(jvParams))[jss::result]; @@ -1345,9 +1345,8 @@ class LedgerRPC_test : public beast::unit_test::suite "json", "ledger", boost::lexical_cast(jvParams))[jss::result]; - BEAST_EXPECT(jrr.isMember(jss::ledger)); - BEAST_EXPECT(jrr.isMember(jss::ledger_hash)); - BEAST_EXPECT(jrr[jss::ledger][jss::ledger_index] == "3"); + BEAST_EXPECT(jrr[jss::error] == "invalidParams"); + BEAST_EXPECT(jrr[jss::error_message] == "ledgerHashMalformed"); // request with non-string ledger_hash jvParams[jss::ledger_hash] = 2; diff --git a/src/test/rpc/Submit_test.cpp b/src/test/rpc/Submit_test.cpp index bd8047372ef..252e1c32a8a 100644 --- a/src/test/rpc/Submit_test.cpp +++ b/src/test/rpc/Submit_test.cpp @@ -17,6 +17,7 @@ */ //============================================================================== +#include #include #include #include @@ -73,18 +74,11 @@ class Submit_test : public beast::unit_test::suite env.trust(bob["USD"](TestData::fund), alice); env.close(); - auto toBinary = [](std::string const& text) { - std::string binary; - for (size_t i = 0; i < text.size(); ++i) - { - unsigned int c = charUnHex(text[i]); - c = c << 4; - ++i; - c = c | charUnHex(text[i]); - binary.push_back(c); - } - - return binary; + auto toBinary = [this](std::string const& text) { + auto blob = strUnHex(text); + BEAST_EXPECT(blob); + return std::string{ + reinterpret_cast(blob->data()), blob->size()}; }; // use a websocket client to fill transaction blobs diff --git a/src/test/rpc/TransactionEntry_test.cpp b/src/test/rpc/TransactionEntry_test.cpp index 1d81355c9d7..a477a624859 100644 --- a/src/test/rpc/TransactionEntry_test.cpp +++ b/src/test/rpc/TransactionEntry_test.cpp @@ -66,7 +66,7 @@ class TransactionEntry_test : public beast::unit_test::suite auto const result = env.client().invoke("transaction_entry", params)[jss::result]; BEAST_EXPECT(!result[jss::ledger_hash].asString().empty()); - BEAST_EXPECT(result[jss::error] == "transactionNotFound"); + BEAST_EXPECT(result[jss::error] == "malformedRequest"); BEAST_EXPECT(result[jss::status] == "error"); } diff --git a/src/test/shamap/FetchPack_test.cpp b/src/test/shamap/FetchPack_test.cpp index 348e59e704a..ce58ba05ee3 100644 --- a/src/test/shamap/FetchPack_test.cpp +++ b/src/test/shamap/FetchPack_test.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -64,7 +65,7 @@ class FetchPack_test : public beast::unit_test::suite SHAMapHash const& nodeHash, std::uint32_t ledgerSeq, Blob&& nodeData, - SHAMapTreeNode::TNType type) const override + SHAMapNodeType type) const override { } @@ -99,7 +100,8 @@ class FetchPack_test : public beast::unit_test::suite while (n--) { std::shared_ptr item(make_random_item(r)); - auto const result(t.addItem(std::move(*item), false, false)); + auto const result( + t.addItem(SHAMapNodeType::tnACCOUNT_STATE, std::move(*item))); assert(result); (void)result; } diff --git a/src/test/shamap/SHAMapSync_test.cpp b/src/test/shamap/SHAMapSync_test.cpp index 61b4eaae150..a5ca735380a 100644 --- a/src/test/shamap/SHAMapSync_test.cpp +++ b/src/test/shamap/SHAMapSync_test.cpp @@ -59,7 +59,7 @@ class SHAMapSync_test : public beast::unit_test::suite std::shared_ptr item = makeRandomAS(); items.push_back(item->key()); - if (!map.addItem(std::move(*item), false, false)) + if (!map.addItem(SHAMapNodeType::tnACCOUNT_STATE, std::move(*item))) { log << "Unable to add item to map\n"; return false; @@ -98,7 +98,8 @@ class SHAMapSync_test : public beast::unit_test::suite int items = 10000; for (int i = 0; i < items; ++i) { - source.addItem(std::move(*makeRandomAS()), false, false); + source.addItem( + SHAMapNodeType::tnACCOUNT_STATE, std::move(*makeRandomAS())); if (i % 100 == 0) source.invariants(); } diff --git a/src/test/shamap/SHAMap_test.cpp b/src/test/shamap/SHAMap_test.cpp index 49d5d5638ec..98f61abec07 100644 --- a/src/test/shamap/SHAMap_test.cpp +++ b/src/test/shamap/SHAMap_test.cpp @@ -64,12 +64,12 @@ static_assert(std::is_copy_assignable{}, ""); static_assert(std::is_move_constructible{}, ""); static_assert(std::is_move_assignable{}, ""); -static_assert(!std::is_nothrow_destructible{}, ""); -static_assert(!std::is_default_constructible{}, ""); -static_assert(!std::is_copy_constructible{}, ""); -static_assert(!std::is_copy_assignable{}, ""); -static_assert(!std::is_move_constructible{}, ""); -static_assert(!std::is_move_assignable{}, ""); +static_assert(std::is_nothrow_destructible{}, ""); +static_assert(!std::is_default_constructible{}, ""); +static_assert(!std::is_copy_constructible{}, ""); +static_assert(!std::is_copy_assignable{}, ""); +static_assert(!std::is_move_constructible{}, ""); +static_assert(!std::is_move_assignable{}, ""); static_assert(std::is_nothrow_destructible{}, ""); static_assert(!std::is_default_constructible{}, ""); @@ -78,12 +78,12 @@ static_assert(!std::is_copy_assignable{}, ""); static_assert(!std::is_move_constructible{}, ""); static_assert(!std::is_move_assignable{}, ""); -static_assert(std::is_nothrow_destructible{}, ""); -static_assert(!std::is_default_constructible{}, ""); -static_assert(!std::is_copy_constructible{}, ""); -static_assert(!std::is_copy_assignable{}, ""); -static_assert(!std::is_move_constructible{}, ""); -static_assert(!std::is_move_assignable{}, ""); +static_assert(std::is_nothrow_destructible{}, ""); +static_assert(!std::is_default_constructible{}, ""); +static_assert(!std::is_copy_constructible{}, ""); +static_assert(!std::is_copy_assignable{}, ""); +static_assert(!std::is_move_constructible{}, ""); +static_assert(!std::is_move_assignable{}, ""); #endif inline bool @@ -143,15 +143,15 @@ class SHAMap_test : public beast::unit_test::suite // h3 and h4 differ only in the leaf, same terminal node (level 19) uint256 h1, h2, h3, h4, h5; - h1.SetHex( + (void)h1.parseHex( "092891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7"); - h2.SetHex( + (void)h2.parseHex( "436ccbac3347baa1f1e53baeef1f43334da88f1f6d70d963b833afd6dfa289fe"); - h3.SetHex( + (void)h3.parseHex( "b92891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6ca8"); - h4.SetHex( + (void)h4.parseHex( "b92891fe4ef6cee585fdc6fda2e09eb4d386363158ec3321b8123e5a772c6ca8"); - h5.SetHex( + (void)h5.parseHex( "a92891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7"); SHAMap sMap(SHAMapType::FREE, f); @@ -161,9 +161,13 @@ class SHAMap_test : public beast::unit_test::suite SHAMapItem i1(h1, IntToVUC(1)), i2(h2, IntToVUC(2)), i3(h3, IntToVUC(3)), i4(h4, IntToVUC(4)), i5(h5, IntToVUC(5)); - unexpected(!sMap.addItem(SHAMapItem{i2}, true, false), "no add"); + unexpected( + !sMap.addItem(SHAMapNodeType::tnTRANSACTION_NM, SHAMapItem{i2}), + "no add"); sMap.invariants(); - unexpected(!sMap.addItem(SHAMapItem{i1}, true, false), "no add"); + unexpected( + !sMap.addItem(SHAMapNodeType::tnTRANSACTION_NM, SHAMapItem{i1}), + "no add"); sMap.invariants(); auto i = sMap.begin(); @@ -173,11 +177,11 @@ class SHAMap_test : public beast::unit_test::suite unexpected(i == e || (*i != i2), "bad traverse"); ++i; unexpected(i != e, "bad traverse"); - sMap.addItem(SHAMapItem{i4}, true, false); + sMap.addItem(SHAMapNodeType::tnTRANSACTION_NM, SHAMapItem{i4}); sMap.invariants(); sMap.delItem(i2.key()); sMap.invariants(); - sMap.addItem(SHAMapItem{i3}, true, false); + sMap.addItem(SHAMapNodeType::tnTRANSACTION_NM, SHAMapItem{i3}); sMap.invariants(); i = sMap.begin(); e = sMap.end(); @@ -223,54 +227,54 @@ class SHAMap_test : public beast::unit_test::suite testcase("build/tear unbacked"); { std::vector keys(8); - keys[0].SetHex( + (void)keys[0].parseHex( "b92891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6c" "a8"); - keys[1].SetHex( + (void)keys[1].parseHex( "b92881fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6c" "a8"); - keys[2].SetHex( + (void)keys[2].parseHex( "b92691fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6c" "a8"); - keys[3].SetHex( + (void)keys[3].parseHex( "b92791fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6c" "a8"); - keys[4].SetHex( + (void)keys[4].parseHex( "b91891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6c" "a8"); - keys[5].SetHex( + (void)keys[5].parseHex( "b99891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6c" "a8"); - keys[6].SetHex( + (void)keys[6].parseHex( "f22891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6c" "a8"); - keys[7].SetHex( + (void)keys[7].parseHex( "292891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6c" "a8"); std::vector hashes(8); - hashes[0].SetHex( + (void)hashes[0].parseHex( "B7387CFEA0465759ADC718E8C42B52D2309D179B326E239EB5075C64B6281F" "7F"); - hashes[1].SetHex( + (void)hashes[1].parseHex( "FBC195A9592A54AB44010274163CB6BA95F497EC5BA0A8831845467FB2ECE2" "66"); - hashes[2].SetHex( + (void)hashes[2].parseHex( "4E7D2684B65DFD48937FFB775E20175C43AF0C94066F7D5679F51AE756795B" "75"); - hashes[3].SetHex( + (void)hashes[3].parseHex( "7A2F312EB203695FFD164E038E281839EEF06A1B99BFC263F3CECC6C74F93E" "07"); - hashes[4].SetHex( + (void)hashes[4].parseHex( "395A6691A372387A703FB0F2C6D2C405DAF307D0817F8F0E207596462B0E3A" "3E"); - hashes[5].SetHex( + (void)hashes[5].parseHex( "D044C0A696DE3169CC70AE216A1564D69DE96582865796142CE7D98A84D9DD" "E4"); - hashes[6].SetHex( + (void)hashes[6].parseHex( "76DCC77C4027309B5A91AD164083264D70B77B5E43E08AEDA5EBF943611436" "15"); - hashes[7].SetHex( + (void)hashes[7].parseHex( "DF4220E93ADC6F5569063A01B4DC79F8DB9553B6A3222ADE23DEA02BBE7230" "E5"); @@ -282,7 +286,8 @@ class SHAMap_test : public beast::unit_test::suite for (int k = 0; k < keys.size(); ++k) { SHAMapItem item(keys[k], IntToVUC(k)); - BEAST_EXPECT(map.addItem(std::move(item), true, false)); + BEAST_EXPECT(map.addItem( + SHAMapNodeType::tnTRANSACTION_NM, std::move(item))); BEAST_EXPECT(map.getHash().as_uint256() == hashes[k]); map.invariants(); } @@ -302,28 +307,28 @@ class SHAMap_test : public beast::unit_test::suite { std::vector keys(8); - keys[0].SetHex( + (void)keys[0].parseHex( "f22891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6c" "a8"); - keys[1].SetHex( + (void)keys[1].parseHex( "b99891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6c" "a8"); - keys[2].SetHex( + (void)keys[2].parseHex( "b92891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6c" "a8"); - keys[3].SetHex( + (void)keys[3].parseHex( "b92881fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6c" "a8"); - keys[4].SetHex( + (void)keys[4].parseHex( "b92791fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6c" "a8"); - keys[5].SetHex( + (void)keys[5].parseHex( "b92691fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6c" "a8"); - keys[6].SetHex( + (void)keys[6].parseHex( "b91891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6c" "a8"); - keys[7].SetHex( + (void)keys[7].parseHex( "292891fe4ef6cee585fdc6fda1e09eb4d386363158ec3321b8123e5a772c6c" "a8"); @@ -333,7 +338,9 @@ class SHAMap_test : public beast::unit_test::suite map.setUnbacked(); for (auto const& k : keys) { - map.addItem(SHAMapItem{k, IntToVUC(0)}, true, false); + map.addItem( + SHAMapNodeType::tnTRANSACTION_NM, + SHAMapItem{k, IntToVUC(0)}); map.invariants(); }