Enforce account RPC limits by account objects traversed

CMakeLists.txt

@@ -6,6 +6,15 @@ endif ()
 
 project (rippled)
 
+# make GIT_COMMIT_HASH define available to all sources
+find_package(Git)
+if(Git_FOUND)
+    execute_process(COMMAND ${GIT_EXECUTABLE} describe --always --abbrev=40
+        OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE GIT_COMMIT_HASH)
+    message(STATUS gch: ${GIT_COMMIT_HASH})
+    add_definitions(-DGIT_COMMIT_HASH="${GIT_COMMIT_HASH}")
+endif() #git
+
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake")
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake/deps")
 

README.md

@@ -1,18 +1,18 @@
 # The XRP Ledger
 
-The [XRP Ledger](https://xrpl.org/) is a decentralized cryptographic ledger powered by a network of peer-to-peer servers. The XRP Ledger uses a novel Byzantine Fault Tolerant consensus algorithm to settle and record transactions in a secure distributed database without a central operator.
+The [XRP Ledger](https://xrpl.org/) is a decentralized cryptographic ledger powered by a network of peer-to-peer nodes. The XRP Ledger uses a novel Byzantine Fault Tolerant consensus algorithm to settle and record transactions in a secure distributed database without a central operator.
 
 ## XRP
 [XRP](https://xrpl.org/xrp.html) is a public, counterparty-free asset native to the XRP Ledger, and is designed to bridge the many different currencies in use worldwide. XRP is traded on the open-market and is available for anyone to access. The XRP Ledger was created in 2012 with a finite supply of 100 billion units of XRP. Its creators gifted 80 billion XRP to a company, now called [Ripple](https://ripple.com/), to develop the XRP Ledger and its ecosystem. Ripple uses XRP to help build the Internet of Value, ushering in a world in which money moves as fast and efficiently as information does today.
 
 ## rippled
-The server software that powers the XRP Ledger is called `rippled` and is available in this repository under the permissive [ISC open-source license](LICENSE). The `rippled` server is written primarily in C++ and runs on a variety of platforms.
+The server software that powers the XRP Ledger is called `rippled` and is available in this repository under the permissive [ISC open-source license](LICENSE.md). The `rippled` server software is written primarily in C++ and runs on a variety of platforms. The `rippled` server software can run in several modes depending on its [configuration](https://xrpl.org/rippled-server-modes.html).  
 
 ### Build from Source
 
 * [Linux](Builds/linux/README.md)
-* [Mac](Builds/macos/README.md)
-* [Windows](Builds/VisualStudio2017/README.md)
+* [Mac](Builds/macos/README.md) (Not recommended for production)
+* [Windows](Builds/VisualStudio2017/README.md) (Not recommended for production)
 
 ## Key Features of the XRP Ledger
 

cfg/rippled-example.cfg

@@ -622,18 +622,28 @@
 #
 # [relay_proposals]
 #
-#   Controls the relaying behavior for proposals received by this server that
-#   are issued by validators that are not on the server's UNL.
+#   Controls the relay and processing behavior for proposals received by this
+#   server that are issued by validators that are not on the server's UNL.
 #
-#   Legal values are: "trusted" and "all". The default is "trusted".
+#   Legal values are:
+#     "all"            - Relay and process all incoming proposals
+#     "trusted"        - Relay only trusted proposals, but locally process all
+#     "drop_untrusted" - Relay only trusted proposals, do not process untrusted
+#
+#   The default is "trusted".
 #
 #
 # [relay_validations]
 #
-#   Controls the relaying behavior for validations received by this server that
-#   are issued by validators that are not on the server's UNL.
+#   Controls the relay and processing behavior for validations received by this
+#   server that are issued by validators that are not on the server's UNL.
+#
+#   Legal values are:
+#     "all"            - Relay and process all incoming validations
+#     "trusted"        - Relay only trusted validations, but locally process all
+#     "drop_untrusted" - Relay only trusted validations, do not process untrusted
 #
-#   Legal values are: "trusted" and "all". The default is "all".
+#   The default is "all".
 #
 #
 #
@@ -769,6 +779,14 @@
 #   number of processor threads plus 2 for networked nodes. Nodes running in
 #   stand alone mode default to 1 worker.
 #
+# [io_workers]
+#
+#   Configures the number of threads for processing raw inbound and outbound IO.
+#
+# [prefetch_workers]
+#
+#   Configures the number of threads for performing nodestore prefetching.
+#
 #
 #
 # [network_id]
@@ -1132,6 +1150,9 @@
 #                           cluster. Setting this option can help eliminate
 #                           write timeouts and other write errors due to the
 #                           cluster being overloaded.
+#       io_threads
+#                           Set the number of IO threads used by the
+#                           Cassandra driver. Defaults to 4.
 #
 #   Notes:
 #       The 'node_db' entry configures the primary, persistent storage.

docs/consensus.md

@@ -469,7 +469,7 @@ struct Ledger
   // Whether the ledger's close time was a non-trivial consensus result
   bool closeAgree() const;
 
-  // The close time resolution used in determing the close time
+  // The close time resolution used in determining the close time
   NetClock::duration closeTimeResolution() const;
 
   // The (effective) close time, based on the closeTimeResolution

src/ripple/app/ledger/OpenLedger.h

@@ -185,7 +185,6 @@ class OpenLedger
         FwdRange const& txs,
         OrderedTxs& retries,
         ApplyFlags flags,
-        std::map<uint256, bool>& shouldRecover,
         beast::Journal j);
 
     enum Result { success, failure, retry };
@@ -200,7 +199,6 @@ class OpenLedger
         std::shared_ptr<STTx const> const& tx,
         bool retry,
         ApplyFlags flags,
-        bool shouldRecover,
         beast::Journal j);
 };
 
@@ -215,7 +213,6 @@ OpenLedger::apply(
     FwdRange const& txs,
     OrderedTxs& retries,
     ApplyFlags flags,
-    std::map<uint256, bool>& shouldRecover,
     beast::Journal j)
 {
     for (auto iter = txs.begin(); iter != txs.end(); ++iter)
@@ -227,8 +224,7 @@ OpenLedger::apply(
             auto const txId = tx->getTransactionID();
             if (check.txExists(txId))
                 continue;
-            auto const result =
-                apply_one(app, view, tx, true, flags, shouldRecover[txId], j);
+            auto const result = apply_one(app, view, tx, true, flags, j);
             if (result == Result::retry)
                 retries.insert(tx);
         }
@@ -245,14 +241,7 @@ OpenLedger::apply(
         auto iter = retries.begin();
         while (iter != retries.end())
         {
-            switch (apply_one(
-                app,
-                view,
-                iter->second,
-                retry,
-                flags,
-                shouldRecover[iter->second->getTransactionID()],
-                j))
+            switch (apply_one(app, view, iter->second, retry, flags, j))
             {
                 case Result::success:
                     ++changes;

src/ripple/app/ledger/impl/LedgerMaster.cpp

@@ -1080,6 +1080,16 @@ LedgerMaster::checkAccept(std::shared_ptr<Ledger const> const& ledger)
     if (!fees.empty())
     {
         std::sort(fees.begin(), fees.end());
+        if (auto stream = m_journal.debug())
+        {
+            std::stringstream s;
+            s << "Received fees from validations: (" << fees.size() << ") ";
+            for (auto const fee1 : fees)
+            {
+                s << " " << fee1;
+            }
+            stream << s.str();
+        }
         fee = fees[fees.size() / 2];  // median
     }
     else

src/ripple/app/ledger/impl/OpenLedger.cpp

@@ -81,17 +81,11 @@ OpenLedger::accept(
 {
     JLOG(j_.trace()) << "accept ledger " << ledger->seq() << " " << suffix;
     auto next = create(rules, ledger);
-    std::map<uint256, bool> shouldRecover;
     if (retriesFirst)
     {
-        for (auto const& tx : retries)
-        {
-            auto const txID = tx.second->getTransactionID();
-            shouldRecover[txID] = app.getHashRouter().shouldRecover(txID);
-        }
         // Handle disputed tx, outside lock
         using empty = std::vector<std::shared_ptr<STTx const>>;
-        apply(app, *next, *ledger, empty{}, retries, flags, shouldRecover, j_);
+        apply(app, *next, *ledger, empty{}, retries, flags, j_);
     }
     // Block calls to modify, otherwise
     // new tx going into the open ledger
@@ -100,17 +94,6 @@ OpenLedger::accept(
     // Apply tx from the current open view
     if (!current_->txs.empty())
     {
-        for (auto const& tx : current_->txs)
-        {
-            auto const txID = tx.first->getTransactionID();
-            auto iter = shouldRecover.lower_bound(txID);
-            if (iter != shouldRecover.end() && iter->first == txID)
-                // already had a chance via disputes
-                iter->second = false;
-            else
-                shouldRecover.emplace_hint(
-                    iter, txID, app.getHashRouter().shouldRecover(txID));
-        }
         apply(
             app,
             *next,
@@ -124,7 +107,6 @@ OpenLedger::accept(
                 }),
             retries,
             flags,
-            shouldRecover,
             j_);
     }
     // Call the modifier
@@ -179,21 +161,12 @@ OpenLedger::apply_one(
     std::shared_ptr<STTx const> const& tx,
     bool retry,
     ApplyFlags flags,
-    bool shouldRecover,
     beast::Journal j) -> Result
 {
     if (retry)
         flags = flags | tapRETRY;
-    auto const result = [&] {
-        auto const queueResult =
-            app.getTxQ().apply(app, view, tx, flags | tapPREFER_QUEUE, j);
-        // If the transaction can't get into the queue for intrinsic
-        // reasons, and it can still be recovered, try to put it
-        // directly into the open ledger, else drop it.
-        if (queueResult.first == telCAN_NOT_QUEUE && shouldRecover)
-            return ripple::apply(app, view, *tx, flags, j);
-        return queueResult;
-    }();
+    // If it's in anybody's proposed set, try to keep it in the ledger
+    auto const result = ripple::apply(app, view, *tx, flags, j);
     if (result.second || result.first == terQUEUED)
         return Result::success;
     if (isTefFailure(result.first) || isTemMalformed(result.first) ||

src/ripple/app/main/Application.cpp

@@ -246,6 +246,10 @@ class ApplicationImp : public Application, public BasicApp
 #if RIPPLE_SINGLE_IO_SERVICE_THREAD
         return 1;
 #else
+
+        if (config.IO_WORKERS > 0)
+            return config.IO_WORKERS;
+
         auto const cores = std::thread::hardware_concurrency();
 
         // Use a single thread when running on under-provisioned systems
@@ -342,7 +346,8 @@ class ApplicationImp : public Application, public BasicApp
               m_collectorManager->collector(),
               logs_->journal("Resource")))
 
-        , m_nodeStore(m_shaMapStore->makeNodeStore(4))
+        , m_nodeStore(m_shaMapStore->makeNodeStore(
+              config_->PREFETCH_WORKERS > 0 ? config_->PREFETCH_WORKERS : 4))
 
         , nodeFamily_(*this, *m_collectorManager)
 
@@ -443,8 +448,7 @@ class ApplicationImp : public Application, public BasicApp
 
         , hashRouter_(std::make_unique<HashRouter>(
               stopwatch(),
-              HashRouter::getDefaultHoldTime(),
-              HashRouter::getDefaultRecoverLimit()))
+              HashRouter::getDefaultHoldTime()))
 
         , mValidations(
               ValidationParms(),

src/ripple/app/main/GRPCServer.cpp

@@ -144,11 +144,11 @@ GRPCServerImpl::CallData<Request, Response>::process(
     {
         auto usage = getUsage();
         bool isUnlimited = clientIsUnlimited();
-        if (!isUnlimited && usage.disconnect())
+        if (!isUnlimited && usage.disconnect(app_.journal("gRPCServer")))
         {
             grpc::Status status{
                 grpc::StatusCode::RESOURCE_EXHAUSTED,
-                "usage balance exceeds threshhold"};
+                "usage balance exceeds threshold"};
             responder_.FinishWithError(status, this);
         }
         else

src/ripple/app/misc/FeeEscalation.md

@@ -92,7 +92,7 @@ traffic periods, and give those transactions a much better chance to
 succeed.
 
 1. If an incoming transaction meets both the base [fee
-level](#fee-level) and the load fee minimum, but does not have a high
+level](#fee-level) and the [load fee](#load-fee) minimum, but does not have a high
 enough [fee level](#fee-level) to immediately go into the open ledger,
 it is instead put into the queue and broadcast to peers. Each peer will
 then make an independent decision about whether to put the transaction
@@ -173,6 +173,10 @@ This demonstrates that a simpler transaction paying less XRP can be more
 likely to get into the open ledger, or be sorted earlier in the queue
 than a more complex transaction paying more XRP.
 
+### Load Fee
+
+Each rippled server maintains a minimum cost threshold based on its current load. If you submit a transaction with a fee that is lower than the current load-based transaction cost of the rippled server, the server neither applies nor relays the transaction to its peers. A transaction is very unlikely to survive the consensus process unless its transaction fee value meets the requirements of a majority of servers.
+
 ### Reference Transaction
 
 In this document, a "Reference Transaction" is any currently implemented

src/ripple/app/misc/HashRouter.cpp

@@ -128,14 +128,4 @@ HashRouter::shouldRelay(uint256 const& key)
     return s.releasePeerSet();
 }
 
-bool
-HashRouter::shouldRecover(uint256 const& key)
-{
-    std::lock_guard lock(mutex_);
-
-    auto& s = emplace(key).first;
-
-    return s.shouldRecover(recoverLimit_);
-}
-
 }  // namespace ripple

src/ripple/app/misc/HashRouter.h

@@ -116,20 +116,6 @@ class HashRouter
             return true;
         }
 
-        /** Determines if this item should be recovered from the open ledger.
-
-            Counts the number of times the item has been recovered.
-            Every `limit` times the function is called, return false.
-            Else return true.
-
-            @note The limit must be > 0
-        */
-        bool
-        shouldRecover(std::uint32_t limit)
-        {
-            return ++recoveries_ % limit != 0;
-        }
-
         bool
         shouldProcess(Stopwatch::time_point now, std::chrono::seconds interval)
         {
@@ -146,7 +132,6 @@ class HashRouter
         // than one flag needs to expire independently.
         std::optional<Stopwatch::time_point> relayed_;
         std::optional<Stopwatch::time_point> processed_;
-        std::uint32_t recoveries_ = 0;
     };
 
 public:
@@ -158,19 +143,8 @@ class HashRouter
         return 300s;
     }
 
-    static inline std::uint32_t
-    getDefaultRecoverLimit()
-    {
-        return 1;
-    }
-
-    HashRouter(
-        Stopwatch& clock,
-        std::chrono::seconds entryHoldTimeInSeconds,
-        std::uint32_t recoverLimit)
-        : suppressionMap_(clock)
-        , holdTime_(entryHoldTimeInSeconds)
-        , recoverLimit_(recoverLimit + 1u)
+    HashRouter(Stopwatch& clock, std::chrono::seconds entryHoldTimeInSeconds)
+        : suppressionMap_(clock), holdTime_(entryHoldTimeInSeconds)
     {
     }
 
@@ -231,15 +205,6 @@ class HashRouter
     std::optional<std::set<PeerShortID>>
     shouldRelay(uint256 const& key);
 
-    /** Determines whether the hashed item should be recovered
-        from the open ledger into the next open ledger or the transaction
-        queue.
-
-        @return `bool` indicates whether the item should be recovered
-    */
-    bool
-    shouldRecover(uint256 const& key);
-
 private:
     // pair.second indicates whether the entry was created
     std::pair<Entry&, bool>
@@ -256,8 +221,6 @@ class HashRouter
         suppressionMap_;
 
     std::chrono::seconds const holdTime_;
-
-    std::uint32_t const recoverLimit_;
 };
 
 }  // namespace ripple