Implement Shards

XRPLF · Dec 19, 2017 · de40a27 · de40a27
1 parent 9b1fde6
commit de40a27
Show file tree

Hide file tree

Showing 76 changed files with 3,989 additions and 1,398 deletions.
diff --git a/Builds/VisualStudio2015/RippleD.vcxproj b/Builds/VisualStudio2015/RippleD.vcxproj
@@ -2257,6 +2257,8 @@
     </ClInclude>
     <ClInclude Include="..\..\src\ripple\nodestore\DatabaseRotating.h">
     </ClInclude>
+    <ClInclude Include="..\..\src\ripple\nodestore\DatabaseShard.h">
+    </ClInclude>
     <ClInclude Include="..\..\src\ripple\nodestore\DummyScheduler.h">
     </ClInclude>
     <ClInclude Include="..\..\src\ripple\nodestore\Factory.h">
@@ -2271,7 +2273,19 @@
     </ClInclude>
     <ClInclude Include="..\..\src\ripple\nodestore\impl\codec.h">
     </ClInclude>
-    <ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseImp.h">
+    <ClCompile Include="..\..\src\ripple\nodestore\impl\Database.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='debug.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='release.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple\nodestore\impl\DatabaseNodeImp.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='debug.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='release.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+    </ClCompile>
+    <ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseNodeImp.h">
     </ClInclude>
     <ClCompile Include="..\..\src\ripple\nodestore\impl\DatabaseRotatingImp.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
@@ -2281,6 +2295,14 @@
     </ClCompile>
     <ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseRotatingImp.h">
     </ClInclude>
+    <ClCompile Include="..\..\src\ripple\nodestore\impl\DatabaseShardImp.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='debug.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='release.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+    </ClCompile>
+    <ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseShardImp.h">
+    </ClInclude>
     <ClCompile Include="..\..\src\ripple\nodestore\impl\DecodedBlob.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
@@ -2317,6 +2339,14 @@
       <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='debug.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='release.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
+    <ClCompile Include="..\..\src\ripple\nodestore\impl\Shard.cpp">
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='debug.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='release.classic|x64'">..\..\src\rocksdb2\include;..\..\src\snappy\config;..\..\src\snappy\snappy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+    </ClCompile>
+    <ClInclude Include="..\..\src\ripple\nodestore\impl\Shard.h">
+    </ClInclude>
     <ClInclude Include="..\..\src\ripple\nodestore\impl\Tuning.h">
     </ClInclude>
     <ClInclude Include="..\..\src\ripple\nodestore\impl\varint.h">

diff --git a/Builds/VisualStudio2015/RippleD.vcxproj.filters b/Builds/VisualStudio2015/RippleD.vcxproj.filters
@@ -2913,6 +2913,9 @@
     <ClInclude Include="..\..\src\ripple\nodestore\DatabaseRotating.h">
       <Filter>ripple\nodestore</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\src\ripple\nodestore\DatabaseShard.h">
+      <Filter>ripple\nodestore</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\src\ripple\nodestore\DummyScheduler.h">
       <Filter>ripple\nodestore</Filter>
     </ClInclude>
@@ -2928,7 +2931,13 @@
     <ClInclude Include="..\..\src\ripple\nodestore\impl\codec.h">
       <Filter>ripple\nodestore\impl</Filter>
     </ClInclude>
-    <ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseImp.h">
+    <ClCompile Include="..\..\src\ripple\nodestore\impl\Database.cpp">
+      <Filter>ripple\nodestore\impl</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\src\ripple\nodestore\impl\DatabaseNodeImp.cpp">
+      <Filter>ripple\nodestore\impl</Filter>
+    </ClCompile>
+    <ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseNodeImp.h">
       <Filter>ripple\nodestore\impl</Filter>
     </ClInclude>
     <ClCompile Include="..\..\src\ripple\nodestore\impl\DatabaseRotatingImp.cpp">
@@ -2937,6 +2946,12 @@
     <ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseRotatingImp.h">
       <Filter>ripple\nodestore\impl</Filter>
     </ClInclude>
+    <ClCompile Include="..\..\src\ripple\nodestore\impl\DatabaseShardImp.cpp">
+      <Filter>ripple\nodestore\impl</Filter>
+    </ClCompile>
+    <ClInclude Include="..\..\src\ripple\nodestore\impl\DatabaseShardImp.h">
+      <Filter>ripple\nodestore\impl</Filter>
+    </ClInclude>
     <ClCompile Include="..\..\src\ripple\nodestore\impl\DecodedBlob.cpp">
       <Filter>ripple\nodestore\impl</Filter>
     </ClCompile>
@@ -2961,6 +2976,12 @@
     <ClCompile Include="..\..\src\ripple\nodestore\impl\NodeObject.cpp">
       <Filter>ripple\nodestore\impl</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\ripple\nodestore\impl\Shard.cpp">
+      <Filter>ripple\nodestore\impl</Filter>
+    </ClCompile>
+    <ClInclude Include="..\..\src\ripple\nodestore\impl\Shard.h">
+      <Filter>ripple\nodestore\impl</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\src\ripple\nodestore\impl\Tuning.h">
       <Filter>ripple\nodestore\impl</Filter>
     </ClInclude>

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -150,6 +150,7 @@ use_boost(
   filesystem
   program_options
   regex
+  serialization
   system
   thread)
 

diff --git a/SConstruct b/SConstruct
@@ -613,7 +613,8 @@ def config_env(toolchain, variant, env):
             'boost_program_options',
             'boost_regex',
             'boost_system',
-            'boost_thread'
+            'boost_thread',
+            'boost_serialization'
         ]
         env.Append(LIBS=['dl'])
 

diff --git a/doc/rippled-example.cfg b/doc/rippled-example.cfg
@@ -836,6 +836,33 @@
 #   [import_db]     Settings for performing a one-time import (optional)
 #   [database_path]   Path to the book-keeping databases.
 #
+#   [shard_db]      Settings for the Shard Database (optional)
+#
+#   Format (without spaces):
+#       One or more lines of case-insensitive key / value pairs:
+#       <key> '=' <value>
+#       ...
+#
+#   Example:
+#       type=nudb
+#       path=db/nudb
+#
+#   The "type" field must be present and controls the choice of backend:
+#
+#   type = NuDB
+#
+#   type = RocksDB
+#
+#       The RocksDB backend also provides these optional parameters:
+#
+#       compression         0 for none, 1 for Snappy compression
+#
+#   Required keys:
+#       path                Location to store the database (all types)
+#
+#       max_size_gb         Maximum disk space the database will utilize (in gigabytes)
+#
+#
 #   There are 4 bookkeeping SQLite database that the server creates and
 #   maintains. If you omit this configuration setting, it will default to
 #   creating a directory called "db" located in the same place as your

diff --git a/src/ripple/app/consensus/RCLConsensus.cpp b/src/ripple/app/consensus/RCLConsensus.cpp
@@ -36,6 +36,7 @@
 #include <ripple/basics/make_lock.h>
 #include <ripple/beast/core/LexicalCast.h>
 #include <ripple/consensus/LedgerTiming.h>
+#include <ripple/nodestore/DatabaseShard.h>
 #include <ripple/overlay/Overlay.h>
 #include <ripple/overlay/predicates.h>
 #include <ripple/protocol/Feature.h>
@@ -106,7 +107,7 @@ RCLConsensus::Adaptor::acquireLedger(LedgerHash const& ledger)
             app_.getJobQueue().addJob(
                 jtADVANCE, "getConsensusLedger", [app, hash](Job&) {
                     app->getInboundLedgers().acquire(
-                        hash, 0, InboundLedger::fcCONSENSUS);
+                        hash, 0, InboundLedger::Reason::CONSENSUS);
                 });
         }
         return boost::none;
@@ -625,9 +626,16 @@ RCLConsensus::Adaptor::notify(
     }
     s.set_firstseq(uMin);
     s.set_lastseq(uMax);
-    app_.overlay().foreach (
-        send_always(std::make_shared<Message>(s, protocol::mtSTATUS_CHANGE)));
-    JLOG(j_.trace()) << "send status change to peer";
+    if (auto shardStore = app_.getShardStore())
+    {
+        auto shards = shardStore->getCompleteShards();
+        if (! shards.empty())
+            s.set_shardseqs(shards);
+    }
+    app_.overlay ().foreach (send_always (
+        std::make_shared <Message> (
+            s, protocol::mtSTATUS_CHANGE)));
+    JLOG (j_.trace()) << "send status change to peer";
 }
 
 /** Apply a set of transactions to a ledger.

diff --git a/src/ripple/app/ledger/AccountStateSF.cpp b/src/ripple/app/ledger/AccountStateSF.cpp
@@ -22,19 +22,13 @@
 
 namespace ripple {
 
-AccountStateSF::AccountStateSF(Family& f, AbstractFetchPackContainer& fp)
-    : f_(f)
-    , fp_(fp)
+void
+AccountStateSF::gotNode(bool, SHAMapHash const& nodeHash,
+    std::uint32_t ledgerSeq, Blob&& nodeData,
+    SHAMapTreeNode::TNType) const
 {
-}
-
-void AccountStateSF::gotNode (bool fromFilter,
-                              SHAMapHash const& nodeHash,
-                              Blob&& nodeData,
-                              SHAMapTreeNode::TNType) const
-{
-    f_.db().store(hotACCOUNT_NODE, std::move(nodeData),
-        nodeHash.as_uint256());
+    db_.store(hotACCOUNT_NODE, std::move(nodeData),
+        nodeHash.as_uint256(), ledgerSeq);
 }
 
 boost::optional<Blob>

diff --git a/src/ripple/app/ledger/AccountStateSF.h b/src/ripple/app/ledger/AccountStateSF.h
@@ -21,31 +21,32 @@
 #define RIPPLE_APP_LEDGER_ACCOUNTSTATESF_H_INCLUDED
 
 #include <ripple/app/ledger/AbstractFetchPackContainer.h>
+#include <ripple/nodestore/Database.h>
 #include <ripple/shamap/SHAMapSyncFilter.h>
-#include <ripple/shamap/Family.h>
 
 namespace ripple {
 
 // This class is only needed on add functions
 // sync filter for account state nodes during ledger sync
-class AccountStateSF
-    : public SHAMapSyncFilter
+class AccountStateSF : public SHAMapSyncFilter
 {
-private:
-    Family& f_;
-    AbstractFetchPackContainer& fp_;
-
 public:
-    AccountStateSF(Family&, AbstractFetchPackContainer&);
+    AccountStateSF(NodeStore::Database& db, AbstractFetchPackContainer& fp)
+        : db_(db)
+        , fp_(fp)
+    {}
 
-    // Note that the nodeData is overwritten by this call
-    void gotNode (bool fromFilter,
-                  SHAMapHash const& nodeHash,
-                  Blob&& nodeData,
-                  SHAMapTreeNode::TNType) const override;
+    void
+    gotNode(bool fromFilter, SHAMapHash const& nodeHash,
+        std::uint32_t ledgerSeq, Blob&& nodeData,
+            SHAMapTreeNode::TNType type) const override;
 
     boost::optional<Blob>
     getNode(SHAMapHash const& nodeHash) const override;
+
+private:
+    NodeStore::Database& db_;
+    AbstractFetchPackContainer& fp_;
 };
 
 } // ripple

diff --git a/src/ripple/app/ledger/ConsensusTransSetSF.cpp b/src/ripple/app/ledger/ConsensusTransSetSF.cpp
@@ -38,9 +38,9 @@ ConsensusTransSetSF::ConsensusTransSetSF (Application& app, NodeCache& nodeCache
 {
 }
 
-void ConsensusTransSetSF::gotNode (
-    bool fromFilter, SHAMapHash const& nodeHash,
-    Blob&& nodeData, SHAMapTreeNode::TNType type) const
+void
+ConsensusTransSetSF::gotNode(bool fromFilter, SHAMapHash const& nodeHash,
+    std::uint32_t, Blob&& nodeData, SHAMapTreeNode::TNType type) const
 {
     if (fromFilter)
         return;

diff --git a/src/ripple/app/ledger/ConsensusTransSetSF.h b/src/ripple/app/ledger/ConsensusTransSetSF.h
@@ -39,10 +39,10 @@ class ConsensusTransSetSF : public SHAMapSyncFilter
     ConsensusTransSetSF (Application& app, NodeCache& nodeCache);
 
     // Note that the nodeData is overwritten by this call
-    void gotNode (bool fromFilter,
-                  SHAMapHash const& nodeHash,
-                  Blob&& nodeData,
-                  SHAMapTreeNode::TNType) const override;
+    void
+    gotNode(bool fromFilter, SHAMapHash const& nodeHash,
+        std::uint32_t ledgerSeq, Blob&& nodeData,
+            SHAMapTreeNode::TNType type) const override;
 
     boost::optional<Blob>
     getNode (SHAMapHash const& nodeHash) const override;