Enabling BlobDB for column families with static data

Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net>
hyperledger · May 19, 2023 · f944e0b · f944e0b
1 parent ab4020e
commit f944e0b
Show file tree

Hide file tree

Showing 5 changed files with 62 additions and 14 deletions.
diff --git a/...c/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java b/...c/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java
@@ -19,7 +19,7 @@
 import org.bouncycastle.util.Arrays;
 
 public enum KeyValueSegmentIdentifier implements SegmentIdentifier {
-  BLOCKCHAIN(new byte[] {1}),
+  BLOCKCHAIN(new byte[] {1}, true),
   WORLD_STATE(new byte[] {2}, new int[] {0, 1}),
   PRIVATE_TRANSACTIONS(new byte[] {3}),
   PRIVATE_STATE(new byte[] {4}),
@@ -28,7 +28,7 @@ public enum KeyValueSegmentIdentifier implements SegmentIdentifier {
   CODE_STORAGE(new byte[] {7}, new int[] {2}),
   ACCOUNT_STORAGE_STORAGE(new byte[] {8}, new int[] {2}),
   TRIE_BRANCH_STORAGE(new byte[] {9}, new int[] {2}),
-  TRIE_LOG_STORAGE(new byte[] {10}, new int[] {2}),
+  TRIE_LOG_STORAGE(new byte[] {10}, new int[] {2}, true),
   VARIABLES(new byte[] {11}), // formerly GOQUORUM_PRIVATE_WORLD_STATE
 
   // previously supported GoQuorum private states
@@ -44,14 +44,25 @@ public enum KeyValueSegmentIdentifier implements SegmentIdentifier {
 
   private final byte[] id;
   private final int[] versionList;
+  private final boolean containsStaticData;
 
   KeyValueSegmentIdentifier(final byte[] id) {
     this(id, new int[] {0, 1, 2});
   }
 
+  KeyValueSegmentIdentifier(final byte[] id, final boolean containsStaticData) {
+    this(id, new int[] {0, 1, 2}, containsStaticData);
+  }
+
   KeyValueSegmentIdentifier(final byte[] id, final int[] versionList) {
+    this(id, versionList, false);
+  }
+
+  KeyValueSegmentIdentifier(
+      final byte[] id, final int[] versionList, final boolean containsStaticData) {
     this.id = id;
     this.versionList = versionList;
+    this.containsStaticData = containsStaticData;
   }
 
   @Override
@@ -64,6 +75,11 @@ public byte[] getId() {
     return id;
   }
 
+  @Override
+  public boolean containsStaticData() {
+    return containsStaticData;
+  }
+
   @Override
   public boolean includeInDatabaseVersion(final int version) {
     return Arrays.contains(versionList, version);

diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle
@@ -69,7 +69,7 @@ Calculated : ${currentHash}
 tasks.register('checkAPIChanges', FileStateChecker) {
   description = "Checks that the API for the Plugin-API project does not change without deliberate thought"
   files = sourceSets.main.allJava.files
-  knownHash = '3/qsZ9+jA10YbctpPtQ5Rn7cvYHwKRWBv6jXa+7WQMY='
+  knownHash = 'kfPcUkAYaOMIvnckj9QVufV9Qg0BJJsF17u0itdjY9o='
 }
 check.dependsOn('checkAPIChanges')
 

diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java
@@ -47,4 +47,14 @@ public interface SegmentIdentifier {
   default boolean includeInDatabaseVersion(final int version) {
     return true;
   }
+
+  /**
+   * Define if this segment contains data that is never updated, but only added and optionally
+   * deleted. Example is append only data like the blockchain.
+   * This information can be used by the underlying implementation to apply specific
+   * optimization for this use case.
+   *
+   * @return true if the segment contains only static data
+   */
+  boolean containsStaticData();
 }
diff --git a/...ledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java b/...ledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java
@@ -151,16 +151,7 @@ public RocksDBColumnarKeyValueStorage(
                       .noneMatch(existed -> Arrays.equals(existed, ignorableSegment.getId())))
           .forEach(trimmedSegments::remove);
       columnDescriptors =
-          trimmedSegments.stream()
-              .map(
-                  segment ->
-                      new ColumnFamilyDescriptor(
-                          segment.getId(),
-                          new ColumnFamilyOptions()
-                              .setTtl(0)
-                              .setCompressionType(CompressionType.LZ4_COMPRESSION)
-                              .setTableFormatConfig(createBlockBasedTableConfig(configuration))))
-              .collect(Collectors.toList());
+          trimmedSegments.stream().map(this::createColumnDescriptor).collect(Collectors.toList());
       columnDescriptors.add(
           new ColumnFamilyDescriptor(
               DEFAULT_COLUMN.getBytes(StandardCharsets.UTF_8),
@@ -178,6 +169,24 @@ public RocksDBColumnarKeyValueStorage(
     }
   }
 
+  private ColumnFamilyDescriptor createColumnDescriptor(final SegmentIdentifier segment) {
+    final var options =
+        new ColumnFamilyOptions()
+            .setTtl(0)
+            .setCompressionType(CompressionType.LZ4_COMPRESSION)
+            .setTableFormatConfig(createBlockBasedTableConfig(configuration));
+
+    if (segment.containsStaticData()) {
+      options
+          .setEnableBlobFiles(true)
+          .setEnableBlobGarbageCollection(false)
+          .setMinBlobSize(100)
+          .setBlobCompressionType(CompressionType.LZ4_COMPRESSION);
+    }
+
+    return new ColumnFamilyDescriptor(segment.getId(), options);
+  }
+
   private void setGlobalOptions(final RocksDBConfiguration configuration, final Statistics stats) {
     options = new DBOptions();
     options

diff --git a/...er/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java b/...er/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java
@@ -261,14 +261,22 @@ public void dbWillBeBackwardIncompatibleAfterExperimentalSegmentsAreAdded() thro
   public enum TestSegment implements SegmentIdentifier {
     FOO(new byte[] {1}),
     BAR(new byte[] {2}),
-    EXPERIMENTAL(new byte[] {3});
+    EXPERIMENTAL(new byte[] {3}),
+
+    STATIC_DATA(new byte[] {4}, true);
 
     private final byte[] id;
     private final String nameAsUtf8;
+    private final boolean containsStaticData;
 
     TestSegment(final byte[] id) {
+      this(id, false);
+    }
+
+    TestSegment(final byte[] id, final boolean containsStaticData) {
       this.id = id;
       this.nameAsUtf8 = new String(id, StandardCharsets.UTF_8);
+      this.containsStaticData = containsStaticData;
     }
 
     @Override
@@ -280,6 +288,11 @@ public String getName() {
     public byte[] getId() {
       return id;
     }
+
+    @Override
+    public boolean containsStaticData() {
+      return containsStaticData;
+    }
   }
 
   protected abstract SegmentedKeyValueStorage<RocksDbSegmentIdentifier> createSegmentedStore()