diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java index 43b02f224de..3d3f6778655 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java @@ -19,7 +19,7 @@ import org.bouncycastle.util.Arrays; public enum KeyValueSegmentIdentifier implements SegmentIdentifier { - BLOCKCHAIN(new byte[] {1}), + BLOCKCHAIN(new byte[] {1}, true), WORLD_STATE(new byte[] {2}, new int[] {0, 1}), PRIVATE_TRANSACTIONS(new byte[] {3}), PRIVATE_STATE(new byte[] {4}), @@ -28,7 +28,7 @@ public enum KeyValueSegmentIdentifier implements SegmentIdentifier { CODE_STORAGE(new byte[] {7}, new int[] {2}), ACCOUNT_STORAGE_STORAGE(new byte[] {8}, new int[] {2}), TRIE_BRANCH_STORAGE(new byte[] {9}, new int[] {2}), - TRIE_LOG_STORAGE(new byte[] {10}, new int[] {2}), + TRIE_LOG_STORAGE(new byte[] {10}, new int[] {2}, true), VARIABLES(new byte[] {11}), // formerly GOQUORUM_PRIVATE_WORLD_STATE // previously supported GoQuorum private states @@ -44,14 +44,25 @@ public enum KeyValueSegmentIdentifier implements SegmentIdentifier { private final byte[] id; private final int[] versionList; + private final boolean containsStaticData; KeyValueSegmentIdentifier(final byte[] id) { this(id, new int[] {0, 1, 2}); } + KeyValueSegmentIdentifier(final byte[] id, final boolean containsStaticData) { + this(id, new int[] {0, 1, 2}, containsStaticData); + } + KeyValueSegmentIdentifier(final byte[] id, final int[] versionList) { + this(id, versionList, false); + } + + KeyValueSegmentIdentifier( + final byte[] id, final int[] versionList, final boolean containsStaticData) { this.id = id; this.versionList = versionList; + this.containsStaticData = containsStaticData; } @Override @@ -64,6 +75,11 @@ public byte[] getId() { return id; } + @Override + public boolean containsStaticData() { + return containsStaticData; + } + @Override public boolean includeInDatabaseVersion(final int version) { return Arrays.contains(versionList, version); diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index a465d64b0f6..711f88876e4 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -69,7 +69,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = '3/qsZ9+jA10YbctpPtQ5Rn7cvYHwKRWBv6jXa+7WQMY=' + knownHash = 'kfPcUkAYaOMIvnckj9QVufV9Qg0BJJsF17u0itdjY9o=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java index 9daf97fdeb9..0a545e259a6 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java @@ -47,4 +47,14 @@ public interface SegmentIdentifier { default boolean includeInDatabaseVersion(final int version) { return true; } + + /** + * Define if this segment contains data that is never updated, but only added and optionally + * deleted. Example is append only data like the blockchain. + * This information can be used by the underlying implementation to apply specific + * optimization for this use case. + * + * @return true if the segment contains only static data + */ + boolean containsStaticData(); } diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java index 484952cb384..720ecef4620 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java @@ -151,16 +151,7 @@ public RocksDBColumnarKeyValueStorage( .noneMatch(existed -> Arrays.equals(existed, ignorableSegment.getId()))) .forEach(trimmedSegments::remove); columnDescriptors = - trimmedSegments.stream() - .map( - segment -> - new ColumnFamilyDescriptor( - segment.getId(), - new ColumnFamilyOptions() - .setTtl(0) - .setCompressionType(CompressionType.LZ4_COMPRESSION) - .setTableFormatConfig(createBlockBasedTableConfig(configuration)))) - .collect(Collectors.toList()); + trimmedSegments.stream().map(this::createColumnDescriptor).collect(Collectors.toList()); columnDescriptors.add( new ColumnFamilyDescriptor( DEFAULT_COLUMN.getBytes(StandardCharsets.UTF_8), @@ -178,6 +169,24 @@ public RocksDBColumnarKeyValueStorage( } } + private ColumnFamilyDescriptor createColumnDescriptor(final SegmentIdentifier segment) { + final var options = + new ColumnFamilyOptions() + .setTtl(0) + .setCompressionType(CompressionType.LZ4_COMPRESSION) + .setTableFormatConfig(createBlockBasedTableConfig(configuration)); + + if (segment.containsStaticData()) { + options + .setEnableBlobFiles(true) + .setEnableBlobGarbageCollection(false) + .setMinBlobSize(100) + .setBlobCompressionType(CompressionType.LZ4_COMPRESSION); + } + + return new ColumnFamilyDescriptor(segment.getId(), options); + } + private void setGlobalOptions(final RocksDBConfiguration configuration, final Statistics stats) { options = new DBOptions(); options diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java index deb88797122..d58da12ce2f 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorageTest.java @@ -261,14 +261,22 @@ public void dbWillBeBackwardIncompatibleAfterExperimentalSegmentsAreAdded() thro public enum TestSegment implements SegmentIdentifier { FOO(new byte[] {1}), BAR(new byte[] {2}), - EXPERIMENTAL(new byte[] {3}); + EXPERIMENTAL(new byte[] {3}), + + STATIC_DATA(new byte[] {4}, true); private final byte[] id; private final String nameAsUtf8; + private final boolean containsStaticData; TestSegment(final byte[] id) { + this(id, false); + } + + TestSegment(final byte[] id, final boolean containsStaticData) { this.id = id; this.nameAsUtf8 = new String(id, StandardCharsets.UTF_8); + this.containsStaticData = containsStaticData; } @Override @@ -280,6 +288,11 @@ public String getName() { public byte[] getId() { return id; } + + @Override + public boolean containsStaticData() { + return containsStaticData; + } } protected abstract SegmentedKeyValueStorage createSegmentedStore()