From 9d9afedc3a830a9f83e493cb74f589a734124630 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Wed, 12 Jul 2023 20:10:53 -0700 Subject: [PATCH] a million tiny little cuts, but compiles many broken tests still Signed-off-by: garyschulte --- .../besu/cli/CommandTestAbstract.java | 1 - .../storage/StorageSubCommandTest.java | 1 - .../controller/BesuControllerBuilderTest.java | 1 - .../MergeBesuControllerBuilderTest.java | 1 - .../QbftBesuControllerBuilderTest.java | 1 - .../besu/services/BesuEventsImplTest.java | 1 - ...ewBlockHeadersSubscriptionServiceTest.java | 1 - .../worldstate/PrunerIntegrationTest.java | 1 - ...nsaiSnapshotWorldStateKeyValueStorage.java | 25 +- .../BonsaiWorldStateKeyValueStorage.java | 147 +++++------ .../storage/BonsaiWorldStateLayerStorage.java | 20 +- .../storage/flat/FlatDbReaderStrategy.java | 44 ++-- .../flat/FullFlatDbReaderStrategy.java | 17 +- .../flat/PartialFlatDbReaderStrategy.java | 14 +- .../bonsai/worldview/BonsaiWorldState.java | 48 +++- .../keyvalue/KeyValueStorageProvider.java | 6 +- .../KeyValueStorageProviderBuilder.java | 1 - .../core/InMemoryKeyValueStorageProvider.java | 1 - .../BonsaiWorldStateKeyValueStorageTest.java | 12 +- .../besu/ethereum/bonsai/LogRollingTests.java | 1 - .../besu/ethereum/bonsai/RollingImport.java | 1 - .../ethereum/chain/ChainDataPrunerTest.java | 1 - .../ethereum/chain/DefaultBlockchainTest.java | 1 - .../mainnet/PrivacyBlockProcessorTest.java | 1 - .../PrivateStateKeyValueStorageTest.java | 2 - .../PrivateStorageMigrationTest.java | 1 - .../proof/WorldStateProofProviderTest.java | 1 - .../WorldStateRangeProofProviderTest.java | 1 - ...oragePrefixedKeyBlockchainStorageTest.java | 1 - .../KeyValueStorageWorldStateStorageTest.java | 1 - .../DefaultMutableWorldStateTest.java | 1 - .../worldstate/MarkSweepPrunerTest.java | 1 - .../besu/ethereum/worldstate/PrunerTest.java | 1 - .../backwardsync/BackwardSyncContextTest.java | 1 - .../backwardsync/BackwardSyncStepTest.java | 1 - .../backwardsync/ForwardSyncStepTest.java | 1 - .../InMemoryBackwardChainTest.java | 1 - .../CheckPointBlockImportStepTest.java | 1 - .../FastWorldDownloadStateTest.java | 1 - .../FastWorldStateDownloaderTest.java | 1 - .../eth/sync/snapsync/RangeManagerTest.java | 1 - .../snapsync/SnapWorldDownloadStateTest.java | 1 - .../eth/sync/snapsync/StackTrieTest.java | 1 - ...ntFlatDatabaseHealingRangeRequestTest.java | 1 - .../RangeStorageEntriesCollectorTest.java | 1 - .../ethereum/trie/TrieNodeDecoderTest.java | 1 - .../AbstractMerklePatriciaTrieTest.java | 1 - .../StoredMerklePatriciaTrieTest.java | 1 - .../RocksDBKeyValueStorageFactory.java | 36 ++- ...imisticRocksDBColumnarKeyValueStorage.java | 1 + .../RocksDBColumnarKeyValueStorage.java | 1 - .../kvstore/InMemoryKeyValueStorage.java | 228 +++--------------- .../InMemoryKeyValueStorageAdapter.java | 75 ------ .../kvstore/LayeredKeyValueStorage.java | 15 +- 54 files changed, 223 insertions(+), 506 deletions(-) delete mode 100644 services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorageAdapter.java diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java index 3b9aafe35dc..c7c6558da1a 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java @@ -80,7 +80,6 @@ import org.hyperledger.besu.services.SecurityModuleServiceImpl; import org.hyperledger.besu.services.StorageServiceImpl; import org.hyperledger.besu.services.TransactionSelectionServiceImpl; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.io.ByteArrayOutputStream; import java.io.File; diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommandTest.java index a8a1bedc649..22e57ffc3b0 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommandTest.java @@ -29,7 +29,6 @@ import org.hyperledger.besu.cli.CommandTestAbstract; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java index bd8d06d2a5e..5126cf1e191 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java @@ -55,7 +55,6 @@ import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.ObservableMetricsSystem; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.math.BigInteger; import java.time.Clock; diff --git a/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java index 15fb3fd3ec2..dcc47c9b36a 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java @@ -58,7 +58,6 @@ import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.ObservableMetricsSystem; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.math.BigInteger; import java.time.Clock; diff --git a/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java index 395abeb22d6..23fe05c091c 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java @@ -53,7 +53,6 @@ import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.ObservableMetricsSystem; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.math.BigInteger; import java.time.Clock; diff --git a/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java b/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java index 68170f3258c..a789a8cdabd 100644 --- a/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java +++ b/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java @@ -62,7 +62,6 @@ import org.hyperledger.besu.plugin.data.PropagatedBlockContext; import org.hyperledger.besu.plugin.data.SyncStatus; import org.hyperledger.besu.plugin.data.Transaction; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.hyperledger.besu.testutil.TestClock; import java.math.BigInteger; diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/blockheaders/NewBlockHeadersSubscriptionServiceTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/blockheaders/NewBlockHeadersSubscriptionServiceTest.java index 0628cefadbf..fd4390cb670 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/blockheaders/NewBlockHeadersSubscriptionServiceTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/blockheaders/NewBlockHeadersSubscriptionServiceTest.java @@ -37,7 +37,6 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.List; import java.util.function.Consumer; diff --git a/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/worldstate/PrunerIntegrationTest.java b/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/worldstate/PrunerIntegrationTest.java index 86f2694c3c0..6ae0e3a4de8 100644 --- a/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/worldstate/PrunerIntegrationTest.java +++ b/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/worldstate/PrunerIntegrationTest.java @@ -33,7 +33,6 @@ import org.hyperledger.besu.ethereum.worldstate.Pruner.PruningPhase; import org.hyperledger.besu.evm.worldstate.WorldState; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.hyperledger.besu.testutil.MockExecutorService; import java.util.ArrayList; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java index 82d68e0b766..2e89e0f791a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java @@ -42,19 +42,13 @@ public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKey public BonsaiSnapshotWorldStateKeyValueStorage( final BonsaiWorldStateKeyValueStorage parentWorldStateStorage, - final SnappedKeyValueStorage accountStorage, - final SnappedKeyValueStorage codeStorage, - final SnappedKeyValueStorage storageStorage, - final SnappedKeyValueStorage trieBranchStorage, + final SnappedKeyValueStorage segmentedWorldStateStorage, final KeyValueStorage trieLogStorage, final ObservableMetricsSystem metricsSystem) { super( parentWorldStateStorage.flatDbMode, parentWorldStateStorage.flatDbReaderStrategy, - accountStorage, - codeStorage, - storageStorage, - trieBranchStorage, + segmentedWorldStateStorage, trieLogStorage, metricsSystem); this.parentWorldStateStorage = parentWorldStateStorage; @@ -66,10 +60,7 @@ public BonsaiSnapshotWorldStateKeyValueStorage( final ObservableMetricsSystem metricsSystem) { this( worldStateStorage, - ((SnappableKeyValueStorage) worldStateStorage.accountStorage).takeSnapshot(), - ((SnappableKeyValueStorage) worldStateStorage.codeStorage).takeSnapshot(), - ((SnappableKeyValueStorage) worldStateStorage.storageStorage).takeSnapshot(), - ((SnappableKeyValueStorage) worldStateStorage.trieBranchStorage).takeSnapshot(), + ((SnappableKeyValueStorage) worldStateStorage.composedWorldStateStorage).takeSnapshot(), worldStateStorage.trieLogStorage, metricsSystem); } @@ -85,10 +76,7 @@ private boolean isClosedGet() { @Override public BonsaiUpdater updater() { return new Updater( - ((SnappedKeyValueStorage) accountStorage).getSnapshotTransaction(), - ((SnappedKeyValueStorage) codeStorage).getSnapshotTransaction(), - ((SnappedKeyValueStorage) storageStorage).getSnapshotTransaction(), - ((SnappedKeyValueStorage) trieBranchStorage).getSnapshotTransaction(), + ((SnappedKeyValueStorage) composedWorldStateStorage).getSnapshotTransaction(), trieLogStorage.startTransaction()); } @@ -223,10 +211,7 @@ protected synchronized void doClose() throws Exception { subscribers.forEach(BonsaiStorageSubscriber::onCloseStorage); // close all of the SnappedKeyValueStorages: - accountStorage.close(); - codeStorage.close(); - storageStorage.close(); - trieBranchStorage.close(); + composedWorldStateStorage.close(); // unsubscribe the parent worldstate parentWorldStateStorage.unSubscribe(subscribeParentId); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java index 1b8def7fc3d..2c867425171 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java @@ -14,6 +14,11 @@ */ package org.hyperledger.besu.ethereum.bonsai.storage; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; + import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.bonsai.storage.flat.FlatDbReaderStrategy; @@ -29,9 +34,13 @@ import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; +import org.hyperledger.besu.services.kvstore.SegmentedKeyValueStorageAdapter; import org.hyperledger.besu.util.Subscribers; import java.nio.charset.StandardCharsets; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; @@ -60,10 +69,7 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC protected FlatDbMode flatDbMode; protected FlatDbReaderStrategy flatDbReaderStrategy; - protected final KeyValueStorage accountStorage; - protected final KeyValueStorage codeStorage; - protected final KeyValueStorage storageStorage; - protected final KeyValueStorage trieBranchStorage; + protected final SegmentedKeyValueStorage composedWorldStateStorage; protected final KeyValueStorage trieLogStorage; protected final ObservableMetricsSystem metricsSystem; @@ -76,14 +82,9 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC public BonsaiWorldStateKeyValueStorage( final StorageProvider provider, final ObservableMetricsSystem metricsSystem) { - this.accountStorage = - provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE); - this.codeStorage = - provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE); - this.storageStorage = - provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE); - this.trieBranchStorage = - provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE); + this.composedWorldStateStorage = + provider.getStorageBySegmentIdentifiers( + List.of(ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE)); this.trieLogStorage = provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); this.metricsSystem = metricsSystem; @@ -93,18 +94,12 @@ public BonsaiWorldStateKeyValueStorage( public BonsaiWorldStateKeyValueStorage( final FlatDbMode flatDbMode, final FlatDbReaderStrategy flatDbReaderStrategy, - final KeyValueStorage accountStorage, - final KeyValueStorage codeStorage, - final KeyValueStorage storageStorage, - final KeyValueStorage trieBranchStorage, + final SegmentedKeyValueStorage composedWorldStateStorage, final KeyValueStorage trieLogStorage, final ObservableMetricsSystem metricsSystem) { this.flatDbMode = flatDbMode; this.flatDbReaderStrategy = flatDbReaderStrategy; - this.accountStorage = accountStorage; - this.codeStorage = codeStorage; - this.storageStorage = storageStorage; - this.trieBranchStorage = trieBranchStorage; + this.composedWorldStateStorage = composedWorldStateStorage; this.trieLogStorage = trieLogStorage; this.metricsSystem = metricsSystem; } @@ -112,8 +107,8 @@ public BonsaiWorldStateKeyValueStorage( public void loadFlatDbStrategy() { this.flatDbMode = FlatDbMode.fromVersion( - trieBranchStorage - .get(FLAT_DB_MODE) + composedWorldStateStorage + .get(TRIE_BRANCH_STORAGE, FLAT_DB_MODE) .map(Bytes::wrap) .orElse( FlatDbMode.PARTIAL @@ -146,7 +141,7 @@ public Optional getCode(final Bytes32 codeHash, final Hash accountHash) { if (codeHash.equals(Hash.EMPTY)) { return Optional.of(Bytes.EMPTY); } else { - return getFlatDbReaderStrategy().getCode(codeHash, accountHash, codeStorage); + return getFlatDbReaderStrategy().getCode(codeHash, accountHash, composedWorldStateStorage); } } @@ -156,7 +151,7 @@ public Optional getAccount(final Hash accountHash) { this::getWorldStateRootHash, this::getAccountStateTrieNode, accountHash, - accountStorage); + composedWorldStateStorage); } @Override @@ -164,8 +159,8 @@ public Optional getAccountStateTrieNode(final Bytes location, final Bytes if (nodeHash.equals(MerkleTrie.EMPTY_TRIE_NODE_HASH)) { return Optional.of(MerkleTrie.EMPTY_TRIE_NODE); } else { - return trieBranchStorage - .get(location.toArrayUnsafe()) + return composedWorldStateStorage + .get(TRIE_BRANCH_STORAGE, location.toArrayUnsafe()) .map(Bytes::wrap) .filter(b -> Hash.hash(b).equals(nodeHash)); } @@ -186,8 +181,8 @@ public Optional getAccountStorageTrieNode( if (maybeNodeHash.filter(hash -> hash.equals(MerkleTrie.EMPTY_TRIE_NODE_HASH)).isPresent()) { return Optional.of(MerkleTrie.EMPTY_TRIE_NODE); } else { - return trieBranchStorage - .get(Bytes.concatenate(accountHash, location).toArrayUnsafe()) + return composedWorldStateStorage + .get(TRIE_BRANCH_STORAGE, Bytes.concatenate(accountHash, location).toArrayUnsafe()) .map(Bytes::wrap) .filter(data -> maybeNodeHash.map(hash -> Hash.hash(data).equals(hash)).orElse(true)); } @@ -204,15 +199,15 @@ public Optional getTrieLog(final Hash blockHash) { } public Optional getStateTrieNode(final Bytes location) { - return trieBranchStorage.get(location.toArrayUnsafe()).map(Bytes::wrap); + return composedWorldStateStorage.get(TRIE_BRANCH_STORAGE, location.toArrayUnsafe()).map(Bytes::wrap); } public Optional getWorldStateRootHash() { - return trieBranchStorage.get(WORLD_ROOT_HASH_KEY).map(Bytes::wrap); + return composedWorldStateStorage.get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY).map(Bytes::wrap); } public Optional getWorldStateBlockHash() { - return trieBranchStorage.get(WORLD_BLOCK_HASH_KEY).map(Bytes32::wrap).map(Hash::wrap); + return composedWorldStateStorage.get(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY).map(Bytes32::wrap).map(Hash::wrap); } public Optional getStorageValueByStorageSlotKey( @@ -240,21 +235,21 @@ public Optional getStorageValueByStorageSlotKey( (location, hash) -> getAccountStorageTrieNode(accountHash, location, hash), accountHash, storageSlotKey, - storageStorage); + composedWorldStateStorage); } @Override public Map streamFlatAccounts( final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { return getFlatDbReaderStrategy() - .streamAccountFlatDatabase(accountStorage, startKeyHash, endKeyHash, max); + .streamAccountFlatDatabase(composedWorldStateStorage, startKeyHash, endKeyHash, max); } @Override public Map streamFlatStorages( final Hash accountHash, final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { return getFlatDbReaderStrategy() - .streamStorageFlatDatabase(storageStorage, accountHash, startKeyHash, endKeyHash, max); + .streamStorageFlatDatabase(composedWorldStateStorage, accountHash, startKeyHash, endKeyHash, max); } @Override @@ -264,23 +259,23 @@ public Optional getNodeData(final Bytes location, final Bytes32 hash) { @Override public boolean isWorldStateAvailable(final Bytes32 rootHash, final Hash blockHash) { - return trieBranchStorage - .get(WORLD_ROOT_HASH_KEY) + return composedWorldStateStorage + .get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY) .map(Bytes32::wrap) .map(hash -> hash.equals(rootHash) || trieLogStorage.containsKey(blockHash.toArrayUnsafe())) .orElse(false); } public void upgradeToFullFlatDbMode() { - final KeyValueStorageTransaction transaction = trieBranchStorage.startTransaction(); - transaction.put(FLAT_DB_MODE, FlatDbMode.FULL.getVersion().toArrayUnsafe()); + final SegmentedKeyValueStorageTransaction transaction = composedWorldStateStorage.startTransaction(); + transaction.put(TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.FULL.getVersion().toArrayUnsafe()); transaction.commit(); loadFlatDbStrategy(); // force reload of flat db reader strategy } public void downgradeToPartialFlatDbMode() { - final KeyValueStorageTransaction transaction = trieBranchStorage.startTransaction(); - transaction.put(FLAT_DB_MODE, FlatDbMode.PARTIAL.getVersion().toArrayUnsafe()); + final SegmentedKeyValueStorageTransaction transaction = composedWorldStateStorage.startTransaction(); + transaction.put(TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.PARTIAL.getVersion().toArrayUnsafe()); transaction.commit(); loadFlatDbStrategy(); // force reload of flat db reader strategy } @@ -288,8 +283,8 @@ public void downgradeToPartialFlatDbMode() { @Override public void clear() { subscribers.forEach(BonsaiStorageSubscriber::onClearStorage); - getFlatDbReaderStrategy().clearAll(accountStorage, storageStorage, codeStorage); - trieBranchStorage.clear(); + getFlatDbReaderStrategy().clearAll(composedWorldStateStorage); + composedWorldStateStorage.clear(TRIE_BRANCH_STORAGE); trieLogStorage.clear(); loadFlatDbStrategy(); // force reload of flat db reader strategy } @@ -303,16 +298,13 @@ public void clearTrieLog() { @Override public void clearFlatDatabase() { subscribers.forEach(BonsaiStorageSubscriber::onClearFlatDatabaseStorage); - getFlatDbReaderStrategy().resetOnResync(accountStorage, storageStorage); + getFlatDbReaderStrategy().resetOnResync(composedWorldStateStorage); } @Override public BonsaiUpdater updater() { return new Updater( - accountStorage.startTransaction(), - codeStorage.startTransaction(), - storageStorage.startTransaction(), - trieBranchStorage.startTransaction(), + composedWorldStateStorage.startTransaction(), trieLogStorage.startTransaction()); } @@ -343,36 +335,27 @@ BonsaiUpdater putStorageValueBySlotHash( void removeStorageValueBySlotHash(final Hash accountHash, final Hash slotHash); - KeyValueStorageTransaction getTrieBranchStorageTransaction(); + SegmentedKeyValueStorageTransaction getWorldStateTransaction(); KeyValueStorageTransaction getTrieLogStorageTransaction(); } public static class Updater implements BonsaiUpdater { - private final KeyValueStorageTransaction accountStorageTransaction; - private final KeyValueStorageTransaction codeStorageTransaction; - private final KeyValueStorageTransaction storageStorageTransaction; - private final KeyValueStorageTransaction trieBranchStorageTransaction; + private final SegmentedKeyValueStorageTransaction composedWorldStateTransaction; private final KeyValueStorageTransaction trieLogStorageTransaction; public Updater( - final KeyValueStorageTransaction accountStorageTransaction, - final KeyValueStorageTransaction codeStorageTransaction, - final KeyValueStorageTransaction storageStorageTransaction, - final KeyValueStorageTransaction trieBranchStorageTransaction, + final SegmentedKeyValueStorageTransaction composedWorldStateTransaction, final KeyValueStorageTransaction trieLogStorageTransaction) { - this.accountStorageTransaction = accountStorageTransaction; - this.codeStorageTransaction = codeStorageTransaction; - this.storageStorageTransaction = storageStorageTransaction; - this.trieBranchStorageTransaction = trieBranchStorageTransaction; + this.composedWorldStateTransaction = composedWorldStateTransaction; this.trieLogStorageTransaction = trieLogStorageTransaction; } @Override public BonsaiUpdater removeCode(final Hash accountHash) { - codeStorageTransaction.remove(accountHash.toArrayUnsafe()); + composedWorldStateTransaction.remove(CODE_STORAGE, accountHash.toArrayUnsafe()); return this; } @@ -382,13 +365,13 @@ public BonsaiUpdater putCode(final Hash accountHash, final Bytes32 codeHash, fin // Don't save empty values return this; } - codeStorageTransaction.put(accountHash.toArrayUnsafe(), code.toArrayUnsafe()); + composedWorldStateTransaction.put(CODE_STORAGE, accountHash.toArrayUnsafe(), code.toArrayUnsafe()); return this; } @Override public BonsaiUpdater removeAccountInfoState(final Hash accountHash) { - accountStorageTransaction.remove(accountHash.toArrayUnsafe()); + composedWorldStateTransaction.remove(ACCOUNT_INFO_STATE, accountHash.toArrayUnsafe()); return this; } @@ -398,16 +381,16 @@ public BonsaiUpdater putAccountInfoState(final Hash accountHash, final Bytes acc // Don't save empty values return this; } - accountStorageTransaction.put(accountHash.toArrayUnsafe(), accountValue.toArrayUnsafe()); + composedWorldStateTransaction.put(ACCOUNT_INFO_STATE, accountHash.toArrayUnsafe(), accountValue.toArrayUnsafe()); return this; } @Override public WorldStateStorage.Updater saveWorldState( final Bytes blockHash, final Bytes32 nodeHash, final Bytes node) { - trieBranchStorageTransaction.put(Bytes.EMPTY.toArrayUnsafe(), node.toArrayUnsafe()); - trieBranchStorageTransaction.put(WORLD_ROOT_HASH_KEY, nodeHash.toArrayUnsafe()); - trieBranchStorageTransaction.put(WORLD_BLOCK_HASH_KEY, blockHash.toArrayUnsafe()); + composedWorldStateTransaction.put(TRIE_BRANCH_STORAGE, Bytes.EMPTY.toArrayUnsafe(), node.toArrayUnsafe()); + composedWorldStateTransaction.put(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY, nodeHash.toArrayUnsafe()); + composedWorldStateTransaction.put(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY, blockHash.toArrayUnsafe()); return this; } @@ -418,13 +401,13 @@ public BonsaiUpdater putAccountStateTrieNode( // Don't save empty nodes return this; } - trieBranchStorageTransaction.put(location.toArrayUnsafe(), node.toArrayUnsafe()); + composedWorldStateTransaction.put(TRIE_BRANCH_STORAGE, location.toArrayUnsafe(), node.toArrayUnsafe()); return this; } @Override public BonsaiUpdater removeAccountStateTrieNode(final Bytes location, final Bytes32 nodeHash) { - trieBranchStorageTransaction.remove(location.toArrayUnsafe()); + composedWorldStateTransaction.remove(ACCOUNT_INFO_STATE, location.toArrayUnsafe()); return this; } @@ -435,7 +418,7 @@ public synchronized BonsaiUpdater putAccountStorageTrieNode( // Don't save empty nodes return this; } - trieBranchStorageTransaction.put( + composedWorldStateTransaction.put(TRIE_BRANCH_STORAGE, Bytes.concatenate(accountHash, location).toArrayUnsafe(), node.toArrayUnsafe()); return this; } @@ -443,7 +426,7 @@ public synchronized BonsaiUpdater putAccountStorageTrieNode( @Override public synchronized BonsaiUpdater putStorageValueBySlotHash( final Hash accountHash, final Hash slotHash, final Bytes storage) { - storageStorageTransaction.put( + composedWorldStateTransaction.put(ACCOUNT_STORAGE_STORAGE, Bytes.concatenate(accountHash, slotHash).toArrayUnsafe(), storage.toArrayUnsafe()); return this; } @@ -451,12 +434,12 @@ public synchronized BonsaiUpdater putStorageValueBySlotHash( @Override public synchronized void removeStorageValueBySlotHash( final Hash accountHash, final Hash slotHash) { - storageStorageTransaction.remove(Bytes.concatenate(accountHash, slotHash).toArrayUnsafe()); + composedWorldStateTransaction.remove(ACCOUNT_STORAGE_STORAGE, Bytes.concatenate(accountHash, slotHash).toArrayUnsafe()); } @Override - public KeyValueStorageTransaction getTrieBranchStorageTransaction() { - return trieBranchStorageTransaction; + public SegmentedKeyValueStorageTransaction getWorldStateTransaction() { + return composedWorldStateTransaction; } @Override @@ -466,19 +449,14 @@ public KeyValueStorageTransaction getTrieLogStorageTransaction() { @Override public void commit() { - accountStorageTransaction.commit(); - codeStorageTransaction.commit(); - storageStorageTransaction.commit(); - trieBranchStorageTransaction.commit(); + // write the log ahead, then the worldstate trieLogStorageTransaction.commit(); + composedWorldStateTransaction.commit(); } @Override public void rollback() { - accountStorageTransaction.rollback(); - codeStorageTransaction.rollback(); - storageStorageTransaction.rollback(); - trieBranchStorageTransaction.rollback(); + composedWorldStateTransaction.rollback(); trieLogStorageTransaction.rollback(); } } @@ -521,10 +499,7 @@ protected synchronized void doClose() throws Exception { subscribers.forEach(BonsaiStorageSubscriber::onCloseStorage); // close all of the KeyValueStorages: - accountStorage.close(); - codeStorage.close(); - storageStorage.close(); - trieBranchStorage.close(); + composedWorldStateStorage.close(); trieLogStorage.close(); // set storage closed diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java index 2d6a6c407e6..a5ec192ea58 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java @@ -27,29 +27,20 @@ public class BonsaiWorldStateLayerStorage extends BonsaiSnapshotWorldStateKeyVal public BonsaiWorldStateLayerStorage(final BonsaiWorldStateKeyValueStorage parent) { this( - new LayeredKeyValueStorage(parent.accountStorage), - new LayeredKeyValueStorage(parent.codeStorage), - new LayeredKeyValueStorage(parent.storageStorage), - new LayeredKeyValueStorage(parent.trieBranchStorage), + new LayeredKeyValueStorage(parent.composedWorldStateStorage), parent.trieLogStorage, parent, parent.metricsSystem); } public BonsaiWorldStateLayerStorage( - final SnappedKeyValueStorage accountStorage, - final SnappedKeyValueStorage codeStorage, - final SnappedKeyValueStorage storageStorage, - final SnappedKeyValueStorage trieBranchStorage, + final SnappedKeyValueStorage composedWorldStateStorage, final KeyValueStorage trieLogStorage, final BonsaiWorldStateKeyValueStorage parent, final ObservableMetricsSystem metricsSystem) { super( parent, - accountStorage, - codeStorage, - storageStorage, - trieBranchStorage, + composedWorldStateStorage, trieLogStorage, metricsSystem); } @@ -62,10 +53,7 @@ public FlatDbMode getFlatDbMode() { @Override public BonsaiWorldStateLayerStorage clone() { return new BonsaiWorldStateLayerStorage( - ((LayeredKeyValueStorage) accountStorage).clone(), - ((LayeredKeyValueStorage) codeStorage).clone(), - ((LayeredKeyValueStorage) storageStorage).clone(), - ((LayeredKeyValueStorage) trieBranchStorage).clone(), + ((LayeredKeyValueStorage) composedWorldStateStorage).clone(), trieLogStorage, parentWorldStateStorage, metricsSystem); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java index 86e60ae4d6c..2e3e2d7ad12 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java @@ -15,6 +15,11 @@ */ package org.hyperledger.besu.ethereum.bonsai.storage.flat; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; + import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.trie.NodeLoader; @@ -22,6 +27,7 @@ import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; import java.util.Map; import java.util.Optional; @@ -84,7 +90,7 @@ public abstract Optional getAccount( Supplier> worldStateRootHashSupplier, NodeLoader nodeLoader, Hash accountHash, - KeyValueStorage accountStorage); + SegmentedKeyValueStorage storage); /* * Retrieves the storage value for the given account hash and storage slot key, using the world state root hash supplier, storage root supplier, and node loader. @@ -96,46 +102,44 @@ public abstract Optional getStorageValueByStorageSlotKey( NodeLoader nodeLoader, Hash accountHash, StorageSlotKey storageSlotKey, - KeyValueStorage storageStorage); + SegmentedKeyValueStorage storageStorage); /* * Retrieves the code data for the given code hash and account hash. */ public Optional getCode( - final Bytes32 codeHash, final Hash accountHash, final KeyValueStorage codeStorage) { + final Bytes32 codeHash, final Hash accountHash, final SegmentedKeyValueStorage storage) { if (codeHash.equals(Hash.EMPTY)) { return Optional.of(Bytes.EMPTY); } else { - return codeStorage - .get(accountHash.toArrayUnsafe()) + return storage + .get(TRIE_BRANCH_STORAGE, accountHash.toArrayUnsafe()) .map(Bytes::wrap) .filter(b -> Hash.hash(b).equals(codeHash)); } } public void clearAll( - final KeyValueStorage accountStorage, - final KeyValueStorage storageStorage, - final KeyValueStorage codeStorage) { - accountStorage.clear(); - storageStorage.clear(); - codeStorage.clear(); + final SegmentedKeyValueStorage storage) { + storage.clear(ACCOUNT_INFO_STATE); + storage.clear(ACCOUNT_STORAGE_STORAGE); + storage.clear(CODE_STORAGE); } public void resetOnResync( - final KeyValueStorage accountStorage, final KeyValueStorage storageStorage) { - accountStorage.clear(); - storageStorage.clear(); + final SegmentedKeyValueStorage storage) { + storage.clear(ACCOUNT_INFO_STATE); + storage.clear(ACCOUNT_STORAGE_STORAGE); } public Map streamAccountFlatDatabase( - final KeyValueStorage accountStorage, + final SegmentedKeyValueStorage storage, final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { final Stream> pairStream = - accountStorage - .streamFromKey(startKeyHash.toArrayUnsafe()) + storage + .streamFromKey(ACCOUNT_INFO_STATE, startKeyHash.toArrayUnsafe()) .limit(max) .map(pair -> new Pair<>(Bytes32.wrap(pair.getKey()), Bytes.wrap(pair.getValue()))) .takeWhile(pair -> pair.getFirst().compareTo(endKeyHash) <= 0); @@ -148,14 +152,14 @@ public Map streamAccountFlatDatabase( } public Map streamStorageFlatDatabase( - final KeyValueStorage storageStorage, + final SegmentedKeyValueStorage storage, final Hash accountHash, final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { final Stream> pairStream = - storageStorage - .streamFromKey(Bytes.concatenate(accountHash, startKeyHash).toArrayUnsafe()) + storage + .streamFromKey(ACCOUNT_STORAGE_STORAGE, Bytes.concatenate(accountHash, startKeyHash).toArrayUnsafe()) .takeWhile(pair -> Bytes.wrap(pair.getKey()).slice(0, Hash.SIZE).equals(accountHash)) .limit(max) .map( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FullFlatDbReaderStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FullFlatDbReaderStrategy.java index e28ead510f1..34fee03b286 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FullFlatDbReaderStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FullFlatDbReaderStrategy.java @@ -15,13 +15,18 @@ */ package org.hyperledger.besu.ethereum.bonsai.storage.flat; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; + import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.trie.NodeLoader; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; import java.util.Optional; import java.util.function.Supplier; @@ -55,10 +60,10 @@ public Optional getAccount( final Supplier> worldStateRootHashSupplier, final NodeLoader nodeLoader, final Hash accountHash, - final KeyValueStorage accountStorage) { + final SegmentedKeyValueStorage storage) { getAccountCounter.inc(); final Optional accountFound = - accountStorage.get(accountHash.toArrayUnsafe()).map(Bytes::wrap); + storage.get(ACCOUNT_INFO_STATE, accountHash.toArrayUnsafe()).map(Bytes::wrap); if (accountFound.isPresent()) { getAccountFoundInFlatDatabaseCounter.inc(); } else { @@ -74,11 +79,11 @@ public Optional getStorageValueByStorageSlotKey( final NodeLoader nodeLoader, final Hash accountHash, final StorageSlotKey storageSlotKey, - final KeyValueStorage storageStorage) { + final SegmentedKeyValueStorage storage) { getStorageValueCounter.inc(); final Optional storageFound = - storageStorage - .get(Bytes.concatenate(accountHash, storageSlotKey.getSlotHash()).toArrayUnsafe()) + storage + .get(ACCOUNT_STORAGE_STORAGE, Bytes.concatenate(accountHash, storageSlotKey.getSlotHash()).toArrayUnsafe()) .map(Bytes::wrap); if (storageFound.isPresent()) { getStorageValueFlatDatabaseCounter.inc(); @@ -91,7 +96,7 @@ public Optional getStorageValueByStorageSlotKey( @Override public void resetOnResync( - final KeyValueStorage accountStorage, final KeyValueStorage storageStorage) { + final SegmentedKeyValueStorage storage) { // NOOP // not need to reset anything in full mode } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/PartialFlatDbReaderStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/PartialFlatDbReaderStrategy.java index 8ea0fbcde38..7f28549229e 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/PartialFlatDbReaderStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/PartialFlatDbReaderStrategy.java @@ -15,6 +15,9 @@ */ package org.hyperledger.besu.ethereum.bonsai.storage.flat; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; + import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.trie.NodeLoader; @@ -24,6 +27,7 @@ import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; import java.util.Optional; import java.util.function.Function; @@ -81,9 +85,9 @@ public Optional getAccount( final Supplier> worldStateRootHashSupplier, final NodeLoader nodeLoader, final Hash accountHash, - final KeyValueStorage accountStorage) { + final SegmentedKeyValueStorage storage) { getAccountCounter.inc(); - Optional response = accountStorage.get(accountHash.toArrayUnsafe()).map(Bytes::wrap); + Optional response = storage.get(ACCOUNT_INFO_STATE, accountHash.toArrayUnsafe()).map(Bytes::wrap); if (response.isEmpty()) { // after a snapsync/fastsync we only have the trie branches. final Optional worldStateRootHash = worldStateRootHashSupplier.get(); @@ -113,11 +117,11 @@ public Optional getStorageValueByStorageSlotKey( final NodeLoader nodeLoader, final Hash accountHash, final StorageSlotKey storageSlotKey, - final KeyValueStorage storageStorage) { + final SegmentedKeyValueStorage storage) { getStorageValueCounter.inc(); Optional response = - storageStorage - .get(Bytes.concatenate(accountHash, storageSlotKey.getSlotHash()).toArrayUnsafe()) + storage + .get(ACCOUNT_STORAGE_STORAGE, Bytes.concatenate(accountHash, storageSlotKey.getSlotHash()).toArrayUnsafe()) .map(Bytes::wrap); if (response.isEmpty()) { final Optional storageRoot = storageRootSupplier.get(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java index d3f14e8e72e..4af9aa6d95d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java @@ -19,6 +19,8 @@ import static org.hyperledger.besu.ethereum.bonsai.BonsaiAccount.fromRLP; import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; @@ -43,6 +45,8 @@ import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.plugin.services.exception.StorageException; import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; +import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; import java.util.Map; import java.util.Optional; @@ -182,7 +186,7 @@ private Hash calculateRootHash( bonsaiUpdater -> { accountTrie.commit( (location, hash, value) -> - writeTrieNode(bonsaiUpdater.getTrieBranchStorageTransaction(), location, value)); + writeTrieNode(ACCOUNT_INFO_STATE, bonsaiUpdater.getWorldStateTransaction(), location, value)); }); final Bytes32 rootHash = accountTrie.getRootHash(); return Hash.wrap(rootHash); @@ -391,17 +395,17 @@ public void persist(final BlockHeader blockHeader) { }; stateUpdater - .getTrieBranchStorageTransaction() - .put(WORLD_BLOCK_HASH_KEY, blockHeader.getHash().toArrayUnsafe()); + .getWorldStateTransaction() + .put(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY, blockHeader.getHash().toArrayUnsafe()); worldStateBlockHash = blockHeader.getHash(); } else { - stateUpdater.getTrieBranchStorageTransaction().remove(WORLD_BLOCK_HASH_KEY); + stateUpdater.getWorldStateTransaction().remove(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY); worldStateBlockHash = null; } stateUpdater - .getTrieBranchStorageTransaction() - .put(WORLD_ROOT_HASH_KEY, newWorldStateRootHash.toArrayUnsafe()); + .getWorldStateTransaction() + .put(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY, newWorldStateRootHash.toArrayUnsafe()); worldStateRootHash = newWorldStateRootHash; success = true; } finally { @@ -454,11 +458,35 @@ public void rollback() { } }; + static final SegmentedKeyValueStorageTransaction noOpSegmentedTx = + new SegmentedKeyValueStorageTransaction() { + + @Override + public void put(final SegmentIdentifier segmentIdentifier, final byte[] key, final byte[] value) { + // no-op + } + + @Override + public void remove(final SegmentIdentifier segmentIdentifier, final byte[] key) { + // no-op + } + + @Override + public void commit() throws StorageException { + // no-op + } + + @Override + public void rollback() { + // no-op + } + }; + @Override public Hash frontierRootHash() { return calculateRootHash( Optional.of( - new BonsaiWorldStateKeyValueStorage.Updater(noOpTx, noOpTx, noOpTx, noOpTx, noOpTx)), + new BonsaiWorldStateKeyValueStorage.Updater(noOpSegmentedTx, noOpTx)), accumulator.copy()); } @@ -483,9 +511,9 @@ protected Optional getAccountStateTrieNode(final Bytes location, final By return worldStateStorage.getAccountStateTrieNode(location, nodeHash); } - private void writeTrieNode( - final KeyValueStorageTransaction tx, final Bytes location, final Bytes value) { - tx.put(location.toArrayUnsafe(), value.toArrayUnsafe()); + private void writeTrieNode(final SegmentIdentifier segmentId, + final SegmentedKeyValueStorageTransaction tx, final Bytes location, final Bytes value) { + tx.put(segmentId, location.toArrayUnsafe(), value.toArrayUnsafe()); } protected Optional getStorageTrieNode( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java index 2f61a4a6d87..12c54497450 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java @@ -48,7 +48,6 @@ public class KeyValueStorageProvider implements StorageProvider { protected final Function, SegmentedKeyValueStorage> segmentedStorageCreator; private final KeyValueStorage worldStatePreimageStorage; private final boolean isWorldStateIterable; - private final boolean isWorldStateSnappable; protected final Map, SegmentedKeyValueStorage> storageInstances = new HashMap<>(); private final ObservableMetricsSystem metricsSystem; @@ -56,12 +55,10 @@ public KeyValueStorageProvider( final Function, SegmentedKeyValueStorage> segmentedStorageCreator, final KeyValueStorage worldStatePreimageStorage, final boolean segmentIsolationSupported, - final boolean storageSnapshotIsolationSupported, final ObservableMetricsSystem metricsSystem) { this.segmentedStorageCreator = segmentedStorageCreator; this.worldStatePreimageStorage = worldStatePreimageStorage; this.isWorldStateIterable = segmentIsolationSupported; - this.isWorldStateSnappable = storageSnapshotIsolationSupported; this.metricsSystem = metricsSystem; } @@ -124,7 +121,8 @@ public void close() throws IOException { .addArgument(storage.getKey().stream() .map(SegmentIdentifier::getName) .collect(Collectors.joining(","))) - .setCause(e); + .setCause(e) + .log(); } }); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProviderBuilder.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProviderBuilder.java index 0a9a8e7b8df..35720101206 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProviderBuilder.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProviderBuilder.java @@ -67,7 +67,6 @@ public KeyValueStorageProvider build() { segments -> storageFactory.create(segments, commonConfiguration, metricsSystem), worldStatePreImageStorage, storageFactory.isSegmentIsolationSupported(), - storageFactory.isSnapshotIsolationSupported(), (ObservableMetricsSystem) metricsSystem); } } diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java index a8f8e68f4d3..de488846d6f 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java @@ -42,7 +42,6 @@ public InMemoryKeyValueStorageProvider() { segmentIdentifiers -> new SegmentedInMemoryKeyValueStorage(), new InMemoryKeyValueStorage(), SEGMENT_ISOLATION_SUPPORTED, - SNAPSHOT_ISOLATION_UNSUPPORTED, new NoOpMetricsSystem()); } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java index a3314b2ee3b..0cb3b9c049a 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateKeyValueStorageTest.java @@ -197,7 +197,7 @@ public void getAccount_notLoadFromTrieWhenEmptyAndFlatDbFullMode() { // save world state root hash final BonsaiWorldStateKeyValueStorage.BonsaiUpdater updater = storage.updater(); updater - .getTrieBranchStorageTransaction() + .getWorldStateTransaction() .put(WORLD_ROOT_HASH_KEY, trie.getRootHash().toArrayUnsafe()); updater.commit(); @@ -225,7 +225,7 @@ public void getAccount_loadFromTrieWhenEmptyAndFlatDbPartialMode() { // save world state root hash final BonsaiWorldStateKeyValueStorage.BonsaiUpdater updater = storage.updater(); updater - .getTrieBranchStorageTransaction() + .getWorldStateTransaction() .put(WORLD_ROOT_HASH_KEY, trie.getRootHash().toArrayUnsafe()); updater.commit(); @@ -253,7 +253,7 @@ public void shouldUsePartialDBStrategyAfterDowngradingMode() { // save world state root hash final BonsaiWorldStateKeyValueStorage.BonsaiUpdater updater = storage.updater(); updater - .getTrieBranchStorageTransaction() + .getWorldStateTransaction() .put(WORLD_ROOT_HASH_KEY, trie.getRootHash().toArrayUnsafe()); updater.commit(); @@ -298,7 +298,7 @@ public void getStorage_loadFromTrieWhenEmptyWithPartialMode() { // save world state root hash final BonsaiWorldStateKeyValueStorage.BonsaiUpdater updater = storage.updater(); updater - .getTrieBranchStorageTransaction() + .getWorldStateTransaction() .put(WORLD_ROOT_HASH_KEY, trie.getRootHash().toArrayUnsafe()); updater.commit(); @@ -327,7 +327,7 @@ public void getStorage_loadFromTrieWhenEmptyWithFullMode() { // save world state root hash final BonsaiWorldStateKeyValueStorage.BonsaiUpdater updater = storage.updater(); updater - .getTrieBranchStorageTransaction() + .getWorldStateTransaction() .put(WORLD_ROOT_HASH_KEY, trie.getRootHash().toArrayUnsafe()); updater.commit(); @@ -387,7 +387,7 @@ public void isWorldStateAvailable_StateAvailableByRootHash() { final BonsaiWorldStateKeyValueStorage.BonsaiUpdater updater = storage.updater(); final Bytes rootHashKey = Bytes32.fromHexString("0x01"); - updater.getTrieBranchStorageTransaction().put(WORLD_ROOT_HASH_KEY, rootHashKey.toArrayUnsafe()); + updater.getWorldStateTransaction().put(WORLD_ROOT_HASH_KEY, rootHashKey.toArrayUnsafe()); updater.commit(); assertThat(storage.isWorldStateAvailable(Hash.wrap(Bytes32.wrap(rootHashKey)), Hash.EMPTY)) .isTrue(); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java index c6cc9b58c8f..e536fbc5d69 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java @@ -41,7 +41,6 @@ import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.Optional; import java.util.stream.Collectors; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java index 98e991574d8..8777fcdb8a0 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java @@ -28,7 +28,6 @@ import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.hyperledger.besu.util.io.RollingFileReader; import java.io.IOException; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/ChainDataPrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/ChainDataPrunerTest.java index 00d0797d39b..18a667205e4 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/ChainDataPrunerTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/ChainDataPrunerTest.java @@ -23,7 +23,6 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.List; import java.util.concurrent.AbstractExecutorService; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchainTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchainTest.java index b3523e1d9b7..b6059fec4ab 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchainTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchainTest.java @@ -33,7 +33,6 @@ import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.ArrayList; import java.util.Collections; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessorTest.java index 6f1796876e5..6129c848f90 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessorTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessorTest.java @@ -46,7 +46,6 @@ import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.evm.worldstate.WrappedEvmAccount; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.Collections; import java.util.Optional; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateStateKeyValueStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateStateKeyValueStorageTest.java index ce7fdd12311..21ea85c48e2 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateStateKeyValueStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/PrivateStateKeyValueStorageTest.java @@ -18,8 +18,6 @@ import static org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage.SCHEMA_VERSION_1_0_0; import static org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage.SCHEMA_VERSION_1_4_0; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; - import org.junit.Before; import org.junit.Test; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigrationTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigrationTest.java index b17bcddab72..b5e1e0269cf 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigrationTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateStorageMigrationTest.java @@ -53,7 +53,6 @@ import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.plugin.data.TransactionType; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.math.BigInteger; import java.util.ArrayList; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateProofProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateProofProviderTest.java index a6269f9bb24..1ceae887feb 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateProofProviderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateProofProviderTest.java @@ -25,7 +25,6 @@ import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.ArrayList; import java.util.Arrays; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateRangeProofProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateRangeProofProviderTest.java index e09e041b354..f6e053e0b7d 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateRangeProofProviderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateRangeProofProviderTest.java @@ -23,7 +23,6 @@ import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.ArrayList; import java.util.Iterator; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorageTest.java index 1e89581a095..5182302a631 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorageTest.java @@ -31,7 +31,6 @@ import org.hyperledger.besu.ethereum.chain.VariablesStorage.Keys; import org.hyperledger.besu.ethereum.core.BlockHeaderFunctions; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.Map; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageWorldStateStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageWorldStateStorageTest.java index 994964b2982..610f79be877 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageWorldStateStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageWorldStateStorageTest.java @@ -19,7 +19,6 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage.Updater; import org.hyperledger.besu.ethereum.trie.MerkleTrie; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.units.bigints.UInt256; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldStateTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldStateTest.java index d69beaa7951..82442a4ce04 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldStateTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldStateTest.java @@ -30,7 +30,6 @@ import org.hyperledger.besu.evm.worldstate.WorldState.StreamableAccount; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.ArrayList; import java.util.List; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPrunerTest.java index 107a74be968..bdbd3894ba8 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPrunerTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPrunerTest.java @@ -35,7 +35,6 @@ import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.evm.worldstate.WorldState; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.ArrayList; import java.util.HashMap; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/PrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/PrunerTest.java index fa90bf6a8e2..4c5ea97c0ec 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/PrunerTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/PrunerTest.java @@ -33,7 +33,6 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.hyperledger.besu.testutil.MockExecutorService; import java.util.List; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java index 7c744c6b520..4070232680b 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java @@ -49,7 +49,6 @@ import org.hyperledger.besu.ethereum.referencetests.ReferenceTestWorldState; import org.hyperledger.besu.plugin.data.TransactionType; import org.hyperledger.besu.plugin.services.MetricsSystem; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.nio.charset.StandardCharsets; import java.util.Collections; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStepTest.java index 9ddcaf7b8cb..390eb20ad38 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStepTest.java @@ -38,7 +38,6 @@ import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.nio.charset.StandardCharsets; import java.util.List; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java index cf03c676ad9..f2891ba7bdf 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java @@ -39,7 +39,6 @@ import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.referencetests.ReferenceTestWorldState; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.nio.charset.StandardCharsets; import java.util.Collections; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java index 405e06d1d35..18ec08cbebe 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java @@ -24,7 +24,6 @@ import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.nio.charset.StandardCharsets; import java.util.List; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointBlockImportStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointBlockImportStepTest.java index 1520edb0679..84e680c046d 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointBlockImportStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointBlockImportStepTest.java @@ -29,7 +29,6 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; import org.hyperledger.besu.plugin.services.MetricsSystem; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.ArrayList; import java.util.Collections; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java index 0a774292fd7..3ecc9f7b311 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java @@ -31,7 +31,6 @@ import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues; import org.hyperledger.besu.testutil.TestClock; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloaderTest.java index fec755a8849..4f7d8ceff9b 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloaderTest.java @@ -65,7 +65,6 @@ import org.hyperledger.besu.evm.account.AccountStorageEntry; import org.hyperledger.besu.evm.worldstate.WorldState; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues; import org.hyperledger.besu.testutil.MockExecutorService; import org.hyperledger.besu.testutil.TestClock; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManagerTest.java index e4e116b8a8c..888cff9ef79 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManagerTest.java @@ -24,7 +24,6 @@ import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.HashMap; import java.util.List; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java index ac513ebd97e..ccaec8bc628 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java @@ -43,7 +43,6 @@ import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues; import org.hyperledger.besu.testutil.TestClock; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrieTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrieTest.java index 26e5cd5af42..00e1da430c4 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrieTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrieTest.java @@ -22,7 +22,6 @@ import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.List; import java.util.TreeMap; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java index f1063d9ed7a..97196a8a795 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java @@ -33,7 +33,6 @@ import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.HashSet; import java.util.Iterator; diff --git a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/RangeStorageEntriesCollectorTest.java b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/RangeStorageEntriesCollectorTest.java index da44b541c87..820771815ea 100644 --- a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/RangeStorageEntriesCollectorTest.java +++ b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/RangeStorageEntriesCollectorTest.java @@ -15,7 +15,6 @@ package org.hyperledger.besu.ethereum.trie; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.List; diff --git a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/TrieNodeDecoderTest.java b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/TrieNodeDecoderTest.java index 1d654bf6d4c..0f7fb5cb6e0 100644 --- a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/TrieNodeDecoderTest.java +++ b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/TrieNodeDecoderTest.java @@ -20,7 +20,6 @@ import org.hyperledger.besu.ethereum.trie.patricia.TrieNodeDecoder; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.List; import java.util.Objects; diff --git a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/patricia/AbstractMerklePatriciaTrieTest.java b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/patricia/AbstractMerklePatriciaTrieTest.java index d843b467f36..4871e0c9034 100644 --- a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/patricia/AbstractMerklePatriciaTrieTest.java +++ b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/patricia/AbstractMerklePatriciaTrieTest.java @@ -26,7 +26,6 @@ import org.hyperledger.besu.ethereum.trie.Node; import org.hyperledger.besu.ethereum.trie.Proof; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.List; import java.util.Optional; diff --git a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/patricia/StoredMerklePatriciaTrieTest.java b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/patricia/StoredMerklePatriciaTrieTest.java index 40bef52fe4d..5051f29a790 100644 --- a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/patricia/StoredMerklePatriciaTrieTest.java +++ b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/patricia/StoredMerklePatriciaTrieTest.java @@ -20,7 +20,6 @@ import org.hyperledger.besu.ethereum.trie.MerkleStorage; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.nio.charset.StandardCharsets; import java.util.Optional; diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java index 90644bd7703..038f21e74fc 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java @@ -61,26 +61,26 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory { private RocksDBConfiguration rocksDBConfiguration; private final Supplier configuration; - private final List segments; + private final List configuredSegments; private final List ignorableSegments; /** * Instantiates a new RocksDb key value storage factory. * * @param configuration the configuration - * @param segments the segments + * @param configuredSegments the segments * @param ignorableSegments the ignorable segments * @param defaultVersion the default version * @param rocksDBMetricsFactory the rocks db metrics factory */ public RocksDBKeyValueStorageFactory( final Supplier configuration, - final List segments, + final List configuredSegments, final List ignorableSegments, final int defaultVersion, final RocksDBMetricsFactory rocksDBMetricsFactory) { this.configuration = configuration; - this.segments = segments; + this.configuredSegments = configuredSegments; this.ignorableSegments = ignorableSegments; this.defaultVersion = defaultVersion; this.rocksDBMetricsFactory = rocksDBMetricsFactory; @@ -90,46 +90,46 @@ public RocksDBKeyValueStorageFactory( * Instantiates a new RocksDb key value storage factory. * * @param configuration the configuration - * @param segments the segments + * @param configuredSegments the segments * @param defaultVersion the default version * @param rocksDBMetricsFactory the rocks db metrics factory */ public RocksDBKeyValueStorageFactory( final Supplier configuration, - final List segments, + final List configuredSegments, final int defaultVersion, final RocksDBMetricsFactory rocksDBMetricsFactory) { - this(configuration, segments, List.of(), defaultVersion, rocksDBMetricsFactory); + this(configuration, configuredSegments, List.of(), defaultVersion, rocksDBMetricsFactory); } /** * Instantiates a new Rocks db key value storage factory. * * @param configuration the configuration - * @param segments the segments + * @param configuredSegments the segments * @param ignorableSegments the ignorable segments * @param rocksDBMetricsFactory the rocks db metrics factory */ public RocksDBKeyValueStorageFactory( final Supplier configuration, - final List segments, + final List configuredSegments, final List ignorableSegments, final RocksDBMetricsFactory rocksDBMetricsFactory) { - this(configuration, segments, ignorableSegments, DEFAULT_VERSION, rocksDBMetricsFactory); + this(configuration, configuredSegments, ignorableSegments, DEFAULT_VERSION, rocksDBMetricsFactory); } /** * Instantiates a new Rocks db key value storage factory. * * @param configuration the configuration - * @param segments the segments + * @param configuredSegments the segments * @param rocksDBMetricsFactory the rocks db metrics factory */ public RocksDBKeyValueStorageFactory( final Supplier configuration, - final List segments, + final List configuredSegments, final RocksDBMetricsFactory rocksDBMetricsFactory) { - this(configuration, segments, List.of(), DEFAULT_VERSION, rocksDBMetricsFactory); + this(configuration, configuredSegments, List.of(), DEFAULT_VERSION, rocksDBMetricsFactory); } /** @@ -167,6 +167,16 @@ public SegmentedKeyValueStorage create( init(commonConfiguration); } + // safety check to see that segments all exist within configured segments + if (!configuredSegments.containsAll(segments)) { + throw new StorageException( + "Attempted to create storage for segments that are not configured: " + + segments.stream() + .filter(segment -> !configuredSegments.contains(segment)) + .map(SegmentIdentifier::toString) + .collect(Collectors.joining(", "))); + } + // It's probably a good idea for the creation logic to be entirely dependent on the database // version. Introducing intermediate booleans that represent database properties and dispatching // creation logic based on them is error-prone. diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/OptimisticRocksDBColumnarKeyValueStorage.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/OptimisticRocksDBColumnarKeyValueStorage.java index ae3b771a581..e78f5369865 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/OptimisticRocksDBColumnarKeyValueStorage.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/OptimisticRocksDBColumnarKeyValueStorage.java @@ -92,6 +92,7 @@ public SegmentedKeyValueStorageTransaction startTransaction() throws StorageExce * @return the RocksDb columnar key value snapshot * @throws StorageException the storage exception */ + @Override public RocksDBColumnarKeyValueSnapshot takeSnapshot() throws StorageException { throwIfClosed(); diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java index 9e45606bba9..12c920da13c 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java @@ -71,7 +71,6 @@ public abstract class RocksDBColumnarKeyValueStorage private static final Logger LOG = LoggerFactory.getLogger(RocksDBColumnarKeyValueStorage.class); static final String DEFAULT_COLUMN = "default"; - private static final String NO_SPACE_LEFT_ON_DEVICE = "No space left on device"; private static final int ROCKSDB_FORMAT_VERSION = 5; private static final long ROCKSDB_BLOCK_SIZE = 32768; /** RocksDb blockcache size when using the high spec option */ diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java index be57094fc39..c2e81ef68a3 100644 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java @@ -14,225 +14,61 @@ */ package org.hyperledger.besu.services.kvstore; -import static java.util.stream.Collectors.toUnmodifiableSet; +import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; -import org.hyperledger.besu.plugin.services.exception.StorageException; -import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; -import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; - -import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.Optional; -import java.util.Set; -import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.Predicate; -import java.util.stream.Stream; -import com.google.common.collect.ImmutableSet; -import org.apache.commons.lang3.tuple.Pair; import org.apache.tuweni.bytes.Bytes; /** - * Unsegmented In memory key value storage. This class will be deprecated in favor of InMemoryKeyValueStorageAdapter. + * InMemoryKeyValueStorage is just a wrapper around a single segment instance of SegmentedInMemoryKeyValueStorage. * */ -public class InMemoryKeyValueStorage implements KeyValueStorage { - - /** protected access for the backing hash map. */ - protected final Map> hashValueStore; - - /** protected access to the rw lock. */ - protected final ReadWriteLock rwLock = new ReentrantReadWriteLock(); - - /** Instantiates a new In memory key value storage. */ - public InMemoryKeyValueStorage() { - this(new HashMap<>()); - } - - /** - * Instantiates a new In memory key value storage. - * - * @param hashValueStore the hash value store - */ - protected InMemoryKeyValueStorage(final Map> hashValueStore) { - this.hashValueStore = hashValueStore; - } - - @Override - public void clear() { - final Lock lock = rwLock.writeLock(); - lock.lock(); - try { - hashValueStore.clear(); - } finally { - lock.unlock(); - } - } +public class InMemoryKeyValueStorage extends SegmentedKeyValueStorageAdapter { - @Override - public boolean containsKey(final byte[] key) throws StorageException { - return get(key).isPresent(); - } - - @Override - public Optional get(final byte[] key) throws StorageException { - final Lock lock = rwLock.readLock(); - lock.lock(); - try { - return hashValueStore.getOrDefault(Bytes.wrap(key), Optional.empty()); - } finally { - lock.unlock(); - } - } - - @Override - public Set getAllKeysThat(final Predicate returnCondition) { - return stream() - .filter(pair -> returnCondition.test(pair.getKey())) - .map(Pair::getKey) - .collect(toUnmodifiableSet()); - } - - @Override - public Set getAllValuesFromKeysThat(final Predicate returnCondition) { - return stream() - .filter(pair -> returnCondition.test(pair.getKey())) - .map(Pair::getValue) - .collect(toUnmodifiableSet()); - } - - @Override - public Stream> stream() { - final Lock lock = rwLock.readLock(); - lock.lock(); - try { - return ImmutableSet.copyOf(hashValueStore.entrySet()).stream() - .filter(bytesEntry -> bytesEntry.getValue().isPresent()) - .map( - bytesEntry -> - Pair.of(bytesEntry.getKey().toArrayUnsafe(), bytesEntry.getValue().get())); - } finally { - lock.unlock(); + private static final SegmentIdentifier SEGMENT_IDENTIFIER = new SegmentIdentifier() { + private static final String NAME = "SEGMENT_IDENTIFIER"; + @Override + public String getName() { + return NAME; } - } - - @Override - public Stream> streamFromKey(final byte[] startKey) { - return stream().filter(e -> Bytes.wrap(startKey).compareTo(Bytes.wrap(e.getKey())) <= 0); - } - @Override - public Stream streamKeys() { - final Lock lock = rwLock.readLock(); - lock.lock(); - try { - return ImmutableSet.copyOf(hashValueStore.entrySet()).stream() - .map(bytesEntry -> bytesEntry.getKey().toArrayUnsafe()); - } finally { - lock.unlock(); + @Override + public byte[] getId() { + return NAME.getBytes(StandardCharsets.UTF_8); } - } - @Override - public boolean tryDelete(final byte[] key) { - final Lock lock = rwLock.writeLock(); - if (lock.tryLock()) { - try { - hashValueStore.remove(Bytes.wrap(key)); - } finally { - lock.unlock(); - } - return true; + @Override + public boolean containsStaticData() { + return false; } - return false; - } - - @Override - public void close() {} + }; - @Override - public KeyValueStorageTransaction startTransaction() { - return new InMemoryTransaction(); + private static Map>> asSegmentMap(final Map> initialMap) { + final Map>> segmentMap = new HashMap<>(); + segmentMap.put(SEGMENT_IDENTIFIER, initialMap); + return segmentMap; } - @Override - public boolean isClosed() { - return false; - } + /** protected access to the rw lock. */ + protected final ReadWriteLock rwLock; - /** - * Key set. - * - * @return the set of keys - */ - public Set keySet() { - return Set.copyOf(hashValueStore.keySet()); + public InMemoryKeyValueStorage() { + super(SEGMENT_IDENTIFIER, new SegmentedInMemoryKeyValueStorage()); + rwLock = ((SegmentedInMemoryKeyValueStorage) storage).rwLock; } - /** In memory transaction. */ - public class InMemoryTransaction implements KeyValueStorageTransaction { - - /** protected access to updatedValues map for the transaction. */ - protected Map> updatedValues = new HashMap<>(); - /** protected access to deletedValues set for the transaction. */ - protected Set removedKeys = new HashSet<>(); - - @Override - public void put(final byte[] key, final byte[] value) { - updatedValues.put(Bytes.wrap(key), Optional.of(value)); - removedKeys.remove(Bytes.wrap(key)); - } - - @Override - public void remove(final byte[] key) { - removedKeys.add(Bytes.wrap(key)); - updatedValues.remove(Bytes.wrap(key)); - } - - @Override - public void commit() throws StorageException { - final Lock lock = rwLock.writeLock(); - lock.lock(); - try { - hashValueStore.putAll(updatedValues); - removedKeys.forEach(hashValueStore::remove); - updatedValues = null; - removedKeys = null; - } finally { - lock.unlock(); - } - } - - @Override - public void rollback() { - updatedValues.clear(); - removedKeys.clear(); - } + public InMemoryKeyValueStorage(final Map> initialMap) { + super(SEGMENT_IDENTIFIER, new SegmentedInMemoryKeyValueStorage(asSegmentMap(initialMap))); + rwLock = ((SegmentedInMemoryKeyValueStorage) storage).rwLock; } - /** - * Dump. - * - * @param ps the PrintStream where to report the dump - */ - public void dump(final PrintStream ps) { - final Lock lock = rwLock.readLock(); - lock.lock(); - try { - ImmutableSet.copyOf(hashValueStore.entrySet()).stream() - .filter(bytesEntry -> bytesEntry.getValue().isPresent()) - .forEach( - entry -> - ps.printf( - " %s : %s%n", - entry.getKey().toHexString(), - Bytes.wrap(entry.getValue().get()).toHexString())); - } finally { - lock.unlock(); - } - } +// private InMemoryKeyValueStorage(final SegmentedInMemoryKeyValueStorage storage) { +// super(SEGMENT_IDENTIFIER, storage); +// rwLock = storage.rwLock; +// } } diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorageAdapter.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorageAdapter.java deleted file mode 100644 index d91cd910e87..00000000000 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorageAdapter.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.services.kvstore; - -import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; - -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.locks.ReadWriteLock; - -import org.apache.tuweni.bytes.Bytes; - -/** - * InMemoryKeyValueStorage is just a wrapper around a single segment instance of SegmentedInMemoryKeyValueStorage. - * - * Eventually this will replace InMemoryKeyValueStorage. - */ -public class InMemoryKeyValueStorageAdapter extends SegmentedKeyValueStorageAdapter { - - private static final SegmentIdentifier SEGMENT_IDENTIFIER = new SegmentIdentifier() { - private static final String NAME = "SEGMENT_IDENTIFIER"; - @Override - public String getName() { - return NAME; - } - - @Override - public byte[] getId() { - return NAME.getBytes(StandardCharsets.UTF_8); - } - - @Override - public boolean containsStaticData() { - return false; - } - }; - - private static Map>> asSegmentMap(final Map> initialMap) { - final Map>> segmentMap = new HashMap<>(); - segmentMap.put(SEGMENT_IDENTIFIER, initialMap); - return segmentMap; - } - - /** protected access to the rw lock. */ - protected final ReadWriteLock rwLock; - - public InMemoryKeyValueStorageAdapter() { - super(SEGMENT_IDENTIFIER, new SegmentedInMemoryKeyValueStorage()); - rwLock = ((SegmentedInMemoryKeyValueStorage) storage).rwLock; - } - - public InMemoryKeyValueStorageAdapter(final Map> initialMap) { - super(SEGMENT_IDENTIFIER, new SegmentedInMemoryKeyValueStorage(asSegmentMap(initialMap))); - rwLock = ((SegmentedInMemoryKeyValueStorage) storage).rwLock; - } - - private InMemoryKeyValueStorageAdapter(final SegmentedInMemoryKeyValueStorage storage) { - super(SEGMENT_IDENTIFIER, storage); - rwLock = storage.rwLock; - } -} diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java index 2120ba28574..a770b7ca3cd 100644 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java @@ -17,8 +17,8 @@ import org.hyperledger.besu.plugin.services.exception.StorageException; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; -import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; import org.hyperledger.besu.plugin.services.storage.SnappedKeyValueStorage; @@ -30,7 +30,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Streams; import org.apache.commons.lang3.tuple.Pair; import org.apache.tuweni.bytes.Bytes; @@ -43,14 +42,14 @@ public class LayeredKeyValueStorage extends SegmentedInMemoryKeyValueStorage private static final Logger LOG = LoggerFactory.getLogger(LayeredKeyValueStorage.class); - private final KeyValueStorage parent; + private final SegmentedKeyValueStorage parent; /** * Instantiates a new Layered key value storage. * * @param parent the parent key value storage for this layered storage. */ - public LayeredKeyValueStorage(final KeyValueStorage parent) { + public LayeredKeyValueStorage(final SegmentedKeyValueStorage parent) { this(new ConcurrentHashMap<>(), parent); } @@ -61,7 +60,7 @@ public LayeredKeyValueStorage(final KeyValueStorage parent) { * @param parent the parent key value storage for this layered storage. */ public LayeredKeyValueStorage( - final Map>> map, final KeyValueStorage parent) { + final Map>> map, final SegmentedKeyValueStorage parent) { super(map); this.parent = parent; } @@ -82,7 +81,7 @@ public Optional get(final SegmentIdentifier segmentId, final byte[] key) final Optional foundKey = hashValueStore.computeIfAbsent(segmentId, __ -> new HashMap<>()) .get(wrapKey); if (foundKey == null) { - return parent.get(key); + return parent.get(segmentId, key); } else { return foundKey; } @@ -109,7 +108,7 @@ public Stream> stream(final SegmentIdentifier segmentId) { Pair.of(bytesEntry.getKey().toArrayUnsafe(), bytesEntry.getValue().get())) // since we are layered, concat a parent stream filtered by our map entries: , - parent.stream().filter(e -> !ourLayerState.containsKey(Bytes.of(e.getLeft())))); + parent.stream(segmentId).filter(e -> !ourLayerState.containsKey(Bytes.of(e.getLeft())))); } finally { lock.unlock(); } @@ -136,7 +135,7 @@ public Stream streamKeys(final SegmentIdentifier segmentId) { .map(bytesEntry -> bytesEntry.getKey().toArrayUnsafe()) // since we are layered, concat a parent stream filtered by our map entries: , - parent.streamKeys().filter(e -> !ourLayerState.containsKey(Bytes.of(e)))); + parent.streamKeys(segmentId).filter(e -> !ourLayerState.containsKey(Bytes.of(e)))); } finally { lock.unlock();