Skip to content

Commit

Permalink
fix last few tests, fix defect in SegmentedInMemoryKeyValueStorage wh…
Browse files Browse the repository at this point in the history 
…ere snapshots were not cloning sub-maps

Signed-off-by: garyschulte <garyschulte@gmail.com>
  • Loading branch information
@garyschulte
garyschulte committed Jul 25, 2023
1 parent 8f5f682 commit 8fa3b98
Show file tree
Hide file tree
Showing 3 changed files with 38 additions and 47 deletions.
60 changes: 22 additions & 38 deletions ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction;
import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;

import java.util.Optional;
import java.util.stream.Collectors;
Expand All @@ -59,19 +58,19 @@ public class LogRollingTests {
private BonsaiWorldStateProvider archive;

private InMemoryKeyValueStorageProvider provider;
private InMemoryKeyValueStorage accountStorage;
private InMemoryKeyValueStorage codeStorage;
private InMemoryKeyValueStorage storageStorage;
private InMemoryKeyValueStorage trieBranchStorage;
private InMemoryKeyValueStorage trieLogStorage;
private KeyValueStorage accountStorage;
private KeyValueStorage codeStorage;
private KeyValueStorage storageStorage;
private KeyValueStorage trieBranchStorage;
private KeyValueStorage trieLogStorage;

private InMemoryKeyValueStorageProvider secondProvider;
private BonsaiWorldStateProvider secondArchive;
private InMemoryKeyValueStorage secondAccountStorage;
private InMemoryKeyValueStorage secondCodeStorage;
private InMemoryKeyValueStorage secondStorageStorage;
private InMemoryKeyValueStorage secondTrieBranchStorage;
private InMemoryKeyValueStorage secondTrieLogStorage;
private KeyValueStorage secondAccountStorage;
private KeyValueStorage secondCodeStorage;
private KeyValueStorage secondStorageStorage;
private KeyValueStorage secondTrieBranchStorage;
private KeyValueStorage secondTrieLogStorage;
private final Blockchain blockchain = mock(Blockchain.class);

private static final Address addressOne =
Expand Down Expand Up @@ -131,21 +130,14 @@ public void createStorage() {
new BonsaiWorldStateProvider(
provider, blockchain, cachedMerkleTrieLoader, new NoOpMetricsSystem(), null);
accountStorage =
(InMemoryKeyValueStorage)
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);
codeStorage =
(InMemoryKeyValueStorage)
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE);
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);
codeStorage = provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE);
storageStorage =
(InMemoryKeyValueStorage)
provider.getStorageBySegmentIdentifier(
KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE);
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE);
trieBranchStorage =
(InMemoryKeyValueStorage)
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE);
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE);
trieLogStorage =
(InMemoryKeyValueStorage)
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE);
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE);

secondProvider = new InMemoryKeyValueStorageProvider();
final CachedMerkleTrieLoader secondOptimizedMerkleTrieLoader =
Expand All @@ -158,24 +150,16 @@ public void createStorage() {
new NoOpMetricsSystem(),
null);
secondAccountStorage =
(InMemoryKeyValueStorage)
secondProvider.getStorageBySegmentIdentifier(
KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);
secondProvider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);
secondCodeStorage =
(InMemoryKeyValueStorage)
secondProvider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE);
secondProvider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE);
secondStorageStorage =
(InMemoryKeyValueStorage)
secondProvider.getStorageBySegmentIdentifier(
KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE);
secondProvider.getStorageBySegmentIdentifier(
KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE);
secondTrieBranchStorage =
(InMemoryKeyValueStorage)
secondProvider.getStorageBySegmentIdentifier(
KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE);
secondProvider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE);
secondTrieLogStorage =
(InMemoryKeyValueStorage)
secondProvider.getStorageBySegmentIdentifier(
KeyValueSegmentIdentifier.TRIE_LOG_STORAGE);
secondProvider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE);
}

@Test
Expand Down Expand Up @@ -327,7 +311,7 @@ public void rollBackOnce() {
assertThat(secondWorldState.rootHash()).isEqualByComparingTo(worldState.rootHash());
}

private TrieLogLayer getTrieLogLayer(final InMemoryKeyValueStorage storage, final Bytes key) {
private TrieLogLayer getTrieLogLayer(final KeyValueStorage storage, final Bytes key) {
return storage
.get(key.toArrayUnsafe())
.map(bytes -> TrieLogFactoryImpl.readFrom(new BytesValueRLPInput(Bytes.wrap(bytes), false)))
Expand Down
6 changes: 4 additions & 2 deletions ...reum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPrunerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,8 @@ public void sweepBefore_shouldSweepStateRootFirst() {
stateRoots.forEach(
stateRoot -> {
final InOrder thisRootsOrdering = inOrder(hashValueStore, worldStateStorage);
thisRootsOrdering.verify(hashValueStore).remove(stateRoot);
// disable this validation since the underlying storage no longer directly uses .remove()
// thisRootsOrdering.verify(hashValueStore).remove(stateRoot);
thisRootsOrdering.verify(worldStateStorage).prune(any());
});
}
Expand Down Expand Up @@ -184,7 +185,8 @@ public void sweepBefore_shouldNotRemoveMarkedStateRoots() {
stateRoots.forEach(
stateRoot -> {
final InOrder thisRootsOrdering = inOrder(hashValueStore, worldStateStorage);
thisRootsOrdering.verify(hashValueStore).remove(stateRoot);
// disable this validation since the underlying storage no longer directly uses .remove()
// thisRootsOrdering.verify(hashValueStore).remove(stateRoot);
thisRootsOrdering.verify(worldStateStorage).prune(any());
});

Expand Down
19 changes: 12 additions & 7 deletions ...src/main/java/org/hyperledger/besu/services/kvstore/SegmentedInMemoryKeyValueStorage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ public SegmentedInMemoryKeyValueStorage() {
*
* @param hashValueStore the hash value store
*/
SegmentedInMemoryKeyValueStorage(
protected SegmentedInMemoryKeyValueStorage(
final Map<SegmentIdentifier, Map<Bytes, Optional<byte[]>>> hashValueStore) {
this.hashValueStore = hashValueStore;
}
Expand Down Expand Up @@ -200,7 +200,10 @@ public boolean isClosed() {

@Override
public SegmentedInMemoryKeyValueStorage takeSnapshot() {
return new SegmentedInMemoryKeyValueStorage(new HashMap<>(hashValueStore));
// need to clone the submaps also:
return new SegmentedInMemoryKeyValueStorage(
hashValueStore.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> new HashMap<>(e.getValue()))));
}

@Override
Expand Down Expand Up @@ -247,11 +250,13 @@ public void commit() throws StorageException {

removedKeys.entrySet().stream()
.forEach(
entry ->
hashValueStore
.computeIfAbsent(entry.getKey(), __ -> new HashMap<>())
.keySet()
.removeAll(entry.getValue()));
entry -> {
var keyset =
hashValueStore
.computeIfAbsent(entry.getKey(), __ -> new HashMap<>())
.keySet();
keyset.removeAll(entry.getValue());
});

updatedValues.clear();
removedKeys.clear();
Expand Down

0 comments on commit 8fa3b98

Please sign in to comment.