From 048623a2e3fa09ca6816b2bcdade24d7896bd0ef Mon Sep 17 00:00:00 2001 From: Simon Dudley Date: Mon, 23 Oct 2023 13:29:10 +1000 Subject: [PATCH 1/4] Move TrieLogProvider setup code to AbstractTrieLogManager Signed-off-by: Simon Dudley --- .../cache/CachedWorldStorageManager.java | 80 ------------------- .../trielog/AbstractTrieLogManager.java | 76 +++++++++++++++++- 2 files changed, 74 insertions(+), 82 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java index d0f1be97d89..5a340e7e293 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java @@ -21,16 +21,11 @@ import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateLayerStorage; import org.hyperledger.besu.ethereum.bonsai.trielog.AbstractTrieLogManager; -import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogFactoryImpl; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.BesuContext; -import org.hyperledger.besu.plugin.services.TrieLogService; -import org.hyperledger.besu.plugin.services.trielogs.TrieLog; -import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory; -import org.hyperledger.besu.plugin.services.trielogs.TrieLogProvider; import java.util.ArrayList; import java.util.Comparator; @@ -39,10 +34,7 @@ import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; -import java.util.stream.LongStream; -import java.util.stream.Stream; -import com.google.common.annotations.VisibleForTesting; import org.apache.tuweni.bytes.Bytes32; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -227,76 +219,4 @@ public void onClearTrieLog() { public void onCloseStorage() { this.cachedWorldStatesByHash.clear(); } - - @VisibleForTesting - @Override - protected TrieLogFactory setupTrieLogFactory(final BesuContext pluginContext) { - // if we have a TrieLogService from pluginContext, use it. - var trieLogServicez = - Optional.ofNullable(pluginContext) - .flatMap(context -> context.getService(TrieLogService.class)); - - if (trieLogServicez.isPresent()) { - var trieLogService = trieLogServicez.get(); - // push the TrieLogProvider into the TrieLogService - trieLogService.configureTrieLogProvider(getTrieLogProvider()); - - // configure plugin observers: - trieLogService.getObservers().forEach(trieLogObservers::subscribe); - - // return the TrieLogFactory implementation from the TrieLogService - return trieLogService.getTrieLogFactory(); - } else { - // Otherwise default to TrieLogFactoryImpl - return new TrieLogFactoryImpl(); - } - } - - @VisibleForTesting - TrieLogProvider getTrieLogProvider() { - return new TrieLogProvider() { - @Override - public Optional getTrieLogLayer(final Hash blockHash) { - return CachedWorldStorageManager.this.getTrieLogLayer(blockHash); - } - - @Override - public Optional getTrieLogLayer(final long blockNumber) { - return CachedWorldStorageManager.this - .blockchain - .getBlockHeader(blockNumber) - .map(BlockHeader::getHash) - .flatMap(CachedWorldStorageManager.this::getTrieLogLayer); - } - - @Override - public List getTrieLogsByRange( - final long fromBlockNumber, final long toBlockNumber) { - return rangeAsStream(fromBlockNumber, toBlockNumber) - .map(blockchain::getBlockHeader) - .map( - headerOpt -> - headerOpt.flatMap( - header -> - CachedWorldStorageManager.this - .getTrieLogLayer(header.getBlockHash()) - .map( - layer -> - new TrieLogRangeTuple( - header.getBlockHash(), header.getNumber(), layer)))) - .filter(Optional::isPresent) - .map(Optional::get) - .toList(); - } - - Stream rangeAsStream(final long fromBlockNumber, final long toBlockNumber) { - if (Math.abs(toBlockNumber - fromBlockNumber) > LOG_RANGE_LIMIT) { - throw new IllegalArgumentException("Requested Range too large"); - } - long left = Math.min(fromBlockNumber, toBlockNumber); - long right = Math.max(fromBlockNumber, toBlockNumber); - return LongStream.range(left, right).boxed(); - } - }; - } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/AbstractTrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/AbstractTrieLogManager.java index 90aff386e83..7ab357c745c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/AbstractTrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/AbstractTrieLogManager.java @@ -24,13 +24,18 @@ import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.plugin.BesuContext; +import org.hyperledger.besu.plugin.services.TrieLogService; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent.TrieLogObserver; import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory; +import org.hyperledger.besu.plugin.services.trielogs.TrieLogProvider; import org.hyperledger.besu.util.Subscribers; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.LongStream; +import java.util.stream.Stream; import com.google.common.annotations.VisibleForTesting; import org.apache.tuweni.bytes.Bytes32; @@ -63,8 +68,6 @@ protected AbstractTrieLogManager( this.trieLogFactory = setupTrieLogFactory(pluginContext); } - protected abstract TrieLogFactory setupTrieLogFactory(final BesuContext pluginContext); - @Override public synchronized void saveTrieLog( final BonsaiWorldStateUpdateAccumulator localUpdater, @@ -161,4 +164,73 @@ public synchronized long subscribe(final TrieLogObserver sub) { public synchronized void unsubscribe(final long id) { trieLogObservers.unsubscribe(id); } + + private TrieLogFactory setupTrieLogFactory(final BesuContext pluginContext) { + // if we have a TrieLogService from pluginContext, use it. + var trieLogServicez = + Optional.ofNullable(pluginContext) + .flatMap(context -> context.getService(TrieLogService.class)); + + if (trieLogServicez.isPresent()) { + var trieLogService = trieLogServicez.get(); + // push the TrieLogProvider into the TrieLogService + trieLogService.configureTrieLogProvider(getTrieLogProvider()); + + // configure plugin observers: + trieLogService.getObservers().forEach(trieLogObservers::subscribe); + + // return the TrieLogFactory implementation from the TrieLogService + return trieLogService.getTrieLogFactory(); + } else { + // Otherwise default to TrieLogFactoryImpl + return new TrieLogFactoryImpl(); + } + } + + private TrieLogProvider getTrieLogProvider() { + return new TrieLogProvider() { + @Override + public Optional getTrieLogLayer(final Hash blockHash) { + return AbstractTrieLogManager.this.getTrieLogLayer(blockHash); + } + + @Override + public Optional getTrieLogLayer(final long blockNumber) { + return AbstractTrieLogManager.this + .blockchain + .getBlockHeader(blockNumber) + .map(BlockHeader::getHash) + .flatMap(AbstractTrieLogManager.this::getTrieLogLayer); + } + + @Override + public List getTrieLogsByRange( + final long fromBlockNumber, final long toBlockNumber) { + return rangeAsStream(fromBlockNumber, toBlockNumber) + .map(blockchain::getBlockHeader) + .map( + headerOpt -> + headerOpt.flatMap( + header -> + AbstractTrieLogManager.this + .getTrieLogLayer(header.getBlockHash()) + .map( + layer -> + new TrieLogRangeTuple( + header.getBlockHash(), header.getNumber(), layer)))) + .filter(Optional::isPresent) + .map(Optional::get) + .toList(); + } + + Stream rangeAsStream(final long fromBlockNumber, final long toBlockNumber) { + if (Math.abs(toBlockNumber - fromBlockNumber) > LOG_RANGE_LIMIT) { + throw new IllegalArgumentException("Requested Range too large"); + } + long left = Math.min(fromBlockNumber, toBlockNumber); + long right = Math.max(fromBlockNumber, toBlockNumber); + return LongStream.range(left, right).boxed(); + } + }; + } } From 24bdcf68052d31e8636c86331928378df2c5c3b9 Mon Sep 17 00:00:00 2001 From: Simon Dudley Date: Mon, 23 Oct 2023 16:04:12 +1000 Subject: [PATCH 2/4] Decouple CachedWorldStorageManager from TrieLogManager Separate out the concepts of world state caching from trie log management Make AbstractTrieLogManager a concrete implemenation (to be further renamed/refactored next commit) Signed-off-by: Simon Dudley --- .../bonsai/BonsaiWorldStateProvider.java | 37 +++++++----- .../cache/CachedWorldStorageManager.java | 55 ++++++++--------- .../trielog/AbstractTrieLogManager.java | 30 +--------- .../bonsai/trielog/TrieLogManager.java | 15 ----- .../bonsai/worldview/BonsaiWorldState.java | 12 +++- .../bonsai/BonsaiSnapshotIsolationTests.java | 7 ++- .../bonsai/BonsaiWorldStateArchiveTest.java | 12 +++- .../bonsai/trielog/TrieLogManagerTests.java | 12 +--- .../BonsaiReferenceTestWorldState.java | 60 +++++++++++++------ 9 files changed, 119 insertions(+), 121 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java index 299b826369c..b147ab83745 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java @@ -23,6 +23,7 @@ import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.bonsai.cache.CachedWorldStorageManager; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.trielog.AbstractTrieLogManager; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; @@ -62,6 +63,7 @@ public class BonsaiWorldStateProvider implements WorldStateArchive { private final Blockchain blockchain; + private final CachedWorldStorageManager cachedWorldStorageManager; private final TrieLogManager trieLogManager; private final BonsaiWorldState persistedState; private final BonsaiWorldStateKeyValueStorage worldStateStorage; @@ -91,15 +93,12 @@ public BonsaiWorldStateProvider( final ObservableMetricsSystem metricsSystem, final BesuContext pluginContext) { + this.cachedWorldStorageManager = + new CachedWorldStorageManager(this, worldStateStorage, metricsSystem); // TODO: de-dup constructors this.trieLogManager = - new CachedWorldStorageManager( - this, - blockchain, - worldStateStorage, - metricsSystem, - maxLayersToLoad.orElse(RETAINED_LAYERS), - pluginContext); + new AbstractTrieLogManager( + blockchain, worldStateStorage, maxLayersToLoad.orElse(RETAINED_LAYERS), pluginContext); this.blockchain = blockchain; this.worldStateStorage = worldStateStorage; this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; @@ -108,16 +107,18 @@ public BonsaiWorldStateProvider( .getBlockHeader(persistedState.getWorldStateBlockHash()) .ifPresent( blockHeader -> - this.trieLogManager.addCachedLayer( + this.cachedWorldStorageManager.addCachedLayer( blockHeader, persistedState.getWorldStateRootHash(), persistedState)); } @VisibleForTesting BonsaiWorldStateProvider( + final CachedWorldStorageManager cachedWorldStorageManager, final TrieLogManager trieLogManager, final BonsaiWorldStateKeyValueStorage worldStateStorage, final Blockchain blockchain, final CachedMerkleTrieLoader cachedMerkleTrieLoader) { + this.cachedWorldStorageManager = cachedWorldStorageManager; this.trieLogManager = trieLogManager; this.blockchain = blockchain; this.worldStateStorage = worldStateStorage; @@ -127,13 +128,13 @@ public BonsaiWorldStateProvider( .getBlockHeader(persistedState.getWorldStateBlockHash()) .ifPresent( blockHeader -> - this.trieLogManager.addCachedLayer( + this.cachedWorldStorageManager.addCachedLayer( blockHeader, persistedState.getWorldStateRootHash(), persistedState)); } @Override public Optional get(final Hash rootHash, final Hash blockHash) { - return trieLogManager + return cachedWorldStorageManager .getWorldState(blockHash) .or( () -> { @@ -148,7 +149,7 @@ public Optional get(final Hash rootHash, final Hash blockHash) { @Override public boolean isWorldStateAvailable(final Hash rootHash, final Hash blockHash) { - return trieLogManager.containWorldStateStorage(blockHash) + return cachedWorldStorageManager.containWorldStateStorage(blockHash) || persistedState.blockHash().equals(blockHash) || worldStateStorage.isWorldStateAvailable(rootHash, blockHash); } @@ -167,10 +168,10 @@ public Optional getMutable( trieLogManager.getMaxLayersToLoad()); return Optional.empty(); } - return trieLogManager + return cachedWorldStorageManager .getWorldState(blockHeader.getHash()) - .or(() -> trieLogManager.getNearestWorldState(blockHeader)) - .or(() -> trieLogManager.getHeadWorldState(blockchain::getBlockHeader)) + .or(() -> cachedWorldStorageManager.getNearestWorldState(blockHeader)) + .or(() -> cachedWorldStorageManager.getHeadWorldState(blockchain::getBlockHeader)) .flatMap( bonsaiWorldState -> rollMutableStateToBlockHash(bonsaiWorldState, blockHeader.getHash())) @@ -354,11 +355,15 @@ public TrieLogManager getTrieLogManager() { return trieLogManager; } + public CachedWorldStorageManager getCachedWorldStorageManager() { + return cachedWorldStorageManager; + } + @Override public void resetArchiveStateTo(final BlockHeader blockHeader) { persistedState.resetWorldStateTo(blockHeader); - this.trieLogManager.reset(); - this.trieLogManager.addCachedLayer( + this.cachedWorldStorageManager.reset(); + this.cachedWorldStorageManager.addCachedLayer( blockHeader, persistedState.getWorldStateRootHash(), persistedState); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java index 5a340e7e293..f2434bbf912 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java @@ -20,12 +20,9 @@ import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateLayerStorage; -import org.hyperledger.besu.ethereum.bonsai.trielog.AbstractTrieLogManager; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.metrics.ObservableMetricsSystem; -import org.hyperledger.besu.plugin.BesuContext; import java.util.ArrayList; import java.util.Comparator; @@ -39,44 +36,34 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class CachedWorldStorageManager extends AbstractTrieLogManager - implements BonsaiStorageSubscriber { +public class CachedWorldStorageManager implements BonsaiStorageSubscriber { + public static final long RETAINED_LAYERS = 512; // at least 256 + typical rollbacks private static final Logger LOG = LoggerFactory.getLogger(CachedWorldStorageManager.class); private final BonsaiWorldStateProvider archive; private final ObservableMetricsSystem metricsSystem; - CachedWorldStorageManager( + private final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; + private final Map cachedWorldStatesByHash; + + private CachedWorldStorageManager( final BonsaiWorldStateProvider archive, - final Blockchain blockchain, final BonsaiWorldStateKeyValueStorage worldStateStorage, - final long maxLayersToLoad, final Map cachedWorldStatesByHash, - final BesuContext pluginContext, final ObservableMetricsSystem metricsSystem) { - super(blockchain, worldStateStorage, maxLayersToLoad, cachedWorldStatesByHash, pluginContext); worldStateStorage.subscribe(this); + this.rootWorldStateStorage = worldStateStorage; + this.cachedWorldStatesByHash = cachedWorldStatesByHash; this.archive = archive; this.metricsSystem = metricsSystem; } public CachedWorldStorageManager( final BonsaiWorldStateProvider archive, - final Blockchain blockchain, final BonsaiWorldStateKeyValueStorage worldStateStorage, - final ObservableMetricsSystem metricsSystem, - final long maxLayersToLoad, - final BesuContext pluginContext) { - this( - archive, - blockchain, - worldStateStorage, - maxLayersToLoad, - new ConcurrentHashMap<>(), - pluginContext, - metricsSystem); + final ObservableMetricsSystem metricsSystem) { + this(archive, worldStateStorage, new ConcurrentHashMap<>(), metricsSystem); } - @Override public synchronized void addCachedLayer( final BlockHeader blockHeader, final Hash worldStateRootHash, @@ -124,7 +111,20 @@ public synchronized void addCachedLayer( scrubCachedLayers(blockHeader.getNumber()); } - @Override + private synchronized void scrubCachedLayers(final long newMaxHeight) { + if (cachedWorldStatesByHash.size() > RETAINED_LAYERS) { + final long waterline = newMaxHeight - RETAINED_LAYERS; + cachedWorldStatesByHash.values().stream() + .filter(layer -> layer.getBlockNumber() < waterline) + .toList() + .forEach( + layer -> { + cachedWorldStatesByHash.remove(layer.getBlockHash()); + layer.close(); + }); + } + } + public Optional getWorldState(final Hash blockHash) { if (cachedWorldStatesByHash.containsKey(blockHash)) { // return a new worldstate using worldstate storage and an isolated copy of the updater @@ -142,7 +142,6 @@ public Optional getWorldState(final Hash blockHash) { return Optional.empty(); } - @Override public Optional getNearestWorldState(final BlockHeader blockHeader) { LOG.atDebug() .setMessage("getting nearest worldstate for {}") @@ -175,7 +174,6 @@ public Optional getNearestWorldState(final BlockHeader blockHe archive, new BonsaiWorldStateLayerStorage(storage))); } - @Override public Optional getHeadWorldState( final Function> hashBlockHeaderFunction) { @@ -195,7 +193,10 @@ public Optional getHeadWorldState( }); } - @Override + public boolean containWorldStateStorage(final Hash blockHash) { + return cachedWorldStatesByHash.containsKey(blockHash); + } + public void reset() { this.cachedWorldStatesByHash.clear(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/AbstractTrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/AbstractTrieLogManager.java index 7ab357c745c..14afc2ef3b5 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/AbstractTrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/AbstractTrieLogManager.java @@ -16,7 +16,6 @@ package org.hyperledger.besu.ethereum.bonsai.trielog; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.cache.CachedBonsaiWorldView; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiUpdater; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; @@ -32,38 +31,32 @@ import org.hyperledger.besu.util.Subscribers; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.stream.LongStream; import java.util.stream.Stream; import com.google.common.annotations.VisibleForTesting; -import org.apache.tuweni.bytes.Bytes32; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class AbstractTrieLogManager implements TrieLogManager { +public class AbstractTrieLogManager implements TrieLogManager { private static final Logger LOG = LoggerFactory.getLogger(AbstractTrieLogManager.class); - public static final long RETAINED_LAYERS = 512; // at least 256 + typical rollbacks public static final long LOG_RANGE_LIMIT = 1000; // restrict trielog range queries to 1k logs protected final Blockchain blockchain; protected final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; - protected final Map cachedWorldStatesByHash; protected final long maxLayersToLoad; protected final Subscribers trieLogObservers = Subscribers.create(); protected final TrieLogFactory trieLogFactory; - protected AbstractTrieLogManager( + public AbstractTrieLogManager( final Blockchain blockchain, final BonsaiWorldStateKeyValueStorage worldStateStorage, final long maxLayersToLoad, - final Map cachedWorldStatesByHash, final BesuContext pluginContext) { this.blockchain = blockchain; this.rootWorldStateStorage = worldStateStorage; - this.cachedWorldStatesByHash = cachedWorldStatesByHash; this.maxLayersToLoad = maxLayersToLoad; this.trieLogFactory = setupTrieLogFactory(pluginContext); } @@ -110,20 +103,6 @@ TrieLog prepareTrieLog( return trieLog; } - public synchronized void scrubCachedLayers(final long newMaxHeight) { - if (cachedWorldStatesByHash.size() > RETAINED_LAYERS) { - final long waterline = newMaxHeight - RETAINED_LAYERS; - cachedWorldStatesByHash.values().stream() - .filter(layer -> layer.getBlockNumber() < waterline) - .toList() - .forEach( - layer -> { - cachedWorldStatesByHash.remove(layer.getBlockHash()); - layer.close(); - }); - } - } - private void persistTrieLog( final BlockHeader blockHeader, final Hash worldStateRootHash, @@ -140,11 +119,6 @@ private void persistTrieLog( .put(blockHeader.getHash().toArrayUnsafe(), trieLogFactory.serialize(trieLog)); } - @Override - public boolean containWorldStateStorage(final Hash blockHash) { - return cachedWorldStatesByHash.containsKey(blockHash); - } - @Override public long getMaxLayersToLoad() { return maxLayersToLoad; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java index 5da83180a1c..4a7b1d8c55d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java @@ -23,7 +23,6 @@ import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent; import java.util.Optional; -import java.util.function.Function; public interface TrieLogManager { @@ -33,22 +32,8 @@ void saveTrieLog( final BlockHeader forBlockHeader, final BonsaiWorldState forWorldState); - void addCachedLayer( - BlockHeader blockHeader, Hash worldStateRootHash, BonsaiWorldState forWorldState); - - boolean containWorldStateStorage(final Hash blockHash); - - Optional getWorldState(final Hash blockHash); - - Optional getNearestWorldState(final BlockHeader blockHeader); - - Optional getHeadWorldState( - final Function> hashBlockHeaderFunction); - long getMaxLayersToLoad(); - void reset(); - Optional getTrieLogLayer(final Hash blockHash); long subscribe(final TrieLogEvent.TrieLogObserver sub); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java index ed81c90f439..d4c4c223a24 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java @@ -28,6 +28,7 @@ import org.hyperledger.besu.ethereum.bonsai.BonsaiValue; import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedWorldStorageManager; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; @@ -69,6 +70,7 @@ public class BonsaiWorldState protected BonsaiWorldStateKeyValueStorage worldStateStorage; protected final CachedMerkleTrieLoader cachedMerkleTrieLoader; + protected final CachedWorldStorageManager cachedWorldStorageManager; protected final TrieLogManager trieLogManager; private BonsaiWorldStateUpdateAccumulator accumulator; @@ -79,12 +81,17 @@ public class BonsaiWorldState public BonsaiWorldState( final BonsaiWorldStateProvider archive, final BonsaiWorldStateKeyValueStorage worldStateStorage) { - this(worldStateStorage, archive.getCachedMerkleTrieLoader(), archive.getTrieLogManager()); + this( + worldStateStorage, + archive.getCachedMerkleTrieLoader(), + archive.getCachedWorldStorageManager(), + archive.getTrieLogManager()); } protected BonsaiWorldState( final BonsaiWorldStateKeyValueStorage worldStateStorage, final CachedMerkleTrieLoader cachedMerkleTrieLoader, + final CachedWorldStorageManager cachedWorldStorageManager, final TrieLogManager trieLogManager) { this.worldStateStorage = worldStateStorage; this.worldStateRootHash = @@ -101,6 +108,7 @@ protected BonsaiWorldState( (addr, value) -> cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value)); this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; + this.cachedWorldStorageManager = cachedWorldStorageManager; this.trieLogManager = trieLogManager; } @@ -396,7 +404,7 @@ public void persist(final BlockHeader blockHeader) { trieLogManager.saveTrieLog(localCopy, newWorldStateRootHash, blockHeader, this); // not save a frozen state in the cache if (!isFrozen) { - trieLogManager.addCachedLayer(blockHeader, newWorldStateRootHash, this); + cachedWorldStorageManager.addCachedLayer(blockHeader, newWorldStateRootHash, this); } }; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotIsolationTests.java index ed4b4549674..03407d2868f 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotIsolationTests.java @@ -61,8 +61,11 @@ public void testIsolatedFromHead_behindHead() { assertThat(res.isSuccessful()).isTrue(); assertThat(res2.isSuccessful()).isTrue(); - assertThat(archive.getTrieLogManager().containWorldStateStorage(firstBlock.getHash())).isTrue(); - assertThat(archive.getTrieLogManager().containWorldStateStorage(secondBlock.getHash())) + assertThat( + archive.getCachedWorldStorageManager().containWorldStateStorage(firstBlock.getHash())) + .isTrue(); + assertThat( + archive.getCachedWorldStorageManager().containWorldStateStorage(secondBlock.getHash())) .isTrue(); assertThat(archive.getMutable().get(testAddress)).isNotNull(); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java index b492b67ffdb..0721522ffb1 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java @@ -32,6 +32,7 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedWorldStorageManager; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogFactoryImpl; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogLayer; @@ -74,6 +75,7 @@ public class BonsaiWorldStateArchiveTest { @Mock SegmentedKeyValueStorageTransaction segmentedKeyValueStorageTransaction; BonsaiWorldStateProvider bonsaiWorldStateArchive; + @Mock CachedWorldStorageManager cachedWorldStorageManager; @Mock TrieLogManager trieLogManager; @BeforeEach @@ -100,6 +102,7 @@ public void testGetMutableReturnPersistedStateWhenNeeded() { .thenReturn(Optional.of(chainHead.getHash().toArrayUnsafe())); bonsaiWorldStateArchive = new BonsaiWorldStateProvider( + cachedWorldStorageManager, trieLogManager, new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()), blockchain, @@ -123,7 +126,7 @@ public void testGetMutableReturnEmptyWhenLoadMoreThanLimitLayersBack() { final BlockHeader chainHead = blockBuilder.number(512).buildHeader(); when(blockchain.getChainHeadHeader()).thenReturn(chainHead); assertThat(bonsaiWorldStateArchive.getMutable(blockHeader, false)).isEmpty(); - verify(trieLogManager, Mockito.never()).getWorldState(any(Hash.class)); + verify(cachedWorldStorageManager, Mockito.never()).getWorldState(any(Hash.class)); } @Test @@ -131,6 +134,7 @@ public void testGetMutableWhenLoadLessThanLimitLayersBack() { bonsaiWorldStateArchive = new BonsaiWorldStateProvider( + cachedWorldStorageManager, trieLogManager, new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()), blockchain, @@ -142,7 +146,7 @@ public void testGetMutableWhenLoadLessThanLimitLayersBack() { when(mockWorldState.freeze()).thenReturn(mockWorldState); when(trieLogManager.getMaxLayersToLoad()).thenReturn(Long.valueOf(512)); - when(trieLogManager.getWorldState(blockHeader.getHash())) + when(cachedWorldStorageManager.getWorldState(blockHeader.getHash())) .thenReturn(Optional.of(mockWorldState)); when(blockchain.getChainHeadHeader()).thenReturn(chainHead); assertThat(bonsaiWorldStateArchive.getMutable(blockHeader, false)) @@ -162,6 +166,7 @@ public void testGetMutableWithStorageInconsistencyRollbackTheState() { bonsaiWorldStateArchive = spy( new BonsaiWorldStateProvider( + cachedWorldStorageManager, trieLogManager, worldStateStorage, blockchain, @@ -186,6 +191,7 @@ public void testGetMutableWithStorageConsistencyNotRollbackTheState() { bonsaiWorldStateArchive = spy( new BonsaiWorldStateProvider( + cachedWorldStorageManager, trieLogManager, worldStateStorage, blockchain, @@ -222,6 +228,7 @@ public void testGetMutableWithStorageConsistencyToRollbackAndRollForwardTheState bonsaiWorldStateArchive = spy( new BonsaiWorldStateProvider( + cachedWorldStorageManager, trieLogManager, worldStateStorage, blockchain, @@ -261,6 +268,7 @@ public void testGetMutableWithRollbackNotOverrideTrieLogLayer() { bonsaiWorldStateArchive = spy( new BonsaiWorldStateProvider( + cachedWorldStorageManager, trieLogManager, new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()), blockchain, diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java index 35ce1b77c06..d5a1440554b 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java @@ -19,15 +19,12 @@ import static org.mockito.Mockito.spy; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.bonsai.cache.CachedWorldStorageManager; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; -import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import java.util.concurrent.atomic.AtomicBoolean; @@ -48,7 +45,6 @@ public class TrieLogManagerTests { @Mock BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage; @Mock BonsaiWorldState worldState; - @Mock BonsaiWorldStateProvider archive; @Mock Blockchain blockchain; BonsaiWorldStateUpdateAccumulator bonsaiUpdater = spy(new BonsaiWorldStateUpdateAccumulator(worldState, (__, ___) -> {}, (__, ___) -> {})); @@ -58,13 +54,7 @@ public class TrieLogManagerTests { @BeforeEach public void setup() { trieLogManager = - new CachedWorldStorageManager( - archive, - blockchain, - bonsaiWorldStateKeyValueStorage, - new NoOpMetricsSystem(), - 512, - null); + new AbstractTrieLogManager(blockchain, bonsaiWorldStateKeyValueStorage, 512, null); } @Test diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java index ff8a33814cc..972a2d307c5 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -17,6 +17,7 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedWorldStorageManager; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiPreImageProxy; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogAddedEvent; @@ -54,9 +55,10 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState protected BonsaiReferenceTestWorldState( final BonsaiReferenceTestWorldStateStorage worldStateStorage, final CachedMerkleTrieLoader cachedMerkleTrieLoader, + final CachedWorldStorageManager cachedWorldStorageManager, final TrieLogManager trieLogManager, final BonsaiPreImageProxy preImageProxy) { - super(worldStateStorage, cachedMerkleTrieLoader, trieLogManager); + super(worldStateStorage, cachedMerkleTrieLoader, cachedWorldStorageManager, trieLogManager); this.refTestStorage = worldStateStorage; this.preImageProxy = preImageProxy; setAccumulator( @@ -74,7 +76,11 @@ protected BonsaiReferenceTestWorldState( public ReferenceTestWorldState copy() { var layerCopy = new BonsaiReferenceTestWorldStateStorage(worldStateStorage, preImageProxy); return new BonsaiReferenceTestWorldState( - layerCopy, cachedMerkleTrieLoader, trieLogManager, preImageProxy); + layerCopy, + cachedMerkleTrieLoader, + cachedWorldStorageManager, + trieLogManager, + preImageProxy); } /** @@ -109,9 +115,16 @@ public static BonsaiReferenceTestWorldState create( new InMemoryKeyValueStorageProvider(), metricsSystem), preImageProxy); + final NoOpCachedWorldStorageManager noOpCachedWorldStorageManager = + new NoOpCachedWorldStorageManager(); + final BonsaiReferenceTestWorldState worldState = new BonsaiReferenceTestWorldState( - worldStateStorage, cachedMerkleTrieLoader, trieLogManager, preImageProxy); + worldStateStorage, + cachedMerkleTrieLoader, + noOpCachedWorldStorageManager, + trieLogManager, + preImageProxy); final WorldUpdater updater = worldState.updater(); for (final Map.Entry entry : accounts.entrySet()) { @@ -131,19 +144,14 @@ public void disableRootHashVerification() { disableRootHashVerification = true; } - static class NoOpTrieLogManager implements TrieLogManager { - private final Subscribers trieLogObservers = Subscribers.create(); - private final TrieLogFactory trieLogFactory = new TrieLogFactoryImpl(); + static class NoOpCachedWorldStorageManager extends CachedWorldStorageManager { - @Override - public void saveTrieLog( - final BonsaiWorldStateUpdateAccumulator localUpdater, - final Hash forWorldStateRootHash, - final BlockHeader forBlockHeader, - final BonsaiWorldState forWorldState) { - // notify trie log added observers, synchronously - TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); - trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); + public NoOpCachedWorldStorageManager() { + super( + null, + new BonsaiWorldStateKeyValueStorage( + new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem()), + new NoOpMetricsSystem()); } @Override @@ -174,12 +182,28 @@ public Optional getHeadWorldState( } @Override - public long getMaxLayersToLoad() { - return 0; + public void reset() {} + } + + static class NoOpTrieLogManager implements TrieLogManager { + private final Subscribers trieLogObservers = Subscribers.create(); + private final TrieLogFactory trieLogFactory = new TrieLogFactoryImpl(); + + @Override + public void saveTrieLog( + final BonsaiWorldStateUpdateAccumulator localUpdater, + final Hash forWorldStateRootHash, + final BlockHeader forBlockHeader, + final BonsaiWorldState forWorldState) { + // notify trie log added observers, synchronously + TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); + trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); } @Override - public void reset() {} + public long getMaxLayersToLoad() { + return 0; + } @Override public Optional getTrieLogLayer(final Hash blockHash) { From 7a6552c3bbae716e199299f4cb8e14fe4c4204e4 Mon Sep 17 00:00:00 2001 From: Simon Dudley Date: Mon, 23 Oct 2023 16:17:45 +1000 Subject: [PATCH 3/4] Rename AbstractTrieLogManager to DefaultTrieLogManager Signed-off-by: Simon Dudley --- .../ethereum/bonsai/BonsaiWorldStateProvider.java | 4 ++-- ...eLogManager.java => DefaultTrieLogManager.java} | 14 +++++++------- .../bonsai/trielog/TrieLogManagerTests.java | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/{AbstractTrieLogManager.java => DefaultTrieLogManager.java} (94%) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java index b147ab83745..4a35e65e0da 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java @@ -23,7 +23,7 @@ import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.bonsai.cache.CachedWorldStorageManager; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.bonsai.trielog.AbstractTrieLogManager; +import org.hyperledger.besu.ethereum.bonsai.trielog.DefaultTrieLogManager; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; @@ -97,7 +97,7 @@ public BonsaiWorldStateProvider( new CachedWorldStorageManager(this, worldStateStorage, metricsSystem); // TODO: de-dup constructors this.trieLogManager = - new AbstractTrieLogManager( + new DefaultTrieLogManager( blockchain, worldStateStorage, maxLayersToLoad.orElse(RETAINED_LAYERS), pluginContext); this.blockchain = blockchain; this.worldStateStorage = worldStateStorage; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/AbstractTrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/DefaultTrieLogManager.java similarity index 94% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/AbstractTrieLogManager.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/DefaultTrieLogManager.java index 14afc2ef3b5..8b945dac163 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/AbstractTrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/DefaultTrieLogManager.java @@ -39,8 +39,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class AbstractTrieLogManager implements TrieLogManager { - private static final Logger LOG = LoggerFactory.getLogger(AbstractTrieLogManager.class); +public class DefaultTrieLogManager implements TrieLogManager { + private static final Logger LOG = LoggerFactory.getLogger(DefaultTrieLogManager.class); public static final long LOG_RANGE_LIMIT = 1000; // restrict trielog range queries to 1k logs protected final Blockchain blockchain; protected final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; @@ -50,7 +50,7 @@ public class AbstractTrieLogManager implements TrieLogManager { protected final TrieLogFactory trieLogFactory; - public AbstractTrieLogManager( + public DefaultTrieLogManager( final Blockchain blockchain, final BonsaiWorldStateKeyValueStorage worldStateStorage, final long maxLayersToLoad, @@ -165,16 +165,16 @@ private TrieLogProvider getTrieLogProvider() { return new TrieLogProvider() { @Override public Optional getTrieLogLayer(final Hash blockHash) { - return AbstractTrieLogManager.this.getTrieLogLayer(blockHash); + return DefaultTrieLogManager.this.getTrieLogLayer(blockHash); } @Override public Optional getTrieLogLayer(final long blockNumber) { - return AbstractTrieLogManager.this + return DefaultTrieLogManager.this .blockchain .getBlockHeader(blockNumber) .map(BlockHeader::getHash) - .flatMap(AbstractTrieLogManager.this::getTrieLogLayer); + .flatMap(DefaultTrieLogManager.this::getTrieLogLayer); } @Override @@ -186,7 +186,7 @@ public List getTrieLogsByRange( headerOpt -> headerOpt.flatMap( header -> - AbstractTrieLogManager.this + DefaultTrieLogManager.this .getTrieLogLayer(header.getBlockHash()) .map( layer -> diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java index d5a1440554b..a4b5012a80d 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java @@ -54,7 +54,7 @@ public class TrieLogManagerTests { @BeforeEach public void setup() { trieLogManager = - new AbstractTrieLogManager(blockchain, bonsaiWorldStateKeyValueStorage, 512, null); + new DefaultTrieLogManager(blockchain, bonsaiWorldStateKeyValueStorage, 512, null); } @Test From 0bb66f1d21f6ec9c337f1f32bbb5ad7cb619b50f Mon Sep 17 00:00:00 2001 From: Simon Dudley Date: Mon, 23 Oct 2023 17:16:07 +1000 Subject: [PATCH 4/4] Make TrieLogManager the default implementation and remove the interface Signed-off-by: Simon Dudley --- .../bonsai/BonsaiWorldStateProvider.java | 3 +- .../bonsai/trielog/DefaultTrieLogManager.java | 210 ------------------ .../bonsai/trielog/TrieLogManager.java | 175 ++++++++++++++- .../bonsai/trielog/TrieLogManagerTests.java | 3 +- .../BonsaiReferenceTestWorldState.java | 14 +- 5 files changed, 177 insertions(+), 228 deletions(-) delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/DefaultTrieLogManager.java diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java index 4a35e65e0da..bcb7f9f65fe 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java @@ -23,7 +23,6 @@ import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.bonsai.cache.CachedWorldStorageManager; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.bonsai.trielog.DefaultTrieLogManager; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; @@ -97,7 +96,7 @@ public BonsaiWorldStateProvider( new CachedWorldStorageManager(this, worldStateStorage, metricsSystem); // TODO: de-dup constructors this.trieLogManager = - new DefaultTrieLogManager( + new TrieLogManager( blockchain, worldStateStorage, maxLayersToLoad.orElse(RETAINED_LAYERS), pluginContext); this.blockchain = blockchain; this.worldStateStorage = worldStateStorage; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/DefaultTrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/DefaultTrieLogManager.java deleted file mode 100644 index 8b945dac163..00000000000 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/DefaultTrieLogManager.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Copyright Hyperledger Besu Contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ -package org.hyperledger.besu.ethereum.bonsai.trielog; - -import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiUpdater; -import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; -import org.hyperledger.besu.ethereum.chain.Blockchain; -import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.plugin.BesuContext; -import org.hyperledger.besu.plugin.services.TrieLogService; -import org.hyperledger.besu.plugin.services.trielogs.TrieLog; -import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent.TrieLogObserver; -import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory; -import org.hyperledger.besu.plugin.services.trielogs.TrieLogProvider; -import org.hyperledger.besu.util.Subscribers; - -import java.util.List; -import java.util.Optional; -import java.util.stream.LongStream; -import java.util.stream.Stream; - -import com.google.common.annotations.VisibleForTesting; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DefaultTrieLogManager implements TrieLogManager { - private static final Logger LOG = LoggerFactory.getLogger(DefaultTrieLogManager.class); - public static final long LOG_RANGE_LIMIT = 1000; // restrict trielog range queries to 1k logs - protected final Blockchain blockchain; - protected final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; - - protected final long maxLayersToLoad; - protected final Subscribers trieLogObservers = Subscribers.create(); - - protected final TrieLogFactory trieLogFactory; - - public DefaultTrieLogManager( - final Blockchain blockchain, - final BonsaiWorldStateKeyValueStorage worldStateStorage, - final long maxLayersToLoad, - final BesuContext pluginContext) { - this.blockchain = blockchain; - this.rootWorldStateStorage = worldStateStorage; - this.maxLayersToLoad = maxLayersToLoad; - this.trieLogFactory = setupTrieLogFactory(pluginContext); - } - - @Override - public synchronized void saveTrieLog( - final BonsaiWorldStateUpdateAccumulator localUpdater, - final Hash forWorldStateRootHash, - final BlockHeader forBlockHeader, - final BonsaiWorldState forWorldState) { - // do not overwrite a trielog layer that already exists in the database. - // if it's only in memory we need to save it - // for example, in case of reorg we don't replace a trielog layer - if (rootWorldStateStorage.getTrieLog(forBlockHeader.getHash()).isEmpty()) { - final BonsaiUpdater stateUpdater = forWorldState.getWorldStateStorage().updater(); - boolean success = false; - try { - final TrieLog trieLog = prepareTrieLog(forBlockHeader, localUpdater); - persistTrieLog(forBlockHeader, forWorldStateRootHash, trieLog, stateUpdater); - - // notify trie log added observers, synchronously - trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); - - success = true; - } finally { - if (success) { - stateUpdater.commit(); - } else { - stateUpdater.rollback(); - } - } - } - } - - @VisibleForTesting - TrieLog prepareTrieLog( - final BlockHeader blockHeader, final BonsaiWorldStateUpdateAccumulator localUpdater) { - LOG.atDebug() - .setMessage("Adding layered world state for {}") - .addArgument(blockHeader::toLogString) - .log(); - final TrieLog trieLog = trieLogFactory.create(localUpdater, blockHeader); - trieLog.freeze(); - return trieLog; - } - - private void persistTrieLog( - final BlockHeader blockHeader, - final Hash worldStateRootHash, - final TrieLog trieLog, - final BonsaiUpdater stateUpdater) { - LOG.atDebug() - .setMessage("Persisting trie log for block hash {} and world state root {}") - .addArgument(blockHeader::toLogString) - .addArgument(worldStateRootHash::toHexString) - .log(); - - stateUpdater - .getTrieLogStorageTransaction() - .put(blockHeader.getHash().toArrayUnsafe(), trieLogFactory.serialize(trieLog)); - } - - @Override - public long getMaxLayersToLoad() { - return maxLayersToLoad; - } - - @Override - public Optional getTrieLogLayer(final Hash blockHash) { - return rootWorldStateStorage.getTrieLog(blockHash).map(trieLogFactory::deserialize); - } - - @Override - public synchronized long subscribe(final TrieLogObserver sub) { - return trieLogObservers.subscribe(sub); - } - - @Override - public synchronized void unsubscribe(final long id) { - trieLogObservers.unsubscribe(id); - } - - private TrieLogFactory setupTrieLogFactory(final BesuContext pluginContext) { - // if we have a TrieLogService from pluginContext, use it. - var trieLogServicez = - Optional.ofNullable(pluginContext) - .flatMap(context -> context.getService(TrieLogService.class)); - - if (trieLogServicez.isPresent()) { - var trieLogService = trieLogServicez.get(); - // push the TrieLogProvider into the TrieLogService - trieLogService.configureTrieLogProvider(getTrieLogProvider()); - - // configure plugin observers: - trieLogService.getObservers().forEach(trieLogObservers::subscribe); - - // return the TrieLogFactory implementation from the TrieLogService - return trieLogService.getTrieLogFactory(); - } else { - // Otherwise default to TrieLogFactoryImpl - return new TrieLogFactoryImpl(); - } - } - - private TrieLogProvider getTrieLogProvider() { - return new TrieLogProvider() { - @Override - public Optional getTrieLogLayer(final Hash blockHash) { - return DefaultTrieLogManager.this.getTrieLogLayer(blockHash); - } - - @Override - public Optional getTrieLogLayer(final long blockNumber) { - return DefaultTrieLogManager.this - .blockchain - .getBlockHeader(blockNumber) - .map(BlockHeader::getHash) - .flatMap(DefaultTrieLogManager.this::getTrieLogLayer); - } - - @Override - public List getTrieLogsByRange( - final long fromBlockNumber, final long toBlockNumber) { - return rangeAsStream(fromBlockNumber, toBlockNumber) - .map(blockchain::getBlockHeader) - .map( - headerOpt -> - headerOpt.flatMap( - header -> - DefaultTrieLogManager.this - .getTrieLogLayer(header.getBlockHash()) - .map( - layer -> - new TrieLogRangeTuple( - header.getBlockHash(), header.getNumber(), layer)))) - .filter(Optional::isPresent) - .map(Optional::get) - .toList(); - } - - Stream rangeAsStream(final long fromBlockNumber, final long toBlockNumber) { - if (Math.abs(toBlockNumber - fromBlockNumber) > LOG_RANGE_LIMIT) { - throw new IllegalArgumentException("Requested Range too large"); - } - long left = Math.min(fromBlockNumber, toBlockNumber); - long right = Math.max(fromBlockNumber, toBlockNumber); - return LongStream.range(left, right).boxed(); - } - }; - } -} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java index 4a7b1d8c55d..cabd5a2d300 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java @@ -16,27 +16,188 @@ package org.hyperledger.besu.ethereum.bonsai.trielog; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.plugin.BesuContext; +import org.hyperledger.besu.plugin.services.TrieLogService; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent; +import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory; +import org.hyperledger.besu.plugin.services.trielogs.TrieLogProvider; +import org.hyperledger.besu.util.Subscribers; +import java.util.List; import java.util.Optional; +import java.util.stream.LongStream; +import java.util.stream.Stream; -public interface TrieLogManager { +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; - void saveTrieLog( +public class TrieLogManager { + private static final Logger LOG = LoggerFactory.getLogger(TrieLogManager.class); + public static final long LOG_RANGE_LIMIT = 1000; // restrict trielog range queries to 1k logs + protected final Blockchain blockchain; + protected final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; + + protected final long maxLayersToLoad; + protected final Subscribers trieLogObservers = Subscribers.create(); + + protected final TrieLogFactory trieLogFactory; + + public TrieLogManager( + final Blockchain blockchain, + final BonsaiWorldStateKeyValueStorage worldStateStorage, + final long maxLayersToLoad, + final BesuContext pluginContext) { + this.blockchain = blockchain; + this.rootWorldStateStorage = worldStateStorage; + this.maxLayersToLoad = maxLayersToLoad; + this.trieLogFactory = setupTrieLogFactory(pluginContext); + } + + public synchronized void saveTrieLog( final BonsaiWorldStateUpdateAccumulator localUpdater, final Hash forWorldStateRootHash, final BlockHeader forBlockHeader, - final BonsaiWorldState forWorldState); + final BonsaiWorldState forWorldState) { + // do not overwrite a trielog layer that already exists in the database. + // if it's only in memory we need to save it + // for example, in case of reorg we don't replace a trielog layer + if (rootWorldStateStorage.getTrieLog(forBlockHeader.getHash()).isEmpty()) { + final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater = + forWorldState.getWorldStateStorage().updater(); + boolean success = false; + try { + final TrieLog trieLog = prepareTrieLog(forBlockHeader, localUpdater); + persistTrieLog(forBlockHeader, forWorldStateRootHash, trieLog, stateUpdater); + + // notify trie log added observers, synchronously + trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); + + success = true; + } finally { + if (success) { + stateUpdater.commit(); + } else { + stateUpdater.rollback(); + } + } + } + } + + private TrieLog prepareTrieLog( + final BlockHeader blockHeader, final BonsaiWorldStateUpdateAccumulator localUpdater) { + LOG.atDebug() + .setMessage("Adding layered world state for {}") + .addArgument(blockHeader::toLogString) + .log(); + final TrieLog trieLog = trieLogFactory.create(localUpdater, blockHeader); + trieLog.freeze(); + return trieLog; + } + + private void persistTrieLog( + final BlockHeader blockHeader, + final Hash worldStateRootHash, + final TrieLog trieLog, + final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater) { + LOG.atDebug() + .setMessage("Persisting trie log for block hash {} and world state root {}") + .addArgument(blockHeader::toLogString) + .addArgument(worldStateRootHash::toHexString) + .log(); + + stateUpdater + .getTrieLogStorageTransaction() + .put(blockHeader.getHash().toArrayUnsafe(), trieLogFactory.serialize(trieLog)); + } + + public long getMaxLayersToLoad() { + return maxLayersToLoad; + } + + public Optional getTrieLogLayer(final Hash blockHash) { + return rootWorldStateStorage.getTrieLog(blockHash).map(trieLogFactory::deserialize); + } + + public synchronized long subscribe(final TrieLogEvent.TrieLogObserver sub) { + return trieLogObservers.subscribe(sub); + } + + public synchronized void unsubscribe(final long id) { + trieLogObservers.unsubscribe(id); + } + + private TrieLogFactory setupTrieLogFactory(final BesuContext pluginContext) { + // if we have a TrieLogService from pluginContext, use it. + var trieLogServicez = + Optional.ofNullable(pluginContext) + .flatMap(context -> context.getService(TrieLogService.class)); + + if (trieLogServicez.isPresent()) { + var trieLogService = trieLogServicez.get(); + // push the TrieLogProvider into the TrieLogService + trieLogService.configureTrieLogProvider(getTrieLogProvider()); + + // configure plugin observers: + trieLogService.getObservers().forEach(trieLogObservers::subscribe); + + // return the TrieLogFactory implementation from the TrieLogService + return trieLogService.getTrieLogFactory(); + } else { + // Otherwise default to TrieLogFactoryImpl + return new TrieLogFactoryImpl(); + } + } - long getMaxLayersToLoad(); + private TrieLogProvider getTrieLogProvider() { + return new TrieLogProvider() { + @Override + public Optional getTrieLogLayer(final Hash blockHash) { + return TrieLogManager.this.getTrieLogLayer(blockHash); + } - Optional getTrieLogLayer(final Hash blockHash); + @Override + public Optional getTrieLogLayer(final long blockNumber) { + return TrieLogManager.this + .blockchain + .getBlockHeader(blockNumber) + .map(BlockHeader::getHash) + .flatMap(TrieLogManager.this::getTrieLogLayer); + } - long subscribe(final TrieLogEvent.TrieLogObserver sub); + @Override + public List getTrieLogsByRange( + final long fromBlockNumber, final long toBlockNumber) { + return rangeAsStream(fromBlockNumber, toBlockNumber) + .map(blockchain::getBlockHeader) + .map( + headerOpt -> + headerOpt.flatMap( + header -> + TrieLogManager.this + .getTrieLogLayer(header.getBlockHash()) + .map( + layer -> + new TrieLogRangeTuple( + header.getBlockHash(), header.getNumber(), layer)))) + .filter(Optional::isPresent) + .map(Optional::get) + .toList(); + } - void unsubscribe(final long id); + Stream rangeAsStream(final long fromBlockNumber, final long toBlockNumber) { + if (Math.abs(toBlockNumber - fromBlockNumber) > LOG_RANGE_LIMIT) { + throw new IllegalArgumentException("Requested Range too large"); + } + long left = Math.min(fromBlockNumber, toBlockNumber); + long right = Math.max(fromBlockNumber, toBlockNumber); + return LongStream.range(left, right).boxed(); + } + }; + } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java index a4b5012a80d..0c911d33556 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java @@ -53,8 +53,7 @@ public class TrieLogManagerTests { @BeforeEach public void setup() { - trieLogManager = - new DefaultTrieLogManager(blockchain, bonsaiWorldStateKeyValueStorage, 512, null); + trieLogManager = new TrieLogManager(blockchain, bonsaiWorldStateKeyValueStorage, 512, null); } @Test diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java index 972a2d307c5..9fbc60443ef 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -21,7 +21,6 @@ import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiPreImageProxy; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogAddedEvent; -import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogFactoryImpl; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; @@ -32,8 +31,6 @@ import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent; -import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory; -import org.hyperledger.besu.util.Subscribers; import java.util.Map; import java.util.Optional; @@ -185,10 +182,13 @@ public Optional getHeadWorldState( public void reset() {} } - static class NoOpTrieLogManager implements TrieLogManager { - private final Subscribers trieLogObservers = Subscribers.create(); - private final TrieLogFactory trieLogFactory = new TrieLogFactoryImpl(); + static class NoOpTrieLogManager extends TrieLogManager { + public NoOpTrieLogManager() { + super(null, null, 0, null); + } + + @SuppressWarnings("UnsynchronizedOverridesSynchronized") @Override public void saveTrieLog( final BonsaiWorldStateUpdateAccumulator localUpdater, @@ -206,7 +206,7 @@ public long getMaxLayersToLoad() { } @Override - public Optional getTrieLogLayer(final Hash blockHash) { + public Optional getTrieLogLayer(final Hash blockHash) { return Optional.empty(); }