From 62759a2c28e967980356832ea7303e4844d658e7 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Fri, 7 Jul 2023 13:16:36 -0700 Subject: [PATCH] first stab at bonsai reference test worldstate Signed-off-by: garyschulte --- .../bonsai/worldview/BonsaiWorldState.java | 56 +++---- ethereum/referencetests/build.gradle | 4 +- .../BonsaiReferenceTestWorldState.java | 151 ++++++++++++++++++ .../VMReferenceTestCaseSpec.java | 8 +- 4 files changed, 178 insertions(+), 41 deletions(-) create mode 100644 ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java index d3f14e8e72e..ed50f86c610 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount; import org.hyperledger.besu.ethereum.bonsai.BonsaiValue; import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; @@ -64,7 +65,8 @@ public class BonsaiWorldState public BonsaiWorldStateKeyValueStorage worldStateStorage; - private final BonsaiWorldStateProvider archive; + private final CachedMerkleTrieLoader cachedMerkleTrieLoader; + private final TrieLogManager trieLogManager; private final BonsaiWorldStateUpdateAccumulator accumulator; public Hash worldStateRootHash; @@ -75,42 +77,29 @@ public class BonsaiWorldState public BonsaiWorldState( final BonsaiWorldStateProvider archive, final BonsaiWorldStateKeyValueStorage worldStateStorage) { - this.archive = archive; - this.worldStateStorage = worldStateStorage; - worldStateRootHash = - Hash.wrap( - Bytes32.wrap(worldStateStorage.getWorldStateRootHash().orElse(Hash.EMPTY_TRIE_HASH))); - worldStateBlockHash = - Hash.wrap(Bytes32.wrap(worldStateStorage.getWorldStateBlockHash().orElse(Hash.ZERO))); - accumulator = - new BonsaiWorldStateUpdateAccumulator( - this, - (addr, value) -> - archive - .getCachedMerkleTrieLoader() - .preLoadAccount(getWorldStateStorage(), worldStateRootHash, addr), - (addr, value) -> - archive - .getCachedMerkleTrieLoader() - .preLoadStorageSlot(getWorldStateStorage(), addr, value)); + this(worldStateStorage, archive.getCachedMerkleTrieLoader(), archive.getTrieLogManager()); } - public BonsaiWorldState( - final BonsaiWorldStateProvider archive, + protected BonsaiWorldState( final BonsaiWorldStateKeyValueStorage worldStateStorage, - final BonsaiWorldStateUpdateAccumulator updater) { - this.archive = archive; + final CachedMerkleTrieLoader cachedMerkleTrieLoader, + final TrieLogManager trieLogManager) { this.worldStateStorage = worldStateStorage; this.worldStateRootHash = Hash.wrap( Bytes32.wrap(worldStateStorage.getWorldStateRootHash().orElse(Hash.EMPTY_TRIE_HASH))); this.worldStateBlockHash = Hash.wrap(Bytes32.wrap(worldStateStorage.getWorldStateBlockHash().orElse(Hash.ZERO))); - this.accumulator = updater; - } - - public BonsaiWorldStateProvider getArchive() { - return archive; + accumulator = + new BonsaiWorldStateUpdateAccumulator( + this, + (addr, value) -> + cachedMerkleTrieLoader.preLoadAccount( + getWorldStateStorage(), worldStateRootHash, addr), + (addr, value) -> + cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value)); + this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; + this.trieLogManager = trieLogManager; } @Override @@ -168,9 +157,7 @@ private Hash calculateRootHash( final StoredMerklePatriciaTrie accountTrie = createTrie( (location, hash) -> - archive - .getCachedMerkleTrieLoader() - .getAccountStateTrieNode(worldStateStorage, location, hash), + cachedMerkleTrieLoader.getAccountStateTrieNode(worldStateStorage, location, hash), worldStateRootHash); // for manicured tries and composting, collect branches here (not implemented) @@ -253,10 +240,8 @@ private void updateAccountStorageState( final StoredMerklePatriciaTrie storageTrie = createTrie( (location, key) -> - archive - .getCachedMerkleTrieLoader() - .getAccountStorageTrieNode( - worldStateStorage, updatedAddressHash, location, key), + cachedMerkleTrieLoader.getAccountStorageTrieNode( + worldStateStorage, updatedAddressHash, location, key), storageRoot); // for manicured tries and composting, collect branches here (not implemented) @@ -382,7 +367,6 @@ public void persist(final BlockHeader blockHeader) { } saveTrieLog = () -> { - final TrieLogManager trieLogManager = archive.getTrieLogManager(); trieLogManager.saveTrieLog(localCopy, newWorldStateRootHash, blockHeader, this); // not save a frozen state in the cache if (!isFrozen) { diff --git a/ethereum/referencetests/build.gradle b/ethereum/referencetests/build.gradle index 49dc6109074..326e26ed096 100644 --- a/ethereum/referencetests/build.gradle +++ b/ethereum/referencetests/build.gradle @@ -115,6 +115,8 @@ dependencies { implementation project(':crypto:algorithms') implementation project(':datatypes') implementation project(':ethereum:core') + implementation project(':metrics:core') + implementation project(':util') implementation project(path: ':ethereum:core', configuration: 'testSupportArtifacts') implementation project(':ethereum:rlp') implementation project(':evm') @@ -126,7 +128,7 @@ dependencies { referenceTestImplementation project(path: ':config') referenceTestImplementation project(path: ':datatypes') referenceTestImplementation project(path: ':ethereum:core') - referenceTestImplementation project(path: ':ethereum:core', configuration: 'testSupportArtifacts') + referenceTestImplementation project(path: ':ethereum:core') referenceTestImplementation project(path: ':ethereum:rlp') referenceTestImplementation project(path: ':ethereum:rlp', configuration: 'testSupportArtifacts') referenceTestImplementation project(path: ':ethereum:trie') diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java new file mode 100644 index 00000000000..10f3097f73c --- /dev/null +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -0,0 +1,151 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.referencetests; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogAddedEvent; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; +import org.hyperledger.besu.evm.account.MutableAccount; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; +import org.hyperledger.besu.metrics.ObservableMetricsSystem; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.trielogs.TrieLog; +import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent; +import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory; +import org.hyperledger.besu.util.Subscribers; + +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +import com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.tuweni.units.bigints.UInt256; + +public class BonsaiReferenceTestWorldState extends BonsaiWorldState { + + protected BonsaiReferenceTestWorldState( + final BonsaiWorldStateKeyValueStorage worldStateStorage, + final CachedMerkleTrieLoader cachedMerkleTrieLoader, + final TrieLogManager trieLogManager) { + super(worldStateStorage, cachedMerkleTrieLoader, trieLogManager); + } + + @JsonCreator + public static BonsaiReferenceTestWorldState create( + final Map accounts) { + final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem(); + final CachedMerkleTrieLoader cachedMerkleTrieLoader = new CachedMerkleTrieLoader(metricsSystem); + final TrieLogManager trieLogManager = new NoOpTrieLogManager(); + final BonsaiWorldStateKeyValueStorage worldStateStorage = + new BonsaiWorldStateKeyValueStorage(new InMemoryKeyValueStorageProvider(), metricsSystem); + final BonsaiReferenceTestWorldState worldState = + new BonsaiReferenceTestWorldState( + worldStateStorage, cachedMerkleTrieLoader, trieLogManager); + + final WorldUpdater updater = worldState.updater(); + for (final Map.Entry entry : accounts.entrySet()) { + insertAccount(updater, Address.fromHexString(entry.getKey()), entry.getValue()); + } + updater.commit(); + return worldState; + } + + static void insertAccount( + final WorldUpdater updater, + final Address address, + final ReferenceTestWorldState.AccountMock toCopy) { + final MutableAccount account = updater.getOrCreate(address).getMutable(); + account.setNonce(toCopy.getNonce()); + account.setBalance(toCopy.getBalance()); + account.setCode(toCopy.getCode()); + for (final Map.Entry entry : toCopy.getStorage().entrySet()) { + account.setStorageValue(entry.getKey(), entry.getValue()); + } + } + + static class NoOpTrieLogManager implements TrieLogManager { + private final Subscribers trieLogObservers = Subscribers.create(); + private final TrieLogFactory trieLogFactory = new TrieLogFactoryImpl(); + + @Override + public void saveTrieLog( + final BonsaiWorldStateUpdateAccumulator localUpdater, + final Hash forWorldStateRootHash, + final BlockHeader forBlockHeader, + final BonsaiWorldState forWorldState) { + // notify trie log added observers, synchronously + TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); + trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); + } + + @Override + public void addCachedLayer( + final BlockHeader blockHeader, + final Hash worldStateRootHash, + final BonsaiWorldState forWorldState) {} + + @Override + public boolean containWorldStateStorage(final Hash blockHash) { + return false; + } + + @Override + public Optional getWorldState(final Hash blockHash) { + return Optional.empty(); + } + + @Override + public Optional getNearestWorldState(final BlockHeader blockHeader) { + return Optional.empty(); + } + + @Override + public Optional getHeadWorldState( + final Function> hashBlockHeaderFunction) { + return Optional.empty(); + } + + @Override + public long getMaxLayersToLoad() { + return 0; + } + + @Override + public void reset() {} + + @Override + public Optional getTrieLogLayer(final Hash blockHash) { + return Optional.empty(); + } + + @Override + public synchronized long subscribe(final TrieLogEvent.TrieLogObserver sub) { + return trieLogObservers.subscribe(sub); + } + + @Override + public synchronized void unsubscribe(final long id) { + trieLogObservers.unsubscribe(id); + } + } +} diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/VMReferenceTestCaseSpec.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/VMReferenceTestCaseSpec.java index ba1a9e00602..54dddbba502 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/VMReferenceTestCaseSpec.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/VMReferenceTestCaseSpec.java @@ -38,11 +38,11 @@ public class VMReferenceTestCaseSpec { private final long finalGas; - private final ReferenceTestWorldState initialWorldState; + private final BonsaiReferenceTestWorldState initialWorldState; private final boolean exceptionalHaltExpected; - private final ReferenceTestWorldState finalWorldState; + private final BonsaiReferenceTestWorldState finalWorldState; @JsonCreator public VMReferenceTestCaseSpec( @@ -50,8 +50,8 @@ public VMReferenceTestCaseSpec( @JsonProperty("env") final ReferenceTestEnv env, @JsonProperty("gas") final String finalGas, @JsonProperty("out") final String out, - @JsonProperty("pre") final ReferenceTestWorldState initialWorldState, - @JsonProperty("post") final ReferenceTestWorldState finalWorldState) { + @JsonProperty("pre") final BonsaiReferenceTestWorldState initialWorldState, + @JsonProperty("post") final BonsaiReferenceTestWorldState finalWorldState) { this.exec = exec; this.initialWorldState = initialWorldState; this.initialWorldState.persist(null);