minimal approach to preimage hash store. missing optimizations and ex…

…plicit coverage

Signed-off-by: garyschulte <garyschulte@gmail.com>

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/WorldStatePreimageKeyValueStorage.java

@@ -25,16 +25,6 @@
 import org.apache.tuweni.bytes.Bytes32;
 import org.apache.tuweni.units.bigints.UInt256;
 
-/**
- * This store is used in Forest storage format, and expects to use an Updater to insert and commit
- * changes to storage, adjacent to worldstate commits. By not being part of the worldstate
- * transaction and commit data in this implementation can become out-of-sync (by failing to commit
- * preimages by hash) on abnormal exits of besu.
- *
- * <p>This implementation remains here for the deprecated forest storage format, and should be
- * retired along with Forest.
- */
-@Deprecated
 public class WorldStatePreimageKeyValueStorage implements WorldStatePreimageStorage {
   private final KeyValueStorage keyValueStorage;
 
@@ -59,6 +49,11 @@ public Optional<Address> getAccountTrieKeyPreimage(final Bytes32 trieKey) {
         .map(val -> Address.wrap(Bytes.wrap(val)));
   }
 
+  @Override
+  public boolean canSupportStreaming() {
+    return keyValueStorage.isPersistent();
+  }
+
   @Override
   public Updater updater() {
     return new Updater(keyValueStorage.startTransaction());

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java

@@ -44,7 +44,10 @@ public BonsaiSnapshotWorldStateKeyValueStorage(
       final SnappedKeyValueStorage segmentedWorldStateStorage,
       final KeyValueStorage trieLogStorage) {
     super(
-        parentWorldStateStorage.flatDbStrategyProvider, segmentedWorldStateStorage, trieLogStorage);
+        parentWorldStateStorage.flatDbStrategyProvider,
+        segmentedWorldStateStorage,
+        trieLogStorage,
+        parentWorldStateStorage.getPreimageStorage());
     this.parentWorldStateStorage = parentWorldStateStorage;
     this.subscribeParentId = parentWorldStateStorage.subscribe(this);
   }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorage.java

@@ -32,6 +32,7 @@
 import org.hyperledger.besu.ethereum.worldstate.FlatDbMode;
 import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage;
+import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
@@ -58,7 +59,8 @@ public BonsaiWorldStateKeyValueStorage(
         provider.getStorageBySegmentIdentifiers(
             List.of(
                 ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE)),
-        provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE));
+        provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE),
+        provider.createWorldStatePreimageStorage());
     this.flatDbStrategyProvider =
         new BonsaiFlatDbStrategyProvider(metricsSystem, dataStorageConfiguration);
     flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage);
@@ -67,8 +69,9 @@ public BonsaiWorldStateKeyValueStorage(
   public BonsaiWorldStateKeyValueStorage(
       final BonsaiFlatDbStrategyProvider flatDbStrategyProvider,
       final SegmentedKeyValueStorage composedWorldStateStorage,
-      final KeyValueStorage trieLogStorage) {
-    super(composedWorldStateStorage, trieLogStorage);
+      final KeyValueStorage trieLogStorage,
+      final WorldStatePreimageStorage preimageStorage) {
+    super(composedWorldStateStorage, trieLogStorage, preimageStorage);
     this.flatDbStrategyProvider = flatDbStrategyProvider;
   }
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java

@@ -187,22 +187,19 @@ private void updateTheAccounts(
       final MerkleTrie<Bytes, Bytes> accountTrie) {
     for (final Map.Entry<Address, DiffBasedValue<BonsaiAccount>> accountUpdate :
         worldStateUpdater.getAccountsToUpdate().entrySet()) {
-      final Bytes accountKey = accountUpdate.getKey();
+      final Address accountKey = accountUpdate.getKey();
       final DiffBasedValue<BonsaiAccount> bonsaiValue = accountUpdate.getValue();
       final BonsaiAccount updatedAccount = bonsaiValue.getUpdated();
       try {
         if (updatedAccount == null) {
-          final Hash addressHash = getPreImageProxy().hashAndSaveAddressPreImage(accountKey);
-          accountTrie.remove(addressHash);
+          accountTrie.remove(accountKey.addressHash());
           maybeStateUpdater.ifPresent(
-              bonsaiUpdater -> bonsaiUpdater.removeAccountInfoState(addressHash));
+              bonsaiUpdater -> bonsaiUpdater.removeAccountInfoState(accountKey.addressHash()));
         } else {
           final Hash addressHash = updatedAccount.getAddressHash();
           final Bytes accountValue = updatedAccount.serializeAccount();
           maybeStateUpdater.ifPresent(
-              bonsaiUpdater ->
-                  bonsaiUpdater.putAccountInfoState(
-                      getPreImageProxy().hashAndSaveAddressPreImage(accountKey), accountValue));
+              bonsaiUpdater -> bonsaiUpdater.putAccountInfoState(addressHash, accountValue));
           accountTrie.put(addressHash, accountValue);
         }
       } catch (MerkleTrieException e) {

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedWorldStateProvider.java

@@ -29,6 +29,7 @@
 import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldStateConfig;
 import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
+import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator;
 import org.hyperledger.besu.evm.worldstate.WorldState;
 import org.hyperledger.besu.plugin.BesuContext;
@@ -51,6 +52,7 @@ public abstract class DiffBasedWorldStateProvider implements WorldStateArchive {
   protected final Blockchain blockchain;
 
   protected final TrieLogManager trieLogManager;
+  protected final WorldStatePreimageStorage preimageStorage;
   protected DiffBasedCachedWorldStorageManager cachedWorldStorageManager;
   protected DiffBasedWorldState persistedState;
 
@@ -71,6 +73,7 @@ public DiffBasedWorldStateProvider(
             worldStateKeyValueStorage,
             maxLayersToLoad.orElse(DiffBasedCachedWorldStorageManager.RETAINED_LAYERS),
             pluginContext);
+    this.preimageStorage = worldStateKeyValueStorage.getPreimageStorage();
     this.blockchain = blockchain;
     this.defaultWorldStateConfig = new DiffBasedWorldStateConfig();
   }
@@ -85,6 +88,7 @@ public DiffBasedWorldStateProvider(
     this.trieLogManager = trieLogManager;
     this.blockchain = blockchain;
     this.defaultWorldStateConfig = new DiffBasedWorldStateConfig();
+    preimageStorage = worldStateKeyValueStorage.getPreimageStorage();
   }
 
   protected void provideCachedWorldStorageManager(

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedWorldStateKeyValueStorage.java

@@ -24,12 +24,12 @@
 import org.hyperledger.besu.ethereum.storage.StorageProvider;
 import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier;
 import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount;
-import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.CachingPreImageStorage;
 import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber;
 import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy;
 import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView;
 import org.hyperledger.besu.ethereum.worldstate.FlatDbMode;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage;
+import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage;
 import org.hyperledger.besu.evm.account.AccountStorageEntry;
 import org.hyperledger.besu.evm.worldstate.WorldState;
 import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
@@ -64,10 +64,7 @@ public abstract class DiffBasedWorldStateKeyValueStorage
   private static final Logger LOG =
       LoggerFactory.getLogger(DiffBasedWorldStateKeyValueStorage.class);
 
-  // TODO: config this based on dataStorageConfiguration, use real storage where appropriate
-  //   currently hard-coded to Unbounded preimage store so ref/spec tests pass
-  private static final CachingPreImageStorage preImageProxy =
-      new CachingPreImageStorage.UnboundedPreImageStorage();
+  private final WorldStatePreimageStorage preImageProxy;
 
   // 0x776f726c64526f6f74
   public static final byte[] WORLD_ROOT_HASH_KEY = "worldRoot".getBytes(StandardCharsets.UTF_8);
@@ -90,13 +87,16 @@ public DiffBasedWorldStateKeyValueStorage(final StorageProvider provider) {
                 ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE));
     this.trieLogStorage =
         provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE);
+    this.preImageProxy = provider.createWorldStatePreimageStorage();
   }
 
   public DiffBasedWorldStateKeyValueStorage(
       final SegmentedKeyValueStorage composedWorldStateStorage,
-      final KeyValueStorage trieLogStorage) {
+      final KeyValueStorage trieLogStorage,
+      final WorldStatePreimageStorage preImageStorage) {
     this.composedWorldStateStorage = composedWorldStateStorage;
     this.trieLogStorage = trieLogStorage;
+    this.preImageProxy = preImageStorage;
   }
 
   public abstract FlatDbMode getFlatDbMode();
@@ -110,14 +110,14 @@ public SegmentedKeyValueStorage getComposedWorldStateStorage() {
     return composedWorldStateStorage;
   }
 
-  public CachingPreImageStorage getPreImageProxy() {
-    return preImageProxy;
-  }
-
   public KeyValueStorage getTrieLogStorage() {
     return trieLogStorage;
   }
 
+  public WorldStatePreimageStorage getPreimageStorage() {
+    return preImageProxy;
+  }
+
   public Optional<byte[]> getTrieLog(final Hash blockHash) {
     return trieLogStorage.get(blockHash.toArrayUnsafe());
   }