Bonsai TrieLogFactory serializer/deserializer (hyperledger#5372)

* add account zero reads, mark self-destructed accounts, code and storage as cleared
* refactor BonsaiWorldView and TrieLog to use StorageSlotKey record type rather than slotHash
* add isCleared to read/write of BonsaiValues
* for consistency between serialize and deserialize, use null directly in TrieLogLayer.addCodeChange rather than Bytes.EMPTY

Signed-off-by: garyschulte <garyschulte@gmail.com>

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiValue.java

@@ -77,6 +77,11 @@ public void writeInnerRlp(final RLPOutput output, final BiConsumer<RLPOutput, T>
     } else {
       writer.accept(output, updated);
     }
+    if (!cleared) {
+      output.writeNull();
+    } else {
+      output.writeInt(1);
+    }
   }
 
   public boolean isUnchanged() {

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedMerkleTrieLoader.java

@@ -19,6 +19,7 @@
 import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
 import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber;
+import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey;
 import org.hyperledger.besu.ethereum.trie.MerkleTrie;
 import org.hyperledger.besu.ethereum.trie.MerkleTrieException;
 import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie;
@@ -92,15 +93,15 @@ public void cacheAccountNodes(
   public void preLoadStorageSlot(
       final BonsaiWorldStateKeyValueStorage worldStateStorage,
       final Address account,
-      final Hash slotHash) {
-    CompletableFuture.runAsync(() -> cacheStorageNodes(worldStateStorage, account, slotHash));
+      final StorageSlotKey slotKey) {
+    CompletableFuture.runAsync(() -> cacheStorageNodes(worldStateStorage, account, slotKey));
   }
 
   @VisibleForTesting
   public void cacheStorageNodes(
       final BonsaiWorldStateKeyValueStorage worldStateStorage,
       final Address account,
-      final Hash slotHash) {
+      final StorageSlotKey slotKey) {
     final Hash accountHash = Hash.hash(account);
     final long storageSubscriberId = worldStateStorage.subscribe(this);
     try {
@@ -121,7 +122,7 @@ public void cacheStorageNodes(
                           Hash.hash(storageRoot),
                           Function.identity(),
                           Function.identity());
-                  storageTrie.get(slotHash);
+                  storageTrie.get(slotKey.slotHash());
                 } catch (MerkleTrieException e) {
                   // ignore exception for the cache
                 }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java

@@ -17,6 +17,7 @@
 
 import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber;
+import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey;
 import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 import org.hyperledger.besu.plugin.services.exception.StorageException;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
@@ -122,20 +123,21 @@ public Optional<Hash> getWorldStateBlockHash() {
   }
 
   @Override
-  public Optional<Bytes> getStorageValueBySlotHash(final Hash accountHash, final Hash slotHash) {
+  public Optional<Bytes> getStorageValueByStorageSlotKey(
+      final Hash accountHash, final StorageSlotKey storageSlotKey) {
     return isClosed.get()
         ? Optional.empty()
-        : super.getStorageValueBySlotHash(accountHash, slotHash);
+        : super.getStorageValueByStorageSlotKey(accountHash, storageSlotKey);
   }
 
   @Override
-  public Optional<Bytes> getStorageValueBySlotHash(
+  public Optional<Bytes> getStorageValueByStorageSlotKey(
       final Supplier<Optional<Hash>> storageRootSupplier,
       final Hash accountHash,
-      final Hash slotHash) {
+      final StorageSlotKey storageSlotKey) {
     return isClosed.get()
         ? Optional.empty()
-        : super.getStorageValueBySlotHash(storageRootSupplier, accountHash, slotHash);
+        : super.getStorageValueByStorageSlotKey(storageRootSupplier, accountHash, storageSlotKey);
   }
 
   @Override

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java

@@ -15,6 +15,7 @@
 package org.hyperledger.besu.ethereum.bonsai.storage;
 
 import org.hyperledger.besu.datatypes.Hash;
+import org.hyperledger.besu.ethereum.bonsai.worldview.StorageSlotKey;
 import org.hyperledger.besu.ethereum.storage.StorageProvider;
 import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier;
 import org.hyperledger.besu.ethereum.trie.MerkleTrie;
@@ -232,8 +233,9 @@ public Optional<Hash> getWorldStateBlockHash() {
     return trieBranchStorage.get(WORLD_BLOCK_HASH_KEY).map(Bytes32::wrap).map(Hash::wrap);
   }
 
-  public Optional<Bytes> getStorageValueBySlotHash(final Hash accountHash, final Hash slotHash) {
-    return getStorageValueBySlotHash(
+  public Optional<Bytes> getStorageValueByStorageSlotKey(
+      final Hash accountHash, final StorageSlotKey storageSlotKey) {
+    return getStorageValueByStorageSlotKey(
         () ->
             getAccount(accountHash)
                 .map(
@@ -242,17 +244,17 @@ public Optional<Bytes> getStorageValueBySlotHash(final Hash accountHash, final H
                                 org.hyperledger.besu.ethereum.rlp.RLP.input(b))
                             .getStorageRoot()),
         accountHash,
-        slotHash);
+        storageSlotKey);
   }
 
-  public Optional<Bytes> getStorageValueBySlotHash(
+  public Optional<Bytes> getStorageValueByStorageSlotKey(
       final Supplier<Optional<Hash>> storageRootSupplier,
       final Hash accountHash,
-      final Hash slotHash) {
+      final StorageSlotKey storageSlotKey) {
     getStorageValueCounter.inc();
     Optional<Bytes> response =
         storageStorage
-            .get(Bytes.concatenate(accountHash, slotHash).toArrayUnsafe())
+            .get(Bytes.concatenate(accountHash, storageSlotKey.slotHash()).toArrayUnsafe())
             .map(Bytes::wrap);
     if (response.isEmpty()) {
       final Optional<Hash> storageRoot = storageRootSupplier.get();
@@ -265,7 +267,7 @@ public Optional<Bytes> getStorageValueBySlotHash(
                         Function.identity(),
                         Function.identity()),
                     storageRoot.get())
-                .get(slotHash)
+                .get(storageSlotKey.slotHash())
                 .map(bytes -> Bytes32.leftPad(RLP.decodeValue(bytes)));
         if (response.isEmpty()) getStorageValueMissingMerkleTrieCounter.inc();
         else getStorageValueMerkleTrieCounter.inc();

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/AbstractTrieLogManager.java

@@ -24,7 +24,6 @@
 import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator;
 import org.hyperledger.besu.ethereum.chain.Blockchain;
 import org.hyperledger.besu.ethereum.core.BlockHeader;
-import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput;
 import org.hyperledger.besu.util.Subscribers;
 
 import java.util.Map;
@@ -45,6 +44,9 @@ public abstract class AbstractTrieLogManager implements TrieLogManager {
   protected final long maxLayersToLoad;
   private final Subscribers<TrieLogAddedObserver> trieLogAddedObservers = Subscribers.create();
 
+  // TODO plumb factory from plugin service:
+  TrieLogFactory<TrieLogLayer> trieLogFactory = new TrieLogFactoryImpl();
+
   protected AbstractTrieLogManager(
       final Blockchain blockchain,
       final BonsaiWorldStateKeyValueStorage worldStateStorage,
@@ -122,11 +124,10 @@ private void persistTrieLog(
         .addArgument(blockHeader::toLogString)
         .addArgument(worldStateRootHash::toHexString)
         .log();
-    final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput();
-    trieLog.writeTo(rlpLog);
+
     stateUpdater
         .getTrieLogStorageTransaction()
-        .put(blockHeader.getHash().toArrayUnsafe(), rlpLog.encoded().toArrayUnsafe());
+        .put(blockHeader.getHash().toArrayUnsafe(), trieLogFactory.serialize(trieLog));
   }
 
   @Override
@@ -141,7 +142,7 @@ public long getMaxLayersToLoad() {
 
   @Override
   public Optional<TrieLogLayer> getTrieLogLayer(final Hash blockHash) {
-    return rootWorldStateStorage.getTrieLog(blockHash).map(TrieLogLayer::fromBytes);
+    return rootWorldStateStorage.getTrieLog(blockHash).map(trieLogFactory::deserialize);
   }
 
   @Override

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogFactory.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright Hyperledger Besu Contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+package org.hyperledger.besu.ethereum.bonsai.trielog;
+
+import org.hyperledger.besu.datatypes.Hash;
+import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator;
+
+/** Interface for serializing and deserializing {@link TrieLogLayer} objects. */
+public interface TrieLogFactory<T extends TrieLogLayer> {
+  T create(BonsaiWorldStateUpdateAccumulator accumulator, final Hash blockHash);
+
+  T deserialize(final byte[] bytes);
+
+  byte[] serialize(final T layer);
+}