Add metrics for accounts and storage reads (Flat databasae vs MPT) (h…

…yperledger#5315) Add 8 metrics to evaluate the state of the flat database for accounts and storage slots * Total number of calls (for both accounts and storage) * Number of account or storage entries found in the flat database * Number of storage slots located in the merkle trie but not found in the flat database * Number of account or storage entries not found in either the flat database or the merkle trie Signed-off-by: Ameziane H <ameziane.hamlat@consensys.net> Co-authored-by: Sally MacFarlane <macfarla.github@gmail.com>
lacchain · Nov 3, 2023 · 963da0a · 963da0a
1 parent c2aee02
commit 963da0a
Show file tree

Hide file tree

Showing 20 changed files with 238 additions and 58 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,7 @@
 
 ### Additions and Improvements
 - Update most dependencies to latest version [#5269](https://github.com/hyperledger/besu/pull/5269)
+- Add metrics for accounts and storage reads (Flat database vs Merkle Patricia Trie) [#5315](https://github.com/hyperledger/besu/pull/5315)
 
 ### Bug Fixes
 

diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java
@@ -945,7 +945,8 @@ WorldStateArchive createWorldStateArchive(
             (BonsaiWorldStateKeyValueStorage) worldStateStorage,
             blockchain,
             Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()),
-            cachedMerkleTrieLoader);
+            cachedMerkleTrieLoader,
+            metricsSystem);
 
       case FOREST:
       default:

diff --git a/...eum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java b/...eum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java
@@ -39,6 +39,7 @@
 import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
 import org.hyperledger.besu.evm.worldstate.WorldState;
+import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 
 import java.util.ArrayList;
 import java.util.HashSet;
@@ -68,25 +69,32 @@ public class BonsaiWorldStateProvider implements WorldStateArchive {
   public BonsaiWorldStateProvider(
       final StorageProvider provider,
       final Blockchain blockchain,
-      final CachedMerkleTrieLoader cachedMerkleTrieLoader) {
+      final CachedMerkleTrieLoader cachedMerkleTrieLoader,
+      final ObservableMetricsSystem metricsSystem) {
     this(
         (BonsaiWorldStateKeyValueStorage)
             provider.createWorldStateStorage(DataStorageFormat.BONSAI),
         blockchain,
         Optional.empty(),
-        cachedMerkleTrieLoader);
+        cachedMerkleTrieLoader,
+        metricsSystem);
   }
 
   public BonsaiWorldStateProvider(
       final BonsaiWorldStateKeyValueStorage worldStateStorage,
       final Blockchain blockchain,
       final Optional<Long> maxLayersToLoad,
-      final CachedMerkleTrieLoader cachedMerkleTrieLoader) {
+      final CachedMerkleTrieLoader cachedMerkleTrieLoader,
+      final ObservableMetricsSystem metricsSystem) {
 
     // TODO: de-dup constructors
     this.trieLogManager =
         new CachedWorldStorageManager(
-            this, blockchain, worldStateStorage, maxLayersToLoad.orElse(RETAINED_LAYERS));
+            this,
+            blockchain,
+            worldStateStorage,
+            metricsSystem,
+            maxLayersToLoad.orElse(RETAINED_LAYERS));
     this.blockchain = blockchain;
     this.worldStateStorage = worldStateStorage;
     this.persistedState = new BonsaiWorldState(this, worldStateStorage);

diff --git a/...e/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java b/...e/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java
@@ -24,6 +24,7 @@
 import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState;
 import org.hyperledger.besu.ethereum.chain.Blockchain;
 import org.hyperledger.besu.ethereum.core.BlockHeader;
+import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 
 import java.util.ArrayList;
 import java.util.Comparator;
@@ -41,24 +42,34 @@ public class CachedWorldStorageManager extends AbstractTrieLogManager
     implements BonsaiStorageSubscriber {
   private static final Logger LOG = LoggerFactory.getLogger(CachedWorldStorageManager.class);
   private final BonsaiWorldStateProvider archive;
+  private final ObservableMetricsSystem metricsSystem;
 
   CachedWorldStorageManager(
       final BonsaiWorldStateProvider archive,
       final Blockchain blockchain,
       final BonsaiWorldStateKeyValueStorage worldStateStorage,
       final long maxLayersToLoad,
-      final Map<Bytes32, CachedBonsaiWorldView> cachedWorldStatesByHash) {
+      final Map<Bytes32, CachedBonsaiWorldView> cachedWorldStatesByHash,
+      final ObservableMetricsSystem metricsSystem) {
     super(blockchain, worldStateStorage, maxLayersToLoad, cachedWorldStatesByHash);
     worldStateStorage.subscribe(this);
     this.archive = archive;
+    this.metricsSystem = metricsSystem;
   }
 
   public CachedWorldStorageManager(
       final BonsaiWorldStateProvider archive,
       final Blockchain blockchain,
       final BonsaiWorldStateKeyValueStorage worldStateStorage,
+      final ObservableMetricsSystem metricsSystem,
       final long maxLayersToLoad) {
-    this(archive, blockchain, worldStateStorage, maxLayersToLoad, new ConcurrentHashMap<>());
+    this(
+        archive,
+        blockchain,
+        worldStateStorage,
+        maxLayersToLoad,
+        new ConcurrentHashMap<>(),
+        metricsSystem);
   }
 
   @Override
@@ -81,7 +92,8 @@ public synchronized void addCachedLayer(
         cachedBonsaiWorldView
             .get()
             .updateWorldStateStorage(
-                new BonsaiSnapshotWorldStateKeyValueStorage(forWorldState.worldStateStorage));
+                new BonsaiSnapshotWorldStateKeyValueStorage(
+                    forWorldState.worldStateStorage, metricsSystem));
       }
     } else {
       LOG.atDebug()
@@ -94,7 +106,8 @@ public synchronized void addCachedLayer(
             blockHeader.getHash(),
             new CachedBonsaiWorldView(
                 blockHeader,
-                new BonsaiSnapshotWorldStateKeyValueStorage(forWorldState.worldStateStorage)));
+                new BonsaiSnapshotWorldStateKeyValueStorage(
+                    forWorldState.worldStateStorage, metricsSystem)));
       } else {
         // otherwise, add the layer to the cache
         cachedWorldStatesByHash.put(

diff --git a/...org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java b/...org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java
@@ -17,6 +17,7 @@
 
 import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber;
+import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 import org.hyperledger.besu.plugin.services.exception.StorageException;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
 import org.hyperledger.besu.plugin.services.storage.SnappableKeyValueStorage;
@@ -41,21 +42,30 @@ public BonsaiSnapshotWorldStateKeyValueStorage(
       final SnappedKeyValueStorage codeStorage,
       final SnappedKeyValueStorage storageStorage,
       final SnappedKeyValueStorage trieBranchStorage,
-      final KeyValueStorage trieLogStorage) {
-    super(accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage);
+      final KeyValueStorage trieLogStorage,
+      final ObservableMetricsSystem metricsSystem) {
+    super(
+        accountStorage,
+        codeStorage,
+        storageStorage,
+        trieBranchStorage,
+        trieLogStorage,
+        metricsSystem);
     this.parentWorldStateStorage = parentWorldStateStorage;
     this.subscribeParentId = parentWorldStateStorage.subscribe(this);
   }
 
   public BonsaiSnapshotWorldStateKeyValueStorage(
-      final BonsaiWorldStateKeyValueStorage worldStateStorage) {
+      final BonsaiWorldStateKeyValueStorage worldStateStorage,
+      final ObservableMetricsSystem metricsSystem) {
     this(
         worldStateStorage,
         ((SnappableKeyValueStorage) worldStateStorage.accountStorage).takeSnapshot(),
         ((SnappableKeyValueStorage) worldStateStorage.codeStorage).takeSnapshot(),
         ((SnappableKeyValueStorage) worldStateStorage.storageStorage).takeSnapshot(),
         ((SnappableKeyValueStorage) worldStateStorage.trieBranchStorage).takeSnapshot(),
-        worldStateStorage.trieLogStorage);
+        worldStateStorage.trieLogStorage,
+        metricsSystem);
   }
 
   @Override

diff --git a/...in/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/...in/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java
@@ -22,6 +22,9 @@
 import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory;
 import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
+import org.hyperledger.besu.metrics.BesuMetricCategory;
+import org.hyperledger.besu.metrics.ObservableMetricsSystem;
+import org.hyperledger.besu.plugin.services.metrics.Counter;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction;
 import org.hyperledger.besu.util.Subscribers;
@@ -55,32 +58,96 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC
   protected final KeyValueStorage trieBranchStorage;
   protected final KeyValueStorage trieLogStorage;
 
+  private final Counter getAccountCounter;
+  private final Counter getAccountFlatDatabaseCounter;
+  private final Counter getAccountMerkleTrieCounter;
+  private final Counter getAccountMissingMerkleTrieCounter;
+
+  private final Counter getStorageValueCounter;
+  private final Counter getStorageValueFlatDatabaseCounter;
+  private final Counter getStorageValueMerkleTrieCounter;
+  private final Counter getStorageValueMissingMerkleTrieCounter;
+
+  protected final ObservableMetricsSystem metricsSystem;
+
   private final AtomicBoolean shouldClose = new AtomicBoolean(false);
 
   protected final AtomicBoolean isClosed = new AtomicBoolean(false);
 
   protected final Subscribers<BonsaiStorageSubscriber> subscribers = Subscribers.create();
 
-  public BonsaiWorldStateKeyValueStorage(final StorageProvider provider) {
+  public BonsaiWorldStateKeyValueStorage(
+      final StorageProvider provider, final ObservableMetricsSystem metricsSystem) {
     this(
         provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE),
         provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE),
         provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE),
         provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE),
-        provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE));
+        provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE),
+        metricsSystem);
   }
 
   public BonsaiWorldStateKeyValueStorage(
       final KeyValueStorage accountStorage,
       final KeyValueStorage codeStorage,
       final KeyValueStorage storageStorage,
       final KeyValueStorage trieBranchStorage,
-      final KeyValueStorage trieLogStorage) {
+      final KeyValueStorage trieLogStorage,
+      final ObservableMetricsSystem metricsSystem) {
     this.accountStorage = accountStorage;
     this.codeStorage = codeStorage;
     this.storageStorage = storageStorage;
     this.trieBranchStorage = trieBranchStorage;
     this.trieLogStorage = trieLogStorage;
+    this.metricsSystem = metricsSystem;
+
+    getAccountCounter =
+        metricsSystem.createCounter(
+            BesuMetricCategory.BLOCKCHAIN,
+            "get_account_total",
+            "Total number of calls to getAccount");
+
+    getAccountFlatDatabaseCounter =
+        metricsSystem.createCounter(
+            BesuMetricCategory.BLOCKCHAIN,
+            "get_account_flat_database",
+            "Number of accounts found in the flat database");
+
+    getAccountMerkleTrieCounter =
+        metricsSystem.createCounter(
+            BesuMetricCategory.BLOCKCHAIN,
+            "get_account_merkle_trie",
+            "Number of accounts not found in the flat database, but found in the merkle trie");
+
+    getAccountMissingMerkleTrieCounter =
+        metricsSystem.createCounter(
+            BesuMetricCategory.BLOCKCHAIN,
+            "get_account_missing_merkle_trie",
+            "Number of accounts not found (either in the flat database or the merkle trie)");
+
+    getStorageValueCounter =
+        metricsSystem.createCounter(
+            BesuMetricCategory.BLOCKCHAIN,
+            "get_storagevalue_total",
+            "Total number of calls to getStorageValueBySlotHash");
+
+    getStorageValueFlatDatabaseCounter =
+        metricsSystem.createCounter(
+            BesuMetricCategory.BLOCKCHAIN,
+            "get_storagevalue_flat_database",
+            "Number of storage slots found in the flat database");
+
+    getStorageValueMerkleTrieCounter =
+        metricsSystem.createCounter(
+            BesuMetricCategory.BLOCKCHAIN,
+            "get_storagevalue_merkle_trie",
+            "Number of storage slots not found in the flat database, but found in the merkle trie");
+
+    getStorageValueMissingMerkleTrieCounter =
+        metricsSystem.createCounter(
+            BesuMetricCategory.BLOCKCHAIN,
+            "get_storagevalue_missing_merkle_trie",
+            "Number of storage slots not found (either in the flat database or in the merkle trie)");
   }
 
   @Override
@@ -96,6 +163,7 @@ public Optional<Bytes> getCode(final Bytes32 codeHash, final Hash accountHash) {
   }
 
   public Optional<Bytes> getAccount(final Hash accountHash) {
+    getAccountCounter.inc();
     Optional<Bytes> response = accountStorage.get(accountHash.toArrayUnsafe()).map(Bytes::wrap);
     if (response.isEmpty()) {
       // after a snapsync/fastsync we only have the trie branches.
@@ -107,8 +175,13 @@ public Optional<Bytes> getAccount(final Hash accountHash) {
                         this::getAccountStateTrieNode, Function.identity(), Function.identity()),
                     Bytes32.wrap(worldStateRootHash.get()))
                 .get(accountHash);
+        if (response.isEmpty()) getAccountMissingMerkleTrieCounter.inc();
+        else getAccountMerkleTrieCounter.inc();
       }
+    } else {
+      getAccountFlatDatabaseCounter.inc();
     }
+
     return response;
   }
 
@@ -176,6 +249,7 @@ public Optional<Bytes> getStorageValueBySlotHash(
       final Supplier<Optional<Hash>> storageRootSupplier,
       final Hash accountHash,
       final Hash slotHash) {
+    getStorageValueCounter.inc();
     Optional<Bytes> response =
         storageStorage
             .get(Bytes.concatenate(accountHash, slotHash).toArrayUnsafe())
@@ -193,7 +267,11 @@ public Optional<Bytes> getStorageValueBySlotHash(
                     storageRoot.get())
                 .get(slotHash)
                 .map(bytes -> Bytes32.leftPad(RLP.decodeValue(bytes)));
+        if (response.isEmpty()) getStorageValueMissingMerkleTrieCounter.inc();
+        else getStorageValueMerkleTrieCounter.inc();
       }
+    } else {
+      getStorageValueFlatDatabaseCounter.inc();
     }
     return response;
   }

diff --git a/.../main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java b/.../main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java
@@ -16,6 +16,7 @@
 package org.hyperledger.besu.ethereum.bonsai.storage;
 
 import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber;
+import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
 import org.hyperledger.besu.plugin.services.storage.SnappedKeyValueStorage;
 import org.hyperledger.besu.services.kvstore.LayeredKeyValueStorage;
@@ -30,7 +31,8 @@ public BonsaiWorldStateLayerStorage(final BonsaiWorldStateKeyValueStorage parent
         new LayeredKeyValueStorage(parent.storageStorage),
         new LayeredKeyValueStorage(parent.trieBranchStorage),
         parent.trieLogStorage,
-        parent);
+        parent,
+        parent.metricsSystem);
   }
 
   public BonsaiWorldStateLayerStorage(
@@ -39,8 +41,16 @@ public BonsaiWorldStateLayerStorage(
       final SnappedKeyValueStorage storageStorage,
       final SnappedKeyValueStorage trieBranchStorage,
       final KeyValueStorage trieLogStorage,
-      final BonsaiWorldStateKeyValueStorage parent) {
-    super(parent, accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage);
+      final BonsaiWorldStateKeyValueStorage parent,
+      final ObservableMetricsSystem metricsSystem) {
+    super(
+        parent,
+        accountStorage,
+        codeStorage,
+        storageStorage,
+        trieBranchStorage,
+        trieLogStorage,
+        metricsSystem);
   }
 
   @Override
@@ -51,6 +61,7 @@ public BonsaiWorldStateLayerStorage clone() {
         ((LayeredKeyValueStorage) storageStorage).clone(),
         ((LayeredKeyValueStorage) trieBranchStorage).clone(),
         trieLogStorage,
-        parentWorldStateStorage);
+        parentWorldStateStorage,
+        metricsSystem);
   }
 }