Add range tracing with worldstate (hyperledger#5844)

Implement a method to trace a range of blocks and have access to the worldstate before and after the tracing

Signed-off-by: Karim TAAM <karim.t2am@gmail.com>

besu/src/main/java/org/hyperledger/besu/services/TraceServiceImpl.java

@@ -31,13 +31,16 @@
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec;
 import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult;
 import org.hyperledger.besu.ethereum.vm.CachingBlockHashLookup;
+import org.hyperledger.besu.evm.worldstate.WorldUpdater;
 import org.hyperledger.besu.plugin.Unstable;
 import org.hyperledger.besu.plugin.services.TraceService;
 import org.hyperledger.besu.plugin.services.tracer.BlockAwareOperationTracer;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
+import java.util.function.Consumer;
+import java.util.stream.LongStream;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -88,61 +91,110 @@ public void traceBlock(final Hash hash, final BlockAwareOperationTracer tracer)
     block.ifPresent(value -> trace(value, tracer));
   }
 
-  private void trace(final Block block, final BlockAwareOperationTracer tracer) {
-    LOG.debug("Tracing block {}", block.toLogString());
-    final List<TransactionProcessingResult> results = new ArrayList<>();
+  /**
+   * Traces range of blocks
+   *
+   * @param fromBlockNumber the beginning of the range (inclusive)
+   * @param toBlockNumber the end of the range (inclusive)
+   * @param beforeTracing Function which performs an operation on a MutableWorldState before tracing
+   * @param afterTracing Function which performs an operation on a MutableWorldState after tracing
+   * @param tracer an instance of OperationTracer
+   */
+  @Override
+  public void trace(
+      final long fromBlockNumber,
+      final long toBlockNumber,
+      final Consumer<WorldUpdater> beforeTracing,
+      final Consumer<WorldUpdater> afterTracing,
+      final BlockAwareOperationTracer tracer) {
+    checkArgument(tracer != null);
+    LOG.debug("Tracing from block {} to block {}", fromBlockNumber, toBlockNumber);
+    final Blockchain blockchain = blockchainQueries.getBlockchain();
+    final List<Block> blocks =
+        LongStream.rangeClosed(fromBlockNumber, toBlockNumber)
+            .mapToObj(
+                number ->
+                    blockchain
+                        .getBlockByNumber(number)
+                        .orElseThrow(() -> new RuntimeException("Block not found " + number)))
+            .toList();
     Tracer.processTracing(
         blockchainQueries,
-        block.getHash(),
+        blocks.get(0).getHash(),
         traceableState -> {
-          final Blockchain blockchain = blockchainQueries.getBlockchain();
-          final ChainUpdater chainUpdater = new ChainUpdater(traceableState);
-          final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(block.getHeader());
-          final MainnetTransactionProcessor transactionProcessor =
-              protocolSpec.getTransactionProcessor();
-          final BlockHeader header = block.getHeader();
-
-          tracer.traceStartBlock(block.getHeader(), block.getBody());
-
-          block
-              .getBody()
-              .getTransactions()
-              .forEach(
-                  transaction -> {
-                    final Optional<BlockHeader> maybeParentHeader =
-                        blockchain.getBlockHeader(header.getParentHash());
-                    final Wei blobGasPrice =
-                        protocolSpec
-                            .getFeeMarket()
-                            .blobGasPricePerGas(
-                                maybeParentHeader
-                                    .map(
-                                        parent ->
-                                            calculateExcessBlobGasForParent(protocolSpec, parent))
-                                    .orElse(BlobGas.ZERO));
-
-                    tracer.traceStartTransaction(transaction);
-
-                    final TransactionProcessingResult result =
-                        transactionProcessor.processTransaction(
-                            blockchain,
-                            chainUpdater.getNextUpdater(),
-                            header,
-                            transaction,
-                            header.getCoinbase(),
-                            tracer,
-                            new CachingBlockHashLookup(header, blockchain),
-                            false,
-                            blobGasPrice);
-
-                    long transactionGasUsed = transaction.getGasLimit() - result.getGasRemaining();
-                    tracer.traceEndTransaction(result.getOutput(), transactionGasUsed, 0);
-
-                    results.add(result);
-                  });
+          final WorldUpdater worldStateUpdater = traceableState.updater();
+          final ChainUpdater chainUpdater = new ChainUpdater(traceableState, worldStateUpdater);
+          beforeTracing.accept(worldStateUpdater);
+          final List<TransactionProcessingResult> results = new ArrayList<>();
+          blocks.forEach(
+              block -> {
+                results.addAll(trace(blockchain, block, chainUpdater, tracer));
+                tracer.traceEndBlock(block.getHeader(), block.getBody());
+              });
+          afterTracing.accept(chainUpdater.getNextUpdater());
           return Optional.of(results);
         });
+  }
 
+  private void trace(final Block block, final BlockAwareOperationTracer tracer) {
+    LOG.debug("Tracing block {}", block.toLogString());
+    final Blockchain blockchain = blockchainQueries.getBlockchain();
+    Tracer.processTracing(
+        blockchainQueries,
+        block.getHash(),
+        traceableState ->
+            Optional.of(trace(blockchain, block, new ChainUpdater(traceableState), tracer)));
     tracer.traceEndBlock(block.getHeader(), block.getBody());
   }
+
+  private List<TransactionProcessingResult> trace(
+      final Blockchain blockchain,
+      final Block block,
+      final ChainUpdater chainUpdater,
+      final BlockAwareOperationTracer tracer) {
+    final List<TransactionProcessingResult> results = new ArrayList<>();
+    final ProtocolSpec protocolSpec = protocolSchedule.getByBlockHeader(block.getHeader());
+    final MainnetTransactionProcessor transactionProcessor = protocolSpec.getTransactionProcessor();
+    final BlockHeader header = block.getHeader();
+    tracer.traceStartBlock(block.getHeader(), block.getBody());
+
+    block
+        .getBody()
+        .getTransactions()
+        .forEach(
+            transaction -> {
+              final Optional<BlockHeader> maybeParentHeader =
+                  blockchain.getBlockHeader(header.getParentHash());
+              final Wei blobGasPrice =
+                  protocolSpec
+                      .getFeeMarket()
+                      .blobGasPricePerGas(
+                          maybeParentHeader
+                              .map(parent -> calculateExcessBlobGasForParent(protocolSpec, parent))
+                              .orElse(BlobGas.ZERO));
+
+              tracer.traceStartTransaction(transaction);
+
+              final TransactionProcessingResult result =
+                  transactionProcessor.processTransaction(
+                      blockchain,
+                      chainUpdater.getNextUpdater(),
+                      header,
+                      transaction,
+                      header.getCoinbase(),
+                      tracer,
+                      new CachingBlockHashLookup(header, blockchain),
+                      false,
+                      blobGasPrice);
+
+              long transactionGasUsed = transaction.getGasLimit() - result.getGasRemaining();
+              tracer.traceEndTransaction(result.getOutput(), transactionGasUsed, 0);
+
+              results.add(result);
+            });
+
+    tracer.traceEndBlock(block.getHeader(), block.getBody());
+
+    return results;
+  }
 }

besu/src/test/java/org/hyperledger/besu/services/TraceServiceImplTest.java

@@ -0,0 +1,105 @@
+/*
+ * Copyright Hyperledger Besu Contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.services;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.hyperledger.besu.datatypes.Address;
+import org.hyperledger.besu.ethereum.api.query.BlockchainQueries;
+import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
+import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil;
+import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
+import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
+import org.hyperledger.besu.plugin.services.TraceService;
+import org.hyperledger.besu.plugin.services.tracer.BlockAwareOperationTracer;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+class TraceServiceImplTest {
+
+  TraceService traceService;
+  private MutableBlockchain blockchain;
+  private WorldStateArchive worldStateArchive;
+  private BlockchainQueries blockchainQueries;
+
+  /**
+   * The blockchain for testing has a height of 32 and the account
+   * 0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b makes a transaction per block. So, in the head, the
+   * nonce of this account should be 32.
+   */
+  @BeforeEach
+  public void setup() {
+    final BlockchainSetupUtil blockchainSetupUtil =
+        BlockchainSetupUtil.forTesting(DataStorageFormat.BONSAI);
+    blockchainSetupUtil.importAllBlocks();
+    blockchain = blockchainSetupUtil.getBlockchain();
+    worldStateArchive = blockchainSetupUtil.getWorldArchive();
+    blockchainQueries = new BlockchainQueries(blockchain, worldStateArchive);
+    traceService =
+        new TraceServiceImpl(blockchainQueries, blockchainSetupUtil.getProtocolSchedule());
+  }
+
+  @Test
+  void shouldRetrieveStateUpdatePostTracingForOneBlock() {
+
+    final Address addressToVerify =
+        Address.fromHexString("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b");
+
+    final long persistedNonceForAccount =
+        worldStateArchive.getMutable().get(addressToVerify).getNonce();
+
+    traceService.trace(
+        2,
+        2,
+        worldState -> {
+          assertThat(worldState.get(addressToVerify).getNonce()).isEqualTo(1);
+        },
+        worldState -> {
+          assertThat(worldState.get(addressToVerify).getNonce()).isEqualTo(2);
+        },
+        BlockAwareOperationTracer.NO_TRACING);
+
+    assertThat(worldStateArchive.getMutable().get(addressToVerify).getNonce())
+        .isEqualTo(persistedNonceForAccount);
+  }
+
+  @Test
+  void shouldRetrieveStateUpdatePostTracingForAllBlocks() {
+    final Address addressToVerify =
+        Address.fromHexString("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b");
+
+    final long persistedNonceForAccount =
+        worldStateArchive.getMutable().get(addressToVerify).getNonce();
+
+    traceService.trace(
+        0,
+        32,
+        worldState -> {
+          assertThat(worldState.get(addressToVerify).getNonce()).isEqualTo(0);
+        },
+        worldState -> {
+          assertThat(worldState.get(addressToVerify).getNonce())
+              .isEqualTo(persistedNonceForAccount);
+        },
+        BlockAwareOperationTracer.NO_TRACING);
+
+    assertThat(worldStateArchive.getMutable().get(addressToVerify).getNonce())
+        .isEqualTo(persistedNonceForAccount);
+  }
+}

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/TraceBlock.java

@@ -199,6 +199,11 @@ public ChainUpdater(final MutableWorldState worldState) {
       this.worldState = worldState;
     }
 
+    public ChainUpdater(final MutableWorldState worldState, final WorldUpdater updater) {
+      this.worldState = worldState;
+      this.updater = updater;
+    }
+
     public WorldUpdater getNextUpdater() {
       // if we have no prior updater, it must be the first TX, so use the block's initial state
       if (updater == null) {

plugin-api/build.gradle

@@ -69,7 +69,7 @@ Calculated : ${currentHash}
 tasks.register('checkAPIChanges', FileStateChecker) {
   description = "Checks that the API for the Plugin-API project does not change without deliberate thought"
   files = sourceSets.main.allJava.files
-  knownHash = 'eRQ5aFercEH5v+RKwyCDaQ9jSX2l0aujF6mphpSCm+Q='
+  knownHash = 'fPOd/MnNB1PwfTV7HDTXc3oYRcoeUzMJrlzDUdg/HNk='
 }
 check.dependsOn('checkAPIChanges')
 

plugin-api/src/main/java/org/hyperledger/besu/plugin/services/TraceService.java

@@ -15,9 +15,12 @@
 package org.hyperledger.besu.plugin.services;
 
 import org.hyperledger.besu.datatypes.Hash;
+import org.hyperledger.besu.evm.worldstate.WorldUpdater;
 import org.hyperledger.besu.plugin.Unstable;
 import org.hyperledger.besu.plugin.services.tracer.BlockAwareOperationTracer;
 
+import java.util.function.Consumer;
+
 /** The Trace service interface */
 @Unstable
 public interface TraceService extends BesuService {
@@ -36,4 +39,20 @@ public interface TraceService extends BesuService {
    * @param tracer the tracer (OperationTracer)
    */
   void traceBlock(Hash hash, BlockAwareOperationTracer tracer);
+
+  /**
+   * Traces range of blocks
+   *
+   * @param fromBlockNumber the beginning of the range (inclusive)
+   * @param toBlockNumber the end of the range (inclusive)
+   * @param beforeTracing Function which performs an operation on a MutableWorldState before tracing
+   * @param afterTracing Function which performs an operation on a MutableWorldState after tracing
+   * @param tracer an instance of OperationTracer
+   */
+  void trace(
+      final long fromBlockNumber,
+      final long toBlockNumber,
+      final Consumer<WorldUpdater> beforeTracing,
+      final Consumer<WorldUpdater> afterTracing,
+      final BlockAwareOperationTracer tracer);
 }

plugin-api/src/main/java/org/hyperledger/besu/plugin/services/tracer/BlockAwareOperationTracer.java

@@ -24,6 +24,13 @@
  * <p>In both methods, the block header and body are provided.
  */
 public interface BlockAwareOperationTracer extends OperationTracer {
+
+  /**
+   * BlockAwareOperationTracer object with no tracing functionality. This serves as a default for
+   * scenarios where no specific tracing operation is required.
+   */
+  BlockAwareOperationTracer NO_TRACING = new BlockAwareOperationTracer() {};
+
   /**
    * Trace the start of a block.
    *