Snapshot based non-persisting MutableWorldState usage

CHANGELOG.md

@@ -7,6 +7,7 @@
 - Updated jackson-databind library to version 2.13.4.2 addressing [CVE-2022-42003](https://nvd.nist.gov/vuln/detail/CVE-2022-42003)
 - Gradle task allows custom docker image configs e.g. `./gradlew distDocker -PdockerImageName=my/besu -PdockerVariants=openjdk-17,openjdk-19`
 - Update snapsync feature to avoid restarting the download of the world state from scratch when restarting Besu [#4381](https://github.com/hyperledger/besu/pull/4381)
+- Added worldstate snapshot isolation to improve the stability of bonsai (`--Xbonsai-use-snapshots=true`) [#4351](https://github.com/hyperledger/besu/pull/4531)
 
 ### Bug Fixes
 - Fixed default fromBlock value and improved parameter interpretation in eth_getLogs RPC handler [#4513](https://github.com/hyperledger/besu/pull/4513)

besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java

@@ -17,6 +17,7 @@
 package org.hyperledger.besu.cli.options.stable;
 
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
+import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_USE_SNAPSHOTS;
 
 import org.hyperledger.besu.cli.options.CLIOptions;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
@@ -34,6 +35,8 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
   private static final String BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD =
       "--bonsai-maximum-back-layers-to-load";
 
+  private static final String BONSAI_STORAGE_FORMAT_USE_SNAPSHOTS = "--Xbonsai-use-snapshots";
+
   // Use Bonsai DB
   @Option(
       names = {DATA_STORAGE_FORMAT},
@@ -50,6 +53,15 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
       arity = "1")
   private final Long bonsaiMaxLayersToLoad = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
 
+  @Option(
+      names = {BONSAI_STORAGE_FORMAT_USE_SNAPSHOTS},
+      paramLabel = "<BOOLEAN>",
+      hidden = true,
+      description =
+          "Use database snapshots for mutable worldstates with BONSAI (default: ${DEFAULT-VALUE}).",
+      arity = "1")
+  private final Boolean bonsaiUseSnapshots = DEFAULT_BONSAI_USE_SNAPSHOTS;
+
   public static DataStorageOptions create() {
     return new DataStorageOptions();
   }
@@ -59,6 +71,7 @@ public DataStorageConfiguration toDomainObject() {
     return ImmutableDataStorageConfiguration.builder()
         .dataStorageFormat(dataStorageFormat)
         .bonsaiMaxLayersToLoad(bonsaiMaxLayersToLoad)
+        .useBonsaiSnapshots(bonsaiUseSnapshots)
         .build();
   }
 
@@ -68,6 +81,8 @@ public List<String> getCLIOptions() {
         DATA_STORAGE_FORMAT,
         dataStorageFormat.toString(),
         BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD,
-        bonsaiMaxLayersToLoad.toString());
+        bonsaiMaxLayersToLoad.toString(),
+        BONSAI_STORAGE_FORMAT_USE_SNAPSHOTS,
+        bonsaiUseSnapshots.toString());
   }
 }

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -32,7 +32,6 @@
 import org.hyperledger.besu.ethereum.blockcreation.MiningCoordinator;
 import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateArchive;
 import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateKeyValueStorage;
-import org.hyperledger.besu.ethereum.bonsai.TrieLogManager;
 import org.hyperledger.besu.ethereum.chain.Blockchain;
 import org.hyperledger.besu.ethereum.chain.BlockchainStorage;
 import org.hyperledger.besu.ethereum.chain.DefaultBlockchain;
@@ -632,12 +631,11 @@ private WorldStateArchive createWorldStateArchive(
     switch (dataStorageConfiguration.getDataStorageFormat()) {
       case BONSAI:
         return new BonsaiWorldStateArchive(
-            new TrieLogManager(
-                blockchain,
-                (BonsaiWorldStateKeyValueStorage) worldStateStorage,
-                dataStorageConfiguration.getBonsaiMaxLayersToLoad()),
-            storageProvider,
-            blockchain);
+            (BonsaiWorldStateKeyValueStorage) worldStateStorage,
+            blockchain,
+            Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()),
+            dataStorageConfiguration.useBonsaiSnapshots());
+
       case FOREST:
       default:
         final WorldStatePreimageStorage preimageStorage =

besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java

@@ -131,10 +131,12 @@ public void setup() {
     when(worldStatePreimageStorage.updater())
         .thenReturn(mock(WorldStatePreimageStorage.Updater.class));
     when(worldStateStorage.updater()).thenReturn(mock(WorldStateStorage.Updater.class));
-    BonsaiWorldStateKeyValueStorage.Updater bonsaiUpdater =
-        mock(BonsaiWorldStateKeyValueStorage.Updater.class);
+    BonsaiWorldStateKeyValueStorage.BonsaiUpdater bonsaiUpdater =
+        mock(BonsaiWorldStateKeyValueStorage.BonsaiUpdater.class);
     when(bonsaiUpdater.getTrieLogStorageTransaction())
         .thenReturn(mock(KeyValueStorageTransaction.class));
+    when(bonsaiUpdater.getTrieBranchStorageTransaction())
+        .thenReturn(mock(KeyValueStorageTransaction.class));
     when(bonsaiWorldStateStorage.updater()).thenReturn(bonsaiUpdater);
     besuControllerBuilder = visitWithMockConfigs(new MainnetBesuControllerBuilder());
   }
@@ -167,6 +169,7 @@ public void shouldDisablePruningIfBonsaiIsEnabled() {
             ImmutableDataStorageConfiguration.builder()
                 .dataStorageFormat(DataStorageFormat.BONSAI)
                 .bonsaiMaxLayersToLoad(DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD)
+                .useBonsaiSnapshots(DataStorageConfiguration.DEFAULT_BONSAI_USE_SNAPSHOTS)
                 .build());
     besuControllerBuilder.build();
 
@@ -183,6 +186,7 @@ public void shouldUsePruningIfForestIsEnabled() {
             ImmutableDataStorageConfiguration.builder()
                 .dataStorageFormat(DataStorageFormat.FOREST)
                 .bonsaiMaxLayersToLoad(DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD)
+                .useBonsaiSnapshots(DataStorageConfiguration.DEFAULT_BONSAI_USE_SNAPSHOTS)
                 .build());
     besuControllerBuilder.build();
 

besu/src/test/resources/everything_config.toml

@@ -192,6 +192,7 @@ ethstats-contact="contact@mail.n"
 # Data storage
 data-storage-format="BONSAI"
 bonsai-maximum-back-layers-to-load=512
+Xbonsai-use-snapshots=true
 
 # feature flags
 Xsecp256k1-native-enabled=false

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/GraphQLDataFetchers.java

@@ -46,7 +46,6 @@
 import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason;
 import org.hyperledger.besu.evm.account.Account;
 import org.hyperledger.besu.evm.log.LogTopic;
-import org.hyperledger.besu.evm.worldstate.WorldState;
 import org.hyperledger.besu.plugin.data.SyncStatus;
 
 import java.math.BigInteger;
@@ -219,31 +218,37 @@ DataFetcher<Optional<AccountAdapter>> getAccountDataFetcher() {
       final Address addr = dataFetchingEnvironment.getArgument("address");
       final Long bn = dataFetchingEnvironment.getArgument("blockNumber");
       if (bn != null) {
-        final Optional<WorldState> ws = blockchainQuery.getWorldState(bn);
-        if (ws.isPresent()) {
-          final Account account = ws.get().get(addr);
-          if (account == null) {
-            return Optional.of(new EmptyAccountAdapter(addr));
-          }
-          return Optional.of(new AccountAdapter(account));
-        } else if (bn > blockchainQuery.getBlockchain().getChainHeadBlockNumber()) {
-          // block is past chainhead
-          throw new GraphQLException(GraphQLError.INVALID_PARAMS);
-        } else {
-          // we don't have that block
-          throw new GraphQLException(GraphQLError.CHAIN_HEAD_WORLD_STATE_NOT_AVAILABLE);
-        }
+        return blockchainQuery
+            .mapWorldState(
+                bn,
+                ws -> {
+                  final Account account = ws.get(addr);
+                  if (account == null) {
+                    return new EmptyAccountAdapter(addr);
+                  }
+                  return new AccountAdapter(account);
+                })
+            .or(
+                () -> {
+                  if (bn > blockchainQuery.getBlockchain().getChainHeadBlockNumber()) {
+                    // block is past chainhead
+                    throw new GraphQLException(GraphQLError.INVALID_PARAMS);
+                  } else {
+                    // we don't have that block
+                    throw new GraphQLException(GraphQLError.CHAIN_HEAD_WORLD_STATE_NOT_AVAILABLE);
+                  }
+                });
       } else {
         // return account on latest block
         final long latestBn = blockchainQuery.latestBlock().get().getHeader().getNumber();
-        final Optional<WorldState> ows = blockchainQuery.getWorldState(latestBn);
-        return ows.flatMap(
+        return blockchainQuery.mapWorldState(
+            latestBn,
             ws -> {
               final Account account = ws.get(addr);
               if (account == null) {
-                return Optional.of(new EmptyAccountAdapter(addr));
+                return new EmptyAccountAdapter(addr);
               }
-              return Optional.of(new AccountAdapter(account));
+              return new AccountAdapter(account);
             });
       }
     };

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/BlockAdapterBase.java

@@ -96,8 +96,7 @@ public Optional<AdapterBase> getMiner(final DataFetchingEnvironment environment)
     }
 
     return query
-        .getWorldState(blockNumber)
-        .map(ws -> ws.get(header.getCoinbase()))
+        .mapWorldState(blockNumber, ws -> ws.get(header.getCoinbase()))
         .map(account -> (AdapterBase) new AccountAdapter(account))
         .or(() -> Optional.of(new EmptyAccountAdapter(header.getCoinbase())));
   }
@@ -147,13 +146,12 @@ public Optional<AccountAdapter> getAccount(final DataFetchingEnvironment environ
 
     final BlockchainQueries query = getBlockchainQueries(environment);
     final long bn = header.getNumber();
-    return query
-        .getWorldState(bn)
-        .map(
-            ws -> {
-              final Address address = environment.getArgument("address");
-              return new AccountAdapter(ws.get(address));
-            });
+    return query.mapWorldState(
+        bn,
+        ws -> {
+          final Address address = environment.getArgument("address");
+          return new AccountAdapter(ws.get(address));
+        });
   }
 
   public List<LogAdapter> getLogs(final DataFetchingEnvironment environment) {

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/LogAdapter.java

@@ -63,8 +63,7 @@ public Optional<AccountAdapter> getAccount(final DataFetchingEnvironment environ
       blockNumber = bn;
     }
 
-    return query
-        .getWorldState(blockNumber)
-        .map(ws -> new AccountAdapter(ws.get(logWithMetadata.getLogger())));
+    return query.mapWorldState(
+        blockNumber, ws -> new AccountAdapter(ws.get(logWithMetadata.getLogger())));
   }
 }

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/PendingStateAdapter.java

@@ -24,7 +24,6 @@
 import org.hyperledger.besu.ethereum.transaction.CallParameter;
 import org.hyperledger.besu.ethereum.transaction.TransactionSimulator;
 import org.hyperledger.besu.ethereum.transaction.TransactionSimulatorResult;
-import org.hyperledger.besu.evm.worldstate.WorldState;
 
 import java.util.List;
 import java.util.Map;
@@ -65,10 +64,8 @@ public Optional<AccountAdapter> getAccount(
     final Address addr = dataFetchingEnvironment.getArgument("address");
     final Long blockNumber = dataFetchingEnvironment.getArgument("blockNumber");
     final long latestBlockNumber = blockchainQuery.latestBlock().get().getHeader().getNumber();
-    final Optional<WorldState> optionalWorldState =
-        blockchainQuery.getWorldState(latestBlockNumber);
-    return optionalWorldState
-        .flatMap(worldState -> Optional.ofNullable(worldState.get(addr)))
+    return blockchainQuery
+        .mapWorldState(latestBlockNumber, ws -> ws.get(addr))
         .map(AccountAdapter::new);
   }
 

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/TransactionAdapter.java

@@ -25,7 +25,6 @@
 import org.hyperledger.besu.ethereum.core.LogWithMetadata;
 import org.hyperledger.besu.ethereum.core.Transaction;
 import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput;
-import org.hyperledger.besu.evm.worldstate.WorldState;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -83,12 +82,11 @@ public Optional<AccountAdapter> getFrom(final DataFetchingEnvironment environmen
     if (blockNumber == null) {
       blockNumber = transactionWithMetadata.getBlockNumber().orElseGet(query::headBlockNumber);
     }
-    return query
-        .getWorldState(blockNumber)
-        .map(
-            mutableWorldState ->
-                new AccountAdapter(
-                    mutableWorldState.get(transactionWithMetadata.getTransaction().getSender())));
+    return query.mapWorldState(
+        blockNumber,
+        mutableWorldState ->
+            new AccountAdapter(
+                mutableWorldState.get(transactionWithMetadata.getTransaction().getSender())));
   }
 
   public Optional<AccountAdapter> getTo(final DataFetchingEnvironment environment) {
@@ -98,15 +96,15 @@ public Optional<AccountAdapter> getTo(final DataFetchingEnvironment environment)
       blockNumber = transactionWithMetadata.getBlockNumber().orElseGet(query::headBlockNumber);
     }
 
-    return query
-        .getWorldState(blockNumber)
-        .flatMap(
-            ws -> {
-              return transactionWithMetadata
-                  .getTransaction()
-                  .getTo()
-                  .map(address -> new AccountAdapter(address, ws.get(address)));
-            });
+    return query.mapWorldState(
+        blockNumber,
+        ws ->
+            transactionWithMetadata
+                .getTransaction()
+                .getTo()
+                .map(address -> new AccountAdapter(address, ws.get(address)))
+                // safe because mapWorldState returns Optional.ofNullable
+                .orElse(null));
   }
 
   public Optional<Wei> getValue() {
@@ -176,11 +174,7 @@ public Optional<AccountAdapter> getCreatedContract(final DataFetchingEnvironment
           return Optional.empty();
         }
         final long blockNumber = bn.orElseGet(txBlockNumber::get);
-
-        final Optional<WorldState> ws = query.getWorldState(blockNumber);
-        if (ws.isPresent()) {
-          return Optional.of(new AccountAdapter(ws.get().get(addr.get())));
-        }
+        return query.mapWorldState(blockNumber, ws -> new AccountAdapter(ws.get(addr.get())));
       }
     }
     return Optional.empty();

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugAccountRange.java

@@ -24,7 +24,6 @@
 import org.hyperledger.besu.ethereum.api.query.BlockWithMetadata;
 import org.hyperledger.besu.ethereum.api.query.BlockchainQueries;
 import org.hyperledger.besu.ethereum.core.BlockHeader;
-import org.hyperledger.besu.evm.worldstate.WorldState;
 import org.hyperledger.besu.evm.worldstate.WorldState.StreamableAccount;
 
 import java.util.Collections;
@@ -74,34 +73,32 @@ public JsonRpcResponse response(final JsonRpcRequestContext requestContext) {
     }
 
     // TODO deal with mid-block locations
-
-    final Optional<WorldState> state =
-        blockchainQueries.get().getWorldState(blockHeaderOptional.get().getNumber());
-
-    if (state.isEmpty()) {
-      return emptyResponse(requestContext);
-    } else {
-      final List<StreamableAccount> accounts =
-          state
-              .get()
-              .streamAccounts(Bytes32.fromHexStringLenient(addressHash), maxResults + 1)
-              .collect(Collectors.toList());
-      Bytes32 nextKey = Bytes32.ZERO;
-      if (accounts.size() == maxResults + 1) {
-        nextKey = accounts.get(maxResults).getAddressHash();
-        accounts.remove(maxResults);
-      }
-
-      return new JsonRpcSuccessResponse(
-          requestContext.getRequest().getId(),
-          new DebugAccountRangeAtResult(
-              accounts.stream()
-                  .collect(
-                      Collectors.toMap(
-                          account -> account.getAddressHash().toString(),
-                          account -> account.getAddress().orElse(Address.ZERO).toString())),
-              nextKey.toString()));
-    }
+    return blockchainQueries
+        .get()
+        .mapWorldState(
+            blockHeaderOptional.get().getNumber(),
+            state -> {
+              final List<StreamableAccount> accounts =
+                  state
+                      .streamAccounts(Bytes32.fromHexStringLenient(addressHash), maxResults + 1)
+                      .collect(Collectors.toList());
+              Bytes32 nextKey = Bytes32.ZERO;
+              if (accounts.size() == maxResults + 1) {
+                nextKey = accounts.get(maxResults).getAddressHash();
+                accounts.remove(maxResults);
+              }
+
+              return new JsonRpcSuccessResponse(
+                  requestContext.getRequest().getId(),
+                  new DebugAccountRangeAtResult(
+                      accounts.stream()
+                          .collect(
+                              Collectors.toMap(
+                                  account -> account.getAddressHash().toString(),
+                                  account -> account.getAddress().orElse(Address.ZERO).toString())),
+                      nextKey.toString()));
+            })
+        .orElse(emptyResponse(requestContext));
   }
 
   private Optional<Hash> hashFromParameter(final BlockParameterOrBlockHash blockParameter) {

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStorageRangeAt.java

@@ -104,8 +104,8 @@ public JsonRpcResponse response(final JsonRpcRequestContext requestContext) {
             () ->
                 blockchainQueries
                     .get()
-                    .getWorldState(blockHeaderOptional.get().getNumber())
-                    .map(
+                    .mapWorldState(
+                        blockHeaderOptional.get().getNumber(),
                         worldState ->
                             extractStorageAt(
                                 requestContext, accountAddress, startKey, limit, worldState))