Default bonsai to fully flat db and code storage by codehash (#6894)

* change to full flat db and code stored by code hash by default
* deprecate --Xsnapsync-synchronizer-flat-db-healing-enabled, add DataStorageOption for --Xbonsai-full-flat-db-enabled

Signed-off-by: garyschulte <garyschulte@gmail.com>

CHANGELOG.md

@@ -45,6 +45,7 @@
 - Allow users to specify which plugins are registered [#6700](https://github.com/hyperledger/besu/pull/6700)
 - Layered txpool tuning for blob transactions [#6940](https://github.com/hyperledger/besu/pull/6940)
 - Update Gradle to 7.6.4 [#7030](https://github.com/hyperledger/besu/pull/7030)
+- Default bonsai to use full-flat db and code-storage-by-code-hash [#6984](https://github.com/hyperledger/besu/pull/6894)
 
 ### Bug fixes
 - Fix txpool dump/restore race condition [#6665](https://github.com/hyperledger/besu/pull/6665)

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -1706,8 +1706,8 @@ && isOptionSet(commandLine, "--sync-min-peers")) {
 
     CommandLineUtils.failIfOptionDoesntMeetRequirement(
         commandLine,
-        "--Xsnapsync-synchronizer-flat option can only be used when -Xsnapsync-synchronizer-flat-db-healing-enabled is true",
-        unstableSynchronizerOptions.isSnapsyncFlatDbHealingEnabled(),
+        "--Xsnapsync-synchronizer-flat option can only be used when --Xbonsai-full-flat-db-enabled is true",
+        dataStorageOptions.toDomainObject().getUnstable().getBonsaiFullFlatDbEnabled(),
         asList(
             "--Xsnapsync-synchronizer-flat-account-healed-count-per-request",
             "--Xsnapsync-synchronizer-flat-slot-healed-count-per-request"));

besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java

@@ -17,6 +17,7 @@
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_RECEIPT_COMPACTION_ENABLED;
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED;
+import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED;
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED;
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT;
@@ -93,6 +94,18 @@ public static class Unstable {
             "The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})")
     private int bonsaiTrieLogPruningWindowSize = DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
 
+    // TODO: --Xsnapsync-synchronizer-flat-db-healing-enabled is deprecated, remove it in a future
+    // release
+    @CommandLine.Option(
+        hidden = true,
+        names = {
+          "--Xbonsai-full-flat-db-enabled",
+          "--Xsnapsync-synchronizer-flat-db-healing-enabled"
+        },
+        arity = "1",
+        description = "Enables bonsai full flat database strategy. (default: ${DEFAULT-VALUE})")
+    private Boolean bonsaiFullFlatDbEnabled = DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED;
+
     @CommandLine.Option(
         hidden = true,
         names = {"--Xbonsai-code-using-code-hash-enabled"},
@@ -161,6 +174,8 @@ public static DataStorageOptions fromConfig(final DataStorageConfiguration domai
         domainObject.getUnstable().getBonsaiLimitTrieLogsEnabled();
     dataStorageOptions.unstableOptions.bonsaiTrieLogPruningWindowSize =
         domainObject.getUnstable().getBonsaiTrieLogPruningWindowSize();
+    dataStorageOptions.unstableOptions.bonsaiFullFlatDbEnabled =
+        domainObject.getUnstable().getBonsaiFullFlatDbEnabled();
     dataStorageOptions.unstableOptions.bonsaiCodeUsingCodeHashEnabled =
         domainObject.getUnstable().getBonsaiCodeStoredByCodeHashEnabled();
 
@@ -177,6 +192,7 @@ public DataStorageConfiguration toDomainObject() {
             ImmutableDataStorageConfiguration.Unstable.builder()
                 .bonsaiLimitTrieLogsEnabled(unstableOptions.bonsaiLimitTrieLogsEnabled)
                 .bonsaiTrieLogPruningWindowSize(unstableOptions.bonsaiTrieLogPruningWindowSize)
+                .bonsaiFullFlatDbEnabled(unstableOptions.bonsaiFullFlatDbEnabled)
                 .bonsaiCodeStoredByCodeHashEnabled(unstableOptions.bonsaiCodeUsingCodeHashEnabled)
                 .build())
         .build();

besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java

@@ -81,9 +81,6 @@ public class SynchronizerOptions implements CLIOptions<SynchronizerConfiguration
   private static final String SNAP_FLAT_STORAGE_HEALED_COUNT_PER_REQUEST_FLAG =
       "--Xsnapsync-synchronizer-flat-slot-healed-count-per-request";
 
-  private static final String SNAP_FLAT_DB_HEALING_ENABLED_FLAG =
-      "--Xsnapsync-synchronizer-flat-db-healing-enabled";
-
   private static final String SNAP_SERVER_ENABLED_FLAG = "--Xsnapsync-server-enabled";
 
   private static final String CHECKPOINT_POST_MERGE_FLAG = "--Xcheckpoint-post-merge-enabled";
@@ -292,18 +289,11 @@ public void parseBlockPropagationRange(final String arg) {
   private int snapsyncFlatStorageHealedCountPerRequest =
       SnapSyncConfiguration.DEFAULT_LOCAL_FLAT_STORAGE_COUNT_TO_HEAL_PER_REQUEST;
 
-  @CommandLine.Option(
-      names = SNAP_FLAT_DB_HEALING_ENABLED_FLAG,
-      hidden = true,
-      paramLabel = "<Boolean>",
-      description = "Snap sync flat db healing enabled (default: ${DEFAULT-VALUE})")
-  private Boolean snapsyncFlatDbHealingEnabled =
-      SnapSyncConfiguration.DEFAULT_IS_FLAT_DB_HEALING_ENABLED;
-
   @CommandLine.Option(
       names = SNAP_SERVER_ENABLED_FLAG,
       hidden = true,
       paramLabel = "<Boolean>",
+      arity = "0..1",
       description = "Snap sync server enabled (default: ${DEFAULT-VALUE})")
   private Boolean snapsyncServerEnabled = SnapSyncConfiguration.DEFAULT_SNAP_SERVER_ENABLED;
 
@@ -316,15 +306,6 @@ public void parseBlockPropagationRange(final String arg) {
 
   private SynchronizerOptions() {}
 
-  /**
-   * Flag to know whether the flat db healing feature is enabled or disabled.
-   *
-   * @return true is the flat db healing is enabled
-   */
-  public boolean isSnapsyncFlatDbHealingEnabled() {
-    return snapsyncFlatDbHealingEnabled;
-  }
-
   /**
    * Flag to know whether the Snap sync server feature is enabled or disabled.
    *
@@ -382,9 +363,8 @@ public static SynchronizerOptions fromConfig(final SynchronizerConfiguration con
         config.getSnapSyncConfiguration().getLocalFlatAccountCountToHealPerRequest();
     options.snapsyncFlatStorageHealedCountPerRequest =
         config.getSnapSyncConfiguration().getLocalFlatStorageCountToHealPerRequest();
-    options.snapsyncFlatDbHealingEnabled =
-        config.getSnapSyncConfiguration().isFlatDbHealingEnabled();
     options.checkpointPostMergeSyncEnabled = config.isCheckpointPostMergeEnabled();
+    options.snapsyncServerEnabled = config.getSnapSyncConfiguration().isSnapServerEnabled();
     return options;
   }
 
@@ -416,7 +396,6 @@ public SynchronizerConfiguration.Builder toDomainObject() {
             .trienodeCountPerRequest(snapsyncTrieNodeCountPerRequest)
             .localFlatAccountCountToHealPerRequest(snapsyncFlatAccountHealedCountPerRequest)
             .localFlatStorageCountToHealPerRequest(snapsyncFlatStorageHealedCountPerRequest)
-            .isFlatDbHealingEnabled(snapsyncFlatDbHealingEnabled)
             .isSnapServerEnabled(snapsyncServerEnabled)
             .build());
     builder.checkpointPostMergeEnabled(checkpointPostMergeSyncEnabled);
@@ -469,17 +448,13 @@ public List<String> getCLIOptions() {
             SNAP_BYTECODE_COUNT_PER_REQUEST_FLAG,
             OptionParser.format(snapsyncBytecodeCountPerRequest),
             SNAP_TRIENODE_COUNT_PER_REQUEST_FLAG,
-            OptionParser.format(snapsyncTrieNodeCountPerRequest));
-    if (isSnapsyncFlatDbHealingEnabled()) {
-      value.addAll(
-          Arrays.asList(
-              SNAP_FLAT_ACCOUNT_HEALED_COUNT_PER_REQUEST_FLAG,
-              OptionParser.format(snapsyncFlatAccountHealedCountPerRequest),
-              SNAP_FLAT_STORAGE_HEALED_COUNT_PER_REQUEST_FLAG,
-              OptionParser.format(snapsyncFlatStorageHealedCountPerRequest),
-              SNAP_SERVER_ENABLED_FLAG,
-              OptionParser.format(snapsyncServerEnabled)));
-    }
+            OptionParser.format(snapsyncTrieNodeCountPerRequest),
+            SNAP_FLAT_ACCOUNT_HEALED_COUNT_PER_REQUEST_FLAG,
+            OptionParser.format(snapsyncFlatAccountHealedCountPerRequest),
+            SNAP_FLAT_STORAGE_HEALED_COUNT_PER_REQUEST_FLAG,
+            OptionParser.format(snapsyncFlatStorageHealedCountPerRequest),
+            SNAP_SERVER_ENABLED_FLAG,
+            OptionParser.format(snapsyncServerEnabled));
     return value;
   }
 }

besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java

@@ -2338,29 +2338,48 @@ public void kzgTrustedSetupFileLoadedWithCustomGenesisFile()
   }
 
   @Test
-  public void snapsyncHealingOptionShouldBeDisabledByDefault() {
+  public void bonsaiFlatDbShouldBeEnabledByDefault() {
     final TestBesuCommand besuCommand = parseCommand();
-    assertThat(besuCommand.unstableSynchronizerOptions.isSnapsyncFlatDbHealingEnabled()).isFalse();
+    assertThat(
+            besuCommand
+                .getDataStorageOptions()
+                .toDomainObject()
+                .getUnstable()
+                .getBonsaiFullFlatDbEnabled())
+        .isTrue();
   }
 
   @Test
   public void snapsyncHealingOptionShouldWork() {
-    final TestBesuCommand besuCommand =
-        parseCommand("--Xsnapsync-synchronizer-flat-db-healing-enabled", "true");
-    assertThat(besuCommand.unstableSynchronizerOptions.isSnapsyncFlatDbHealingEnabled()).isTrue();
+    final TestBesuCommand besuCommand = parseCommand("--Xbonsai-full-flat-db-enabled", "false");
+    assertThat(
+            besuCommand
+                .dataStorageOptions
+                .toDomainObject()
+                .getUnstable()
+                .getBonsaiFullFlatDbEnabled())
+        .isFalse();
   }
 
   @Test
   public void snapsyncForHealingFeaturesShouldFailWhenHealingIsNotEnabled() {
-    parseCommand("--Xsnapsync-synchronizer-flat-account-healed-count-per-request", "100");
+    parseCommand(
+        "--Xbonsai-full-flat-db-enabled",
+        "false",
+        "--Xsnapsync-synchronizer-flat-account-healed-count-per-request",
+        "100");
     assertThat(commandErrorOutput.toString(UTF_8))
         .contains(
-            "--Xsnapsync-synchronizer-flat option can only be used when -Xsnapsync-synchronizer-flat-db-healing-enabled is true");
+            "--Xsnapsync-synchronizer-flat option can only be used when --Xbonsai-full-flat-db-enabled is true");
 
-    parseCommand("--Xsnapsync-synchronizer-flat-slot-healed-count-per-request", "100");
+    parseCommand(
+        "--Xbonsai-full-flat-db-enabled",
+        "false",
+        "--Xsnapsync-synchronizer-flat-slot-healed-count-per-request",
+        "100");
     assertThat(commandErrorOutput.toString(UTF_8))
         .contains(
-            "--Xsnapsync-synchronizer-flat option can only be used when -Xsnapsync-synchronizer-flat-db-healing-enabled is true");
+            "--Xsnapsync-synchronizer-flat option can only be used when --Xbonsai-full-flat-db-enabled is true");
   }
 
   @Test

besu/src/test/java/org/hyperledger/besu/cli/options/SynchronizerOptionsTest.java

@@ -78,6 +78,7 @@ protected SynchronizerConfiguration.Builder createCustomizedDomainObject() {
                 .storageCountPerRequest(SnapSyncConfiguration.DEFAULT_STORAGE_COUNT_PER_REQUEST + 2)
                 .bytecodeCountPerRequest(
                     SnapSyncConfiguration.DEFAULT_BYTECODE_COUNT_PER_REQUEST + 2)
+                .isSnapServerEnabled(Boolean.TRUE)
                 .build());
   }
 

besu/src/test/resources/everything_config.toml

@@ -218,6 +218,8 @@ receipt-compaction-enabled=true
 # feature flags
 Xsecp256k1-native-enabled=false
 Xaltbn128-native-enabled=false
+Xsnapsync-server-enabled=true
+Xbonsai-full-flat-db-enabled=true
 
 # compatibility flags
 compatibility-eth64-forkid-enabled=false

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java

@@ -16,6 +16,7 @@
 
 import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE;
 import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE;
+import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY;
 
 import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.FullFlatDbStrategy;
 import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.PartialFlatDbStrategy;
@@ -70,12 +71,36 @@ public void loadFlatDbStrategy(final SegmentedKeyValueStorage composedWorldState
 
   @VisibleForTesting
   FlatDbMode deriveFlatDbStrategy(final SegmentedKeyValueStorage composedWorldStateStorage) {
+    final FlatDbMode requestedFlatDbMode =
+        dataStorageConfiguration.getUnstable().getBonsaiFullFlatDbEnabled()
+            ? FlatDbMode.FULL
+            : FlatDbMode.PARTIAL;
+
+    final var existingTrieData =
+        composedWorldStateStorage.get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY).isPresent();
+
     var flatDbMode =
         FlatDbMode.fromVersion(
             composedWorldStateStorage
                 .get(TRIE_BRANCH_STORAGE, FLAT_DB_MODE)
                 .map(Bytes::wrap)
-                .orElse(FlatDbMode.PARTIAL.getVersion()));
+                .orElseGet(
+                    () -> {
+                      // if we do not have a db-supplied config for flatdb, derive it:
+                      // default to partial if trie data exists, but the flat config does not,
+                      // and default to the storage config otherwise
+                      var flatDbModeVal =
+                          existingTrieData
+                              ? FlatDbMode.PARTIAL.getVersion()
+                              : requestedFlatDbMode.getVersion();
+                      // persist this config in the db
+                      var setDbModeTx = composedWorldStateStorage.startTransaction();
+                      setDbModeTx.put(
+                          TRIE_BRANCH_STORAGE, FLAT_DB_MODE, flatDbModeVal.toArrayUnsafe());
+                      setDbModeTx.commit();
+
+                      return flatDbModeVal;
+                    }));
     LOG.info("Bonsai flat db mode found {}", flatDbMode);
 
     return flatDbMode;
@@ -123,7 +148,7 @@ public FlatDbStrategy getFlatDbStrategy(
   public void upgradeToFullFlatDbMode(final SegmentedKeyValueStorage composedWorldStateStorage) {
     final SegmentedKeyValueStorageTransaction transaction =
         composedWorldStateStorage.startTransaction();
-    // TODO: consider ARCHIVE mode
+    LOG.info("setting FlatDbStrategy to FULL");
     transaction.put(
         TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.FULL.getVersion().toArrayUnsafe());
     transaction.commit();
@@ -134,6 +159,7 @@ public void downgradeToPartialFlatDbMode(
       final SegmentedKeyValueStorage composedWorldStateStorage) {
     final SegmentedKeyValueStorageTransaction transaction =
         composedWorldStateStorage.startTransaction();
+    LOG.info("setting FlatDbStrategy to PARTIAL");
     transaction.put(
         TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.PARTIAL.getVersion().toArrayUnsafe());
     transaction.commit();

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java

@@ -38,6 +38,13 @@ public interface DataStorageConfiguration {
           .bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD)
           .build();
 
+  DataStorageConfiguration DEFAULT_BONSAI_PARTIAL_DB_CONFIG =
+      ImmutableDataStorageConfiguration.builder()
+          .dataStorageFormat(DataStorageFormat.BONSAI)
+          .bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD)
+          .unstable(Unstable.DEFAULT_PARTIAL)
+          .build();
+
   DataStorageConfiguration DEFAULT_FOREST_CONFIG =
       ImmutableDataStorageConfiguration.builder()
           .dataStorageFormat(DataStorageFormat.FOREST)
@@ -65,11 +72,15 @@ interface Unstable {
     boolean DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED = false;
     long MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
     int DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE = 30_000;
-    boolean DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED = false;
+    boolean DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED = true;
+    boolean DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED = true;
 
     DataStorageConfiguration.Unstable DEFAULT =
         ImmutableDataStorageConfiguration.Unstable.builder().build();
 
+    DataStorageConfiguration.Unstable DEFAULT_PARTIAL =
+        ImmutableDataStorageConfiguration.Unstable.builder().bonsaiFullFlatDbEnabled(false).build();
+
     @Value.Default
     default boolean getBonsaiLimitTrieLogsEnabled() {
       return DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED;
@@ -80,6 +91,11 @@ default int getBonsaiTrieLogPruningWindowSize() {
       return DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
     }
 
+    @Value.Default
+    default boolean getBonsaiFullFlatDbEnabled() {
+      return DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED;
+    }
+
     @Value.Default
     default boolean getBonsaiCodeStoredByCodeHashEnabled() {
       return DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED;

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java

@@ -152,8 +152,11 @@ public abstract class AbstractIsolationTests {
   @BeforeEach
   public void createStorage() {
     worldStateKeyValueStorage =
+        // FYI: BonsaiSnapshoIsolationTests  work with frozen/cached worldstates, using PARTIAL
+        // flat db strategy allows the tests to make account assertions based on trie
+        // (whereas a full db strategy will not, since the worldstates are frozen/cached)
         createKeyValueStorageProvider()
-            .createWorldStateStorage(DataStorageConfiguration.DEFAULT_BONSAI_CONFIG);
+            .createWorldStateStorage(DataStorageConfiguration.DEFAULT_BONSAI_PARTIAL_DB_CONFIG);
     archive =
         new BonsaiWorldStateProvider(
             (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage,