From 8b2f649149703f9f8426c66f1a831bf570833b55 Mon Sep 17 00:00:00 2001 From: Simon Dudley Date: Fri, 7 Jun 2024 07:53:56 +1000 Subject: [PATCH] Enable --Xbonsai-limit-trie-logs-enabled by default (#7181) * Enable --Xbonsai-limit-trie-logs-enabled by default Signed-off-by: Simon Dudley Signed-off-by: George Tebrean --- CHANGELOG.md | 2 + .../org/hyperledger/besu/cli/BesuCommand.java | 5 ++- .../options/stable/DataStorageOptions.java | 15 ++++++- .../hyperledger/besu/cli/BesuCommandTest.java | 39 ++++++++++++++++--- .../stable/DataStorageOptionsTest.java | 9 +++++ .../profiles/enterprise-private.toml | 3 +- .../resources/profiles/minimalist-staker.toml | 2 +- .../worldstate/DataStorageConfiguration.java | 2 +- 8 files changed, 65 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b5f6eac341..e75845362c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,13 @@ ### Breaking Changes - Java 21 has been enforced as minimum version to build and run Besu. +- With --Xbonsai-limit-trie-logs-enabled by default in this release, historic trie log data will be removed from the database unless sync-mode=FULL. It respects the --bonsai-historical-block-limit setting so shouldn't break any RPCs, but may be breaking if you are accessing this data from the database directly. Can be disabled with --Xbonsai-limit-trie-logs-enabled=false - In profile=ENTERPRISE, use sync-mode=FULL (instead of FAST) and data-storage-format=FOREST (instead of BONSAI) [#7186](https://github.com/hyperledger/besu/pull/7186) - If this breaks your node, you can reset sync-mode=FAST and data-storage-format=BONSAI ### Additions and Improvements - Add two counters to DefaultBlockchain in order to be able to calculate TPS and Mgas/s [#7105](https://github.com/hyperledger/besu/pull/7105) +- Enable --Xbonsai-limit-trie-logs-enabled by default, unless sync-mode=FULL [#7181](https://github.com/hyperledger/besu/pull/7181) - `admin_nodeInfo` JSON/RPC call returns the currently active EVM version [#7127](https://github.com/hyperledger/besu/pull/7127) ### Bug fixes diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index b2593f78357..08c53666bb1 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -1570,7 +1570,7 @@ private void validateTransactionPoolOptions() { } private void validateDataStorageOptions() { - dataStorageOptions.validate(commandLine); + dataStorageOptions.validate(commandLine, syncMode); } private void validateRequiredOptions() { @@ -2799,7 +2799,8 @@ private String generateConfigurationOverview() { builder.setHighSpecEnabled(); } - if (getDataStorageConfiguration().getUnstable().getBonsaiLimitTrieLogsEnabled()) { + if (DataStorageFormat.BONSAI.equals(getDataStorageConfiguration().getDataStorageFormat()) + && getDataStorageConfiguration().getUnstable().getBonsaiLimitTrieLogsEnabled()) { builder.setLimitTrieLogsEnabled(); builder.setTrieLogRetentionLimit(getDataStorageConfiguration().getBonsaiMaxLayersToLoad()); builder.setTrieLogsPruningWindowSize( diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java index 51fb4d3ade1..12fccf313f3 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java @@ -24,6 +24,7 @@ import org.hyperledger.besu.cli.options.CLIOptions; import org.hyperledger.besu.cli.util.CommandLineUtils; +import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; @@ -86,6 +87,7 @@ public static class Unstable { @CommandLine.Option( hidden = true, names = {BONSAI_LIMIT_TRIE_LOGS_ENABLED, "--Xbonsai-trie-log-pruning-enabled"}, + fallbackValue = "true", description = "Limit the number of trie logs that are retained. (default: ${DEFAULT-VALUE})") private boolean bonsaiLimitTrieLogsEnabled = DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED; @@ -134,9 +136,18 @@ public static DataStorageOptions create() { * Validates the data storage options * * @param commandLine the full commandLine to check all the options specified by the user + * @param syncMode the sync mode */ - public void validate(final CommandLine commandLine) { - if (unstableOptions.bonsaiLimitTrieLogsEnabled) { + public void validate(final CommandLine commandLine, final SyncMode syncMode) { + if (DataStorageFormat.BONSAI == dataStorageFormat + && unstableOptions.bonsaiLimitTrieLogsEnabled) { + if (SyncMode.FULL == syncMode) { + throw new CommandLine.ParameterException( + commandLine, + String.format( + "Cannot enable " + Unstable.BONSAI_LIMIT_TRIE_LOGS_ENABLED + " with sync-mode %s", + syncMode)); + } if (bonsaiMaxLayersToLoad < MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT) { throw new CommandLine.ParameterException( commandLine, diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 74579a975d2..1fc512344c8 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -1087,8 +1087,8 @@ public void syncMode_fast() { } @Test - public void syncMode_full() { - parseCommand("--sync-mode", "FULL"); + public void syncMode_full_requires_bonsaiLimitTrieLogsToBeDisabled() { + parseCommand("--sync-mode", "FULL", "--Xbonsai-limit-trie-logs-enabled=false"); verify(mockControllerBuilder).synchronizerConfiguration(syncConfigurationCaptor.capture()); final SynchronizerConfiguration syncConfig = syncConfigurationCaptor.getValue(); @@ -1244,8 +1244,37 @@ public void ethStatsContactOptionCannotBeUsedWithoutEthStatsServerProvided() { } @Test - public void parsesValidBonsaiTrieLimitBackLayersOption() { - parseCommand("--data-storage-format", "BONSAI", "--bonsai-historical-block-limit", "11"); + public void bonsaiLimitTrieLogsEnabledByDefault() { + parseCommand(); + verify(mockControllerBuilder) + .dataStorageConfiguration(dataStorageConfigurationArgumentCaptor.capture()); + + final DataStorageConfiguration dataStorageConfiguration = + dataStorageConfigurationArgumentCaptor.getValue(); + assertThat(dataStorageConfiguration.getDataStorageFormat()).isEqualTo(BONSAI); + assertThat(dataStorageConfiguration.getUnstable().getBonsaiLimitTrieLogsEnabled()).isTrue(); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void parsesInvalidDefaultBonsaiLimitTrieLogsWhenFullSyncEnabled() { + parseCommand("--sync-mode=FULL"); + + Mockito.verifyNoInteractions(mockRunnerBuilder); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains("Cannot enable --Xbonsai-limit-trie-logs-enabled with sync-mode FULL"); + } + + @Test + public void parsesValidBonsaiHistoricalBlockLimitOption() { + parseCommand( + "--Xbonsai-limit-trie-logs-enabled=false", + "--data-storage-format", + "BONSAI", + "--bonsai-historical-block-limit", + "11"); verify(mockControllerBuilder) .dataStorageConfiguration(dataStorageConfigurationArgumentCaptor.capture()); @@ -1258,7 +1287,7 @@ public void parsesValidBonsaiTrieLimitBackLayersOption() { } @Test - public void parsesInvalidBonsaiTrieLimitBackLayersOption() { + public void parsesInvalidBonsaiHistoricalBlockLimitOption() { parseCommand("--data-storage-format", "BONSAI", "--bonsai-maximum-back-layers-to-load", "ten"); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java index 4508f6c4e59..b2ec0efe9be 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java @@ -38,6 +38,15 @@ public void bonsaiTrieLogPruningLimitOption() { "600"); } + @Test + public void bonsaiTrieLogsEnabled_explicitlySetToFalse() { + internalTestSuccess( + dataStorageConfiguration -> + assertThat(dataStorageConfiguration.getUnstable().getBonsaiLimitTrieLogsEnabled()) + .isEqualTo(false), + "--Xbonsai-limit-trie-logs-enabled=false"); + } + @Test public void bonsaiTrieLogPruningWindowSizeShouldBePositive() { internalTestFailure( diff --git a/config/src/main/resources/profiles/enterprise-private.toml b/config/src/main/resources/profiles/enterprise-private.toml index 9ae88a55f27..c78d54ada9b 100644 --- a/config/src/main/resources/profiles/enterprise-private.toml +++ b/config/src/main/resources/profiles/enterprise-private.toml @@ -6,4 +6,5 @@ tx-pool="SEQUENCED" tx-pool-no-local-priority=true tx-pool-limit-by-account-percentage=0.15 rpc-http-max-active-connections=300 -min-gas-price=0 \ No newline at end of file +min-gas-price=0 +Xbonsai-limit-trie-logs-enabled=false \ No newline at end of file diff --git a/config/src/main/resources/profiles/minimalist-staker.toml b/config/src/main/resources/profiles/minimalist-staker.toml index fcdbf64363f..5efba0f6fb4 100644 --- a/config/src/main/resources/profiles/minimalist-staker.toml +++ b/config/src/main/resources/profiles/minimalist-staker.toml @@ -1,4 +1,4 @@ sync-mode="CHECKPOINT" data-storage-format="BONSAI" -bonsai-historical-block-limit=128 +bonsai-historical-block-limit=512 max-peers=25 \ No newline at end of file diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java index a14f86bbb5f..97dd1691a45 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java @@ -69,7 +69,7 @@ default Unstable getUnstable() { @Value.Immutable interface Unstable { - boolean DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED = false; + boolean DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED = true; long MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; int DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE = 30_000; boolean DEFAULT_BONSAI_FULL_FLAT_DB_ENABLED = true;