From 16c0a49b69285e6e12121b574f791258679269f3 Mon Sep 17 00:00:00 2001 From: Simon Dudley Date: Mon, 20 Nov 2023 20:17:01 +1000 Subject: [PATCH 01/37] Add x-trie-log subcommand for one-off backlog prune Signed-off-by: Simon Dudley Signed-off-by: Gabriel Fukushima --- .../storage/StorageSubCommand.java | 2 +- .../subcommands/storage/TrieLogHelper.java | 179 ++++++++++++++++++ .../storage/TrieLogSubCommand.java | 146 ++++++++++++++ .../besu/controller/BesuController.java | 18 +- .../controller/BesuControllerBuilder.java | 3 +- .../trie/bonsai/trielog/TrieLogPruner.java | 15 +- 6 files changed, 355 insertions(+), 8 deletions(-) create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java index e46ffde2739..a7cdc124d06 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java @@ -45,7 +45,7 @@ description = "This command provides storage related actions.", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class, - subcommands = {StorageSubCommand.RevertVariablesStorage.class, RocksDbSubCommand.class}) + subcommands = {StorageSubCommand.RevertVariablesStorage.class, RocksDbSubCommand.class, TrieLogSubCommand.class}) public class StorageSubCommand implements Runnable { /** The constant COMMAND_NAME. */ diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java new file mode 100644 index 00000000000..829a603c2d4 --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -0,0 +1,179 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.cli.subcommands.storage; + +import static com.google.common.base.Preconditions.checkArgument; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; + +import org.hyperledger.besu.controller.BesuController; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.chain.MutableBlockchain; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; + +import java.io.PrintWriter; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.tuweni.bytes.Bytes32; + +/** Helper class for counting and pruning trie logs */ +public class TrieLogHelper { + + static void countAndPrune( + final PrintWriter out, + final DataStorageConfiguration config, + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final MutableBlockchain blockchain, + final BesuController besuController) { + TrieLogHelper.validatePruneConfiguration(config); + + final TrieLogCount count = getCount(rootWorldStateStorage, Integer.MAX_VALUE, blockchain); + + out.println("Counting trie logs before prune..."); + printCount(out, count); + out.println(); + + final long layersToRetain = config.getUnstable().getBonsaiTrieLogRetentionThreshold(); + final int batchSize = config.getUnstable().getBonsaiTrieLogPruningLimit(); + final boolean isProofOfStake = + besuController.getGenesisConfigOptions().getTerminalTotalDifficulty().isPresent(); + TrieLogPruner pruner = + new TrieLogPruner( + rootWorldStateStorage, blockchain, layersToRetain, batchSize, isProofOfStake); + + final long totalToPrune = count.total() - layersToRetain; + out.printf( + """ + Total to prune = %d (total) - %d (retention threshold) = + => %d + """, + count.total(), layersToRetain, totalToPrune); + final long numBatches = Math.max(totalToPrune / batchSize, 1); + out.println(); + out.printf( + "Estimated number of batches = max(%d (total to prune) / %d (batch size), 1) = %d\n", + totalToPrune, batchSize, numBatches); + out.println(); + + int noProgressCounter = 0; + int prevTotalNumberPruned = 0; + int totalNumberPruned = 0; + int numberPrunedInBatch; + int batchNumber = 1; + while (totalNumberPruned < totalToPrune) { + out.printf( + """ + Pruning batch %d + ----------------- + """, batchNumber++); + // do prune + numberPrunedInBatch = pruner.initialize(); + + out.printf("Number pruned in batch = %d \n", numberPrunedInBatch); + totalNumberPruned += numberPrunedInBatch; + out.printf( + """ + Running total number pruned = + => %d of %d + """, + totalNumberPruned, totalToPrune); + + if (totalNumberPruned == prevTotalNumberPruned) { + if (noProgressCounter++ == 5) { + out.println("No progress in 5 batches, exiting"); + return; + } + } + + prevTotalNumberPruned = totalNumberPruned; + out.println(); + } + out.println("Trie log prune complete!"); + out.println(); + + out.println("Counting trie logs after prune..."); + TrieLogHelper.printCount( + out, TrieLogHelper.getCount(rootWorldStateStorage, Integer.MAX_VALUE, blockchain)); + } + + private static void validatePruneConfiguration(final DataStorageConfiguration config) { + checkArgument( + config.getUnstable().getBonsaiTrieLogRetentionThreshold() + >= MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD, + String.format( + "--Xbonsai-trie-log-retention-threshold minimum value is %d", + MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD)); + checkArgument( + config.getUnstable().getBonsaiTrieLogPruningLimit() > 0, + String.format( + "--Xbonsai-trie-log-pruning-limit=%d must be greater than 0", + config.getUnstable().getBonsaiTrieLogPruningLimit())); + checkArgument( + config.getUnstable().getBonsaiTrieLogPruningLimit() + > config.getUnstable().getBonsaiTrieLogRetentionThreshold(), + String.format( + "--Xbonsai-trie-log-pruning-limit=%d must greater than --Xbonsai-trie-log-retention-threshold=%d", + config.getUnstable().getBonsaiTrieLogPruningLimit(), + config.getUnstable().getBonsaiTrieLogRetentionThreshold())); + } + + static TrieLogCount getCount( + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final int limit, + final Blockchain blockchain) { + final AtomicInteger total = new AtomicInteger(); + final AtomicInteger canonicalCount = new AtomicInteger(); + final AtomicInteger forkCount = new AtomicInteger(); + final AtomicInteger orphanCount = new AtomicInteger(); + rootWorldStateStorage + .streamTrieLogKeys(limit) + .map(Bytes32::wrap) + .map(Hash::wrap) + .forEach( + hash -> { + total.getAndIncrement(); + blockchain + .getBlockHeader(hash) + .ifPresentOrElse( + (header) -> { + long number = header.getNumber(); + final Optional headerByNumber = + blockchain.getBlockHeader(number); + if (headerByNumber.isPresent() + && headerByNumber.get().getHash().equals(hash)) { + canonicalCount.getAndIncrement(); + } else { + forkCount.getAndIncrement(); + } + }, + orphanCount::getAndIncrement); + }); + + return new TrieLogCount(total.get(), canonicalCount.get(), forkCount.get(), orphanCount.get()); + } + + static void printCount(final PrintWriter out, final TrieLogCount count) { + out.printf( + "trieLog count: %s\n - canonical count: %s\n - fork count: %s\n - orphaned count: %s\n", + count.total, count.canonicalCount, count.forkCount, count.orphanCount); + } + + record TrieLogCount(int total, int canonicalCount, int forkCount, int orphanCount) {} +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java new file mode 100644 index 00000000000..dea82d7c33f --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -0,0 +1,146 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.subcommands.storage; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import org.hyperledger.besu.cli.util.VersionProvider; +import org.hyperledger.besu.controller.BesuController; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.chain.MutableBlockchain; +import org.hyperledger.besu.ethereum.storage.StorageProvider; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; + +import java.io.PrintWriter; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.config.Configurator; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.ParentCommand; + +/** The Trie Log subcommand. */ +@Command( + name = "x-trie-log", + description = "Manipulate trie logs", + mixinStandardHelpOptions = true, + versionProvider = VersionProvider.class, + subcommands = {TrieLogSubCommand.CountTrieLog.class, TrieLogSubCommand.PruneTrieLog.class}) +public class TrieLogSubCommand implements Runnable { + + @SuppressWarnings("UnusedVariable") + @ParentCommand + private static StorageSubCommand parentCommand; + + @SuppressWarnings("unused") + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec + + @Override + public void run() { + final PrintWriter out = spec.commandLine().getOut(); + spec.commandLine().usage(out); + } + + private static BesuController createBesuController() { + return parentCommand.parentCommand.buildController(); + } + + @Command( + name = "count", + description = "This command counts all the trie logs", + mixinStandardHelpOptions = true, + versionProvider = VersionProvider.class) + static class CountTrieLog implements Runnable { + + @SuppressWarnings("unused") + @ParentCommand + private TrieLogSubCommand parentCommand; + + @SuppressWarnings("unused") + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec + + @Override + public void run() { + TrieLogContext context = getTrieLogContext(); + + final PrintWriter out = spec.commandLine().getOut(); + + out.println("Counting trie logs..."); + TrieLogHelper.printCount( + out, + TrieLogHelper.getCount( + context.rootWorldStateStorage, Integer.MAX_VALUE, context.blockchain)); + } + } + + @Command( + name = "prune", + description = + "This command prunes all trie log layers below the retention threshold, including orphaned trie logs.", + mixinStandardHelpOptions = true, + versionProvider = VersionProvider.class) + static class PruneTrieLog implements Runnable { + + @SuppressWarnings("unused") + @ParentCommand + private TrieLogSubCommand parentCommand; + + @SuppressWarnings("unused") + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec + + @Override + public void run() { + TrieLogContext context = getTrieLogContext(); + + TrieLogHelper.countAndPrune( + spec.commandLine().getOut(), + context.config(), + context.rootWorldStateStorage(), + context.blockchain(), + context.besuController()); + } + } + + record TrieLogContext( + BesuController besuController, + DataStorageConfiguration config, + BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + MutableBlockchain blockchain) {} + + @SuppressWarnings("BannedMethod") + private static TrieLogContext getTrieLogContext() { + Configurator.setLevel(LogManager.getLogger(TrieLogPruner.class).getName(), Level.DEBUG); + checkNotNull(parentCommand); + BesuController besuController = createBesuController(); + final DataStorageConfiguration config = besuController.getDataStorageConfiguration(); + checkArgument( + DataStorageFormat.BONSAI.equals(config.getDataStorageFormat()), + "Subcommand only works with data-storage-format=BONSAI"); + + final StorageProvider storageProvider = besuController.getStorageProvider(); + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage = + (BonsaiWorldStateKeyValueStorage) + storageProvider.createWorldStateStorage(DataStorageFormat.BONSAI); + final MutableBlockchain blockchain = besuController.getProtocolContext().getBlockchain(); + return new TrieLogContext(besuController, config, rootWorldStateStorage, blockchain); + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuController.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuController.java index bb46003b24b..12efc4a9df5 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuController.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuController.java @@ -37,6 +37,7 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.p2p.config.SubProtocolConfiguration; import org.hyperledger.besu.ethereum.storage.StorageProvider; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import java.io.Closeable; import java.io.IOException; @@ -77,6 +78,7 @@ public class BesuController implements java.io.Closeable { private final SyncState syncState; private final EthPeers ethPeers; private final StorageProvider storageProvider; + private final DataStorageConfiguration dataStorageConfiguration; /** * Instantiates a new Besu controller. @@ -96,6 +98,9 @@ public class BesuController implements java.io.Closeable { * @param nodeKey the node key * @param closeables the closeables * @param additionalPluginServices the additional plugin services + * @param ethPeers the eth peers + * @param storageProvider the storage provider + * @param dataStorageConfiguration the data storage configuration */ BesuController( final ProtocolSchedule protocolSchedule, @@ -114,7 +119,8 @@ public class BesuController implements java.io.Closeable { final List closeables, final PluginServiceFactory additionalPluginServices, final EthPeers ethPeers, - final StorageProvider storageProvider) { + final StorageProvider storageProvider, + final DataStorageConfiguration dataStorageConfiguration) { this.protocolSchedule = protocolSchedule; this.protocolContext = protocolContext; this.ethProtocolManager = ethProtocolManager; @@ -132,6 +138,7 @@ public class BesuController implements java.io.Closeable { this.additionalPluginServices = additionalPluginServices; this.ethPeers = ethPeers; this.storageProvider = storageProvider; + this.dataStorageConfiguration = dataStorageConfiguration; } /** @@ -293,6 +300,15 @@ public PluginServiceFactory getAdditionalPluginServices() { return additionalPluginServices; } + /** + * Gets data storage configuration. + * + * @return the data storage configuration + */ + public DataStorageConfiguration getDataStorageConfiguration() { + return dataStorageConfiguration; + } + /** The type Builder. */ public static class Builder { diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 92cbf320e84..6bcfea3cad8 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -803,7 +803,8 @@ public BesuController build() { closeables, additionalPluginServices, ethPeers, - storageProvider); + storageProvider, + dataStorageConfiguration); } /** diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java index 747a82e1621..f7356f9ed2a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java @@ -61,17 +61,18 @@ public TrieLogPruner( this.requireFinalizedBlock = requireFinalizedBlock; } - public void initialize() { - preloadQueue(); + public int initialize() { + return preloadQueue(); } - private void preloadQueue() { + private int preloadQueue() { LOG.atInfo() .setMessage("Loading first {} trie logs from database...") .addArgument(loadingLimit) .log(); try (final Stream trieLogKeys = rootWorldStateStorage.streamTrieLogKeys(loadingLimit)) { final AtomicLong count = new AtomicLong(); + final AtomicLong orphansPruned = new AtomicLong(); trieLogKeys.forEach( blockHashAsBytes -> { final Hash blockHash = Hash.wrap(Bytes32.wrap(blockHashAsBytes)); @@ -82,12 +83,15 @@ private void preloadQueue() { } else { // prune orphaned blocks (sometimes created during block production) rootWorldStateStorage.pruneTrieLog(blockHash); + orphansPruned.getAndIncrement(); } }); + LOG.atDebug().log("Pruned {} orphaned trie logs from database...", orphansPruned.intValue()); LOG.atInfo().log("Loaded {} trie logs from database", count); - pruneFromQueue(); + return pruneFromQueue() + orphansPruned.intValue(); } catch (Exception e) { LOG.error("Error loading trie logs from database, nothing pruned", e); + return 0; } } @@ -176,8 +180,9 @@ private NoOpTrieLogPruner( } @Override - public void initialize() { + public int initialize() { // no-op + return -1; } @Override From 7dd4928397465d2aba212dba1af590b014a846ee Mon Sep 17 00:00:00 2001 From: Simon Dudley Date: Mon, 20 Nov 2023 20:32:48 +1000 Subject: [PATCH 02/37] long -> int Signed-off-by: Simon Dudley Signed-off-by: Gabriel Fukushima --- .../besu/cli/subcommands/storage/TrieLogHelper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 829a603c2d4..4e3869eb22f 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -50,7 +50,7 @@ static void countAndPrune( printCount(out, count); out.println(); - final long layersToRetain = config.getUnstable().getBonsaiTrieLogRetentionThreshold(); + final int layersToRetain = (int) config.getUnstable().getBonsaiTrieLogRetentionThreshold(); final int batchSize = config.getUnstable().getBonsaiTrieLogPruningLimit(); final boolean isProofOfStake = besuController.getGenesisConfigOptions().getTerminalTotalDifficulty().isPresent(); @@ -58,7 +58,7 @@ static void countAndPrune( new TrieLogPruner( rootWorldStateStorage, blockchain, layersToRetain, batchSize, isProofOfStake); - final long totalToPrune = count.total() - layersToRetain; + final int totalToPrune = count.total() - layersToRetain; out.printf( """ Total to prune = %d (total) - %d (retention threshold) = From bf2b0985d22f7d4e65400b6c9980238ef93dfa6f Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Tue, 12 Dec 2023 15:29:10 +1000 Subject: [PATCH 03/37] Removed banned method Signed-off-by: Gabriel Fukushima --- .../besu/cli/subcommands/storage/TrieLogSubCommand.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index dea82d7c33f..aeb296cc1a0 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -29,8 +29,8 @@ import java.io.PrintWriter; import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.config.Configurator; +import org.slf4j.LoggerFactory; import picocli.CommandLine; import picocli.CommandLine.Command; import picocli.CommandLine.ParentCommand; @@ -126,9 +126,9 @@ record TrieLogContext( BonsaiWorldStateKeyValueStorage rootWorldStateStorage, MutableBlockchain blockchain) {} - @SuppressWarnings("BannedMethod") + private static TrieLogContext getTrieLogContext() { - Configurator.setLevel(LogManager.getLogger(TrieLogPruner.class).getName(), Level.DEBUG); + Configurator.setLevel(LoggerFactory.getLogger(TrieLogPruner.class).getName(), Level.DEBUG); checkNotNull(parentCommand); BesuController besuController = createBesuController(); final DataStorageConfiguration config = besuController.getDataStorageConfiguration(); From e67ae5122382374acd0303c504fd23ac2eba16e8 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Tue, 12 Dec 2023 15:29:57 +1000 Subject: [PATCH 04/37] Preload process stream in parallel Signed-off-by: Gabriel Fukushima --- .../besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java index f7356f9ed2a..e8ea655fde1 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java @@ -73,12 +73,12 @@ private int preloadQueue() { try (final Stream trieLogKeys = rootWorldStateStorage.streamTrieLogKeys(loadingLimit)) { final AtomicLong count = new AtomicLong(); final AtomicLong orphansPruned = new AtomicLong(); - trieLogKeys.forEach( + trieLogKeys.parallel().forEach( blockHashAsBytes -> { final Hash blockHash = Hash.wrap(Bytes32.wrap(blockHashAsBytes)); final Optional header = blockchain.getBlockHeader(blockHash); if (header.isPresent()) { - trieLogBlocksAndForksByDescendingBlockNumber.put(header.get().getNumber(), blockHash); + addToPruneQueue(header.get().getNumber(), blockHash); count.getAndIncrement(); } else { // prune orphaned blocks (sometimes created during block production) From 9b4e0c9f3aaf69e2f9f08fec94904bb9428c809f Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Thu, 14 Dec 2023 11:46:48 +1000 Subject: [PATCH 05/37] Drop unwanted trielogs and keep reatain layers only Signed-off-by: Gabriel Fukushima --- .../storage/StorageSubCommand.java | 6 +- .../subcommands/storage/TrieLogHelper.java | 243 +++++++++++++----- .../storage/TrieLogSubCommand.java | 17 +- .../BonsaiWorldStateKeyValueStorage.java | 2 +- .../trie/bonsai/trielog/TrieLogPruner.java | 2 +- 5 files changed, 198 insertions(+), 72 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java index a7cdc124d06..0dbfa3d22c5 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java @@ -45,7 +45,11 @@ description = "This command provides storage related actions.", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class, - subcommands = {StorageSubCommand.RevertVariablesStorage.class, RocksDbSubCommand.class, TrieLogSubCommand.class}) + subcommands = { + StorageSubCommand.RevertVariablesStorage.class, + RocksDbSubCommand.class, + TrieLogSubCommand.class + }) public class StorageSubCommand implements Runnable { /** The constant COMMAND_NAME. */ diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 4e3869eb22f..2586ce2fa9f 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -18,23 +18,47 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; +import com.google.common.base.Splitter; +import org.apache.tuweni.bytes.Bytes; +import org.bouncycastle.util.encoders.Base64; import org.hyperledger.besu.controller.BesuController; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; +import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; +import org.hyperledger.besu.ethereum.rlp.RLPOutput; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; import java.io.PrintWriter; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import org.apache.tuweni.bytes.Bytes32; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** Helper class for counting and pruning trie logs */ public class TrieLogHelper { + private static final String trieLogFile + = "trieLogsToRetain.txt"; + private static final Logger LOG = LoggerFactory.getLogger(TrieLogHelper.class); static void countAndPrune( final PrintWriter out, @@ -42,75 +66,67 @@ static void countAndPrune( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final MutableBlockchain blockchain, final BesuController besuController) { + TrieLogHelper.validatePruneConfiguration(config); + final long layersToRetain = config.getUnstable().getBonsaiTrieLogRetentionThreshold(); + final long chainHeight = blockchain.getChainHeadBlockNumber(); + final Optional finalizedBlockHash = blockchain.getFinalized(); + if (finalizedBlockHash.isEmpty()) { + LOG.debug("No finalized block present, skipping pruning"); + return ; + } - final TrieLogCount count = getCount(rootWorldStateStorage, Integer.MAX_VALUE, blockchain); + final long numberOfTrieLogsToRetain = + finalizedBlockHash + .flatMap(blockchain::getBlockHeader) + .map(ProcessableBlockHeader::getNumber) + .map(finalizedBlock -> Math.min(chainHeight-finalizedBlock, layersToRetain)) + .orElse(layersToRetain); - out.println("Counting trie logs before prune..."); - printCount(out, count); - out.println(); + // retrieve the layersToRetains hashes from blockchain + final List hashesToRetain = new ArrayList<>(); - final int layersToRetain = (int) config.getUnstable().getBonsaiTrieLogRetentionThreshold(); - final int batchSize = config.getUnstable().getBonsaiTrieLogPruningLimit(); - final boolean isProofOfStake = - besuController.getGenesisConfigOptions().getTerminalTotalDifficulty().isPresent(); - TrieLogPruner pruner = - new TrieLogPruner( - rootWorldStateStorage, blockchain, layersToRetain, batchSize, isProofOfStake); + final long lastHashToRetain = chainHeight - numberOfTrieLogsToRetain; + for (long i = chainHeight; i >= lastHashToRetain; i--) { + final Optional header = blockchain.getBlockHeader(i); + header.ifPresent(blockHeader -> hashesToRetain.add(blockHeader.getHash())); + } + + IdentityHashMap trieLogsToRetain = new IdentityHashMap<>(); + + if(hashesToRetain.stream().count() == numberOfTrieLogsToRetain){ + //save trielogs in a flatfile as a fail-safe + saveTrieLogsInFile(trieLogsToRetain); - final int totalToPrune = count.total() - layersToRetain; - out.printf( - """ - Total to prune = %d (total) - %d (retention threshold) = - => %d - """, - count.total(), layersToRetain, totalToPrune); - final long numBatches = Math.max(totalToPrune / batchSize, 1); - out.println(); - out.printf( - "Estimated number of batches = max(%d (total to prune) / %d (batch size), 1) = %d\n", - totalToPrune, batchSize, numBatches); - out.println(); - - int noProgressCounter = 0; - int prevTotalNumberPruned = 0; - int totalNumberPruned = 0; - int numberPrunedInBatch; - int batchNumber = 1; - while (totalNumberPruned < totalToPrune) { - out.printf( - """ - Pruning batch %d - ----------------- - """, batchNumber++); - // do prune - numberPrunedInBatch = pruner.initialize(); - - out.printf("Number pruned in batch = %d \n", numberPrunedInBatch); - totalNumberPruned += numberPrunedInBatch; - out.printf( - """ - Running total number pruned = - => %d of %d - """, - totalNumberPruned, totalToPrune); - - if (totalNumberPruned == prevTotalNumberPruned) { - if (noProgressCounter++ == 5) { - out.println("No progress in 5 batches, exiting"); - return; - } - } - prevTotalNumberPruned = totalNumberPruned; - out.println(); + hashesToRetain.forEach( + hash -> { + rootWorldStateStorage + .getTrieLog(hash) + .ifPresent(trieLog -> trieLogsToRetain.put(hash.toArrayUnsafe(), trieLog)); + }); + } + else{ + //try to read the triLogs from the flatfile + trieLogsToRetain = readTrieLogsFromFile(); + } + + //clear trielogs storage + rootWorldStateStorage.clearTrieLog(); + + //get an update and insert the trielogs we retained + var updater = rootWorldStateStorage.updater(); + trieLogsToRetain.forEach( + (key, value) -> { + System.out.println("key: " + Bytes32.wrap(key).toHexString() + ", value: " + Base64.toBase64String(value)); + updater.getTrieLogStorageTransaction().put(key, value); + }); + updater.getTrieLogStorageTransaction().commit(); + + if(rootWorldStateStorage.streamTrieLogKeys(numberOfTrieLogsToRetain).count() == numberOfTrieLogsToRetain){ + deleteTrieLogFile(); } - out.println("Trie log prune complete!"); - out.println(); - out.println("Counting trie logs after prune..."); - TrieLogHelper.printCount( - out, TrieLogHelper.getCount(rootWorldStateStorage, Integer.MAX_VALUE, blockchain)); } private static void validatePruneConfiguration(final DataStorageConfiguration config) { @@ -169,6 +185,107 @@ static TrieLogCount getCount( return new TrieLogCount(total.get(), canonicalCount.get(), forkCount.get(), orphanCount.get()); } + private static boolean saveTrieLogsInFile(final Map trieLogs){ + + File file = new File(trieLogFile); + + BufferedWriter bf = null; + + try { + + bf = new BufferedWriter(new FileWriter(file, StandardCharsets.UTF_8)); + for (Map.Entry entry : + trieLogs.entrySet()) { + + bf.write(Bytes.of(entry.getKey()) + ":" + + Base64.toBase64String(entry.getValue())); + + bf.newLine(); + } + + bf.flush(); + } + catch (IOException e) { + LOG.error(e.getMessage()); + return false; + } + finally { + + try { + + bf.close(); + } + catch (Exception e) { + LOG.error(e.getMessage()); + } + } + return true; + } + + private static IdentityHashMap readTrieLogsFromFile() { + + + File file = new File(trieLogFile); + + IdentityHashMap trieLogs = new IdentityHashMap<>(); + + + BufferedReader br = null; + + + try { + + + br = new BufferedReader(new FileReader(file, StandardCharsets.UTF_8)); + + + String line; + + while ((line = br.readLine()) != null) { + + + List parts = Splitter.on(':').splitToList(line); + + byte[] key = Bytes.fromHexString(parts.get(0)).toArrayUnsafe(); + + byte[] value = Base64.decode(parts.get(1)); + + + trieLogs.put(key, value); + + } + + } catch (IOException e) { + + LOG.error(e.getMessage()); + + } finally { + + try { + + if (br != null) { + + br.close(); + + } + + } catch (Exception e) { + + LOG.error(e.getMessage()); + + } + + } + + return trieLogs; + + } + + private static void deleteTrieLogFile(){ + File file = new File(trieLogFile); + file.delete(); + } + static void printCount(final PrintWriter out, final TrieLogCount count) { out.printf( "trieLog count: %s\n - canonical count: %s\n - fork count: %s\n - orphaned count: %s\n", @@ -176,4 +293,6 @@ static void printCount(final PrintWriter out, final TrieLogCount count) { } record TrieLogCount(int total, int canonicalCount, int forkCount, int orphanCount) {} + + } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index aeb296cc1a0..6d4428dcd54 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -111,12 +111,16 @@ static class PruneTrieLog implements Runnable { public void run() { TrieLogContext context = getTrieLogContext(); - TrieLogHelper.countAndPrune( - spec.commandLine().getOut(), - context.config(), - context.rootWorldStateStorage(), - context.blockchain(), - context.besuController()); + try { + TrieLogHelper.countAndPrune( + spec.commandLine().getOut(), + context.config(), + context.rootWorldStateStorage(), + context.blockchain(), + context.besuController()); + } catch (Exception e) { + throw new RuntimeException(e); + } } } @@ -126,7 +130,6 @@ record TrieLogContext( BonsaiWorldStateKeyValueStorage rootWorldStateStorage, MutableBlockchain blockchain) {} - private static TrieLogContext getTrieLogContext() { Configurator.setLevel(LoggerFactory.getLogger(TrieLogPruner.class).getName(), Level.DEBUG); checkNotNull(parentCommand); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java index 54089c7e212..97384fd6c12 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java @@ -204,7 +204,7 @@ public Optional getTrieLog(final Hash blockHash) { return trieLogStorage.get(blockHash.toArrayUnsafe()); } - public Stream streamTrieLogKeys(final int limit) { + public Stream streamTrieLogKeys(final long limit) { return trieLogStorage.streamKeys().limit(limit); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java index e8ea655fde1..b1bf75818e9 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java @@ -73,7 +73,7 @@ private int preloadQueue() { try (final Stream trieLogKeys = rootWorldStateStorage.streamTrieLogKeys(loadingLimit)) { final AtomicLong count = new AtomicLong(); final AtomicLong orphansPruned = new AtomicLong(); - trieLogKeys.parallel().forEach( + trieLogKeys.forEach( blockHashAsBytes -> { final Hash blockHash = Hash.wrap(Bytes32.wrap(blockHashAsBytes)); final Optional header = blockchain.getBlockHeader(blockHash); From 0b9fe83ad0797e7d044aca4d26fef3b9a21009f8 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Fri, 15 Dec 2023 14:01:35 +1000 Subject: [PATCH 06/37] Add output to user and cleanup refactor Signed-off-by: Gabriel Fukushima --- .../subcommands/storage/TrieLogHelper.java | 225 ++++++------------ .../storage/TrieLogSubCommand.java | 8 +- 2 files changed, 80 insertions(+), 153 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 2586ce2fa9f..31207a399e6 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -18,18 +18,12 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; -import com.google.common.base.Splitter; -import org.apache.tuweni.bytes.Bytes; -import org.bouncycastle.util.encoders.Base64; import org.hyperledger.besu.controller.BesuController; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; -import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; -import org.hyperledger.besu.ethereum.rlp.RLPOutput; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import java.io.BufferedReader; @@ -39,94 +33,91 @@ import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; -import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; +import com.google.common.base.Splitter; +import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; +import org.bouncycastle.util.encoders.Base64; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Helper class for counting and pruning trie logs */ public class TrieLogHelper { - private static final String trieLogFile - = "trieLogsToRetain.txt"; - private static final Logger LOG = LoggerFactory.getLogger(TrieLogHelper.class); + private static final String trieLogFile = "trieLogsToRetain.txt"; + private static final Logger LOG = LoggerFactory.getLogger(TrieLogHelper.class); - static void countAndPrune( + static void prune( final PrintWriter out, final DataStorageConfiguration config, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final MutableBlockchain blockchain, - final BesuController besuController) { + final MutableBlockchain blockchain) throws IOException { TrieLogHelper.validatePruneConfiguration(config); final long layersToRetain = config.getUnstable().getBonsaiTrieLogRetentionThreshold(); final long chainHeight = blockchain.getChainHeadBlockNumber(); + final long lastBlockToRetainTrieLogsFor = chainHeight - layersToRetain; final Optional finalizedBlockHash = blockchain.getFinalized(); - if (finalizedBlockHash.isEmpty()) { - LOG.debug("No finalized block present, skipping pruning"); - return ; + if (finalizedBlockHash.isEmpty()) { + LOG.error("No finalized block present, skipping pruning"); + return; + } else { + if (blockchain.getBlockHeader(finalizedBlockHash.get()).get().getNumber() + < lastBlockToRetainTrieLogsFor) { + LOG.error("Trying to prune more layers than the finalized block height, skipping pruning"); + return; } - - final long numberOfTrieLogsToRetain = - finalizedBlockHash - .flatMap(blockchain::getBlockHeader) - .map(ProcessableBlockHeader::getNumber) - .map(finalizedBlock -> Math.min(chainHeight-finalizedBlock, layersToRetain)) - .orElse(layersToRetain); + } // retrieve the layersToRetains hashes from blockchain final List hashesToRetain = new ArrayList<>(); - final long lastHashToRetain = chainHeight - numberOfTrieLogsToRetain; - for (long i = chainHeight; i >= lastHashToRetain; i--) { + for (long i = chainHeight; i > lastBlockToRetainTrieLogsFor; i--) { final Optional header = blockchain.getBlockHeader(i); header.ifPresent(blockHeader -> hashesToRetain.add(blockHeader.getHash())); } - IdentityHashMap trieLogsToRetain = new IdentityHashMap<>(); - - if(hashesToRetain.stream().count() == numberOfTrieLogsToRetain){ - //save trielogs in a flatfile as a fail-safe - saveTrieLogsInFile(trieLogsToRetain); - - - hashesToRetain.forEach( - hash -> { - rootWorldStateStorage - .getTrieLog(hash) - .ifPresent(trieLog -> trieLogsToRetain.put(hash.toArrayUnsafe(), trieLog)); - }); - } - else{ - //try to read the triLogs from the flatfile - trieLogsToRetain = readTrieLogsFromFile(); + IdentityHashMap trieLogsToRetain; + if ((long) hashesToRetain.size() == layersToRetain) { + trieLogsToRetain = new IdentityHashMap<>(); + // save trielogs in a flatfile as a fail-safe + out.println("Obtaining trielogs to retain..."); + hashesToRetain.forEach( + hash -> { + rootWorldStateStorage + .getTrieLog(hash) + .ifPresent(trieLog -> trieLogsToRetain.put(hash.toArrayUnsafe(), trieLog)); + }); + out.println("Saving trielogs to retain in file..."); + saveTrieLogsInFile(trieLogsToRetain); + } else { + // try to read the triLogs from the flatfile + trieLogsToRetain = readTrieLogsFromFile(); } - - //clear trielogs storage + out.println("Clear trielogs..."); + // clear trielogs storage rootWorldStateStorage.clearTrieLog(); - //get an update and insert the trielogs we retained + // get an update and insert the trielogs we retained var updater = rootWorldStateStorage.updater(); + out.println("restore trielogs retained into db..."); trieLogsToRetain.forEach( (key, value) -> { - System.out.println("key: " + Bytes32.wrap(key).toHexString() + ", value: " + Base64.toBase64String(value)); updater.getTrieLogStorageTransaction().put(key, value); }); updater.getTrieLogStorageTransaction().commit(); - if(rootWorldStateStorage.streamTrieLogKeys(numberOfTrieLogsToRetain).count() == numberOfTrieLogsToRetain){ - deleteTrieLogFile(); + if (rootWorldStateStorage.streamTrieLogKeys(layersToRetain).count() == layersToRetain) { + out.println("Prune ran successfully. Deleting file..."); + deleteTrieLogFile(); } - + out.println("Enjoy some GBs of storage back!..."); } private static void validatePruneConfiguration(final DataStorageConfiguration config) { @@ -185,106 +176,46 @@ static TrieLogCount getCount( return new TrieLogCount(total.get(), canonicalCount.get(), forkCount.get(), orphanCount.get()); } - private static boolean saveTrieLogsInFile(final Map trieLogs){ - - File file = new File(trieLogFile); - - BufferedWriter bf = null; - - try { - - bf = new BufferedWriter(new FileWriter(file, StandardCharsets.UTF_8)); - for (Map.Entry entry : - trieLogs.entrySet()) { - - bf.write(Bytes.of(entry.getKey()) + ":" - + Base64.toBase64String(entry.getValue())); - - bf.newLine(); - } - - bf.flush(); - } - catch (IOException e) { - LOG.error(e.getMessage()); - return false; - } - finally { - - try { - - bf.close(); - } - catch (Exception e) { - LOG.error(e.getMessage()); - } - } - return true; - } - - private static IdentityHashMap readTrieLogsFromFile() { - - - File file = new File(trieLogFile); - - IdentityHashMap trieLogs = new IdentityHashMap<>(); - - - BufferedReader br = null; - + private static void saveTrieLogsInFile(final Map trieLogs) throws IOException { - try { + File file = new File(trieLogFile); + try (BufferedWriter bf = new BufferedWriter(new FileWriter(file, StandardCharsets.UTF_8))) { + for (Map.Entry entry : trieLogs.entrySet()) { + bf.write(Bytes.of(entry.getKey()) + ":" + Base64.toBase64String(entry.getValue())); + bf.newLine(); + } + bf.flush(); + } catch (IOException e) { + LOG.error(e.getMessage()); + throw e; + } + } - br = new BufferedReader(new FileReader(file, StandardCharsets.UTF_8)); - - - String line; - - while ((line = br.readLine()) != null) { - - - List parts = Splitter.on(':').splitToList(line); - - byte[] key = Bytes.fromHexString(parts.get(0)).toArrayUnsafe(); - - byte[] value = Base64.decode(parts.get(1)); - - - trieLogs.put(key, value); - - } - - } catch (IOException e) { - - LOG.error(e.getMessage()); - - } finally { - - try { - - if (br != null) { - - br.close(); - - } - - } catch (Exception e) { - - LOG.error(e.getMessage()); - - } - - } - - return trieLogs; + private static IdentityHashMap readTrieLogsFromFile() throws IOException { + + File file = new File(trieLogFile); + IdentityHashMap trieLogs = new IdentityHashMap<>(); + try (BufferedReader br = new BufferedReader(new FileReader(file, StandardCharsets.UTF_8))) { + String line; + while ((line = br.readLine()) != null) { + List parts = Splitter.on(':').splitToList(line); + byte[] key = Bytes.fromHexString(parts.get(0)).toArrayUnsafe(); + byte[] value = Base64.decode(parts.get(1)); + trieLogs.put(key, value); + } + } catch (IOException e) { + LOG.error(e.getMessage()); + throw e; + } - } + return trieLogs; + } - private static void deleteTrieLogFile(){ - File file = new File(trieLogFile); - file.delete(); - } + private static void deleteTrieLogFile() { + File file = new File(trieLogFile); + file.delete(); + } static void printCount(final PrintWriter out, final TrieLogCount count) { out.printf( @@ -293,6 +224,4 @@ static void printCount(final PrintWriter out, final TrieLogCount count) { } record TrieLogCount(int total, int canonicalCount, int forkCount, int orphanCount) {} - - } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index 6d4428dcd54..65756e30853 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -112,12 +112,11 @@ public void run() { TrieLogContext context = getTrieLogContext(); try { - TrieLogHelper.countAndPrune( + TrieLogHelper.prune( spec.commandLine().getOut(), context.config(), context.rootWorldStateStorage(), - context.blockchain(), - context.besuController()); + context.blockchain()); } catch (Exception e) { throw new RuntimeException(e); } @@ -125,7 +124,6 @@ public void run() { } record TrieLogContext( - BesuController besuController, DataStorageConfiguration config, BonsaiWorldStateKeyValueStorage rootWorldStateStorage, MutableBlockchain blockchain) {} @@ -144,6 +142,6 @@ private static TrieLogContext getTrieLogContext() { (BonsaiWorldStateKeyValueStorage) storageProvider.createWorldStateStorage(DataStorageFormat.BONSAI); final MutableBlockchain blockchain = besuController.getProtocolContext().getBlockchain(); - return new TrieLogContext(besuController, config, rootWorldStateStorage, blockchain); + return new TrieLogContext(config, rootWorldStateStorage, blockchain); } } From 426848e3697274904fc39ebfd8da743196ed2ba6 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Fri, 15 Dec 2023 14:03:08 +1000 Subject: [PATCH 07/37] small tweak to display cf that had reference dropped by RocksDbSegmentIdentifier.reset() Signed-off-by: Gabriel Fukushima --- .../besu/cli/subcommands/storage/RocksDbUsageHelper.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbUsageHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbUsageHelper.java index 4bfd9f42951..33f551d17f8 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbUsageHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbUsageHelper.java @@ -34,17 +34,19 @@ static void printUsageForColumnFamily( final RocksDB rocksdb, final ColumnFamilyHandle cfHandle, final PrintWriter out) throws RocksDBException, NumberFormatException { final String size = rocksdb.getProperty(cfHandle, "rocksdb.estimate-live-data-size"); + final String numberOfKeys = rocksdb.getProperty(cfHandle, "rocksdb.estimate-num-keys"); boolean emptyColumnFamily = false; - if (!size.isEmpty() && !size.isBlank()) { + if (!size.isEmpty() && !size.isBlank() && !numberOfKeys.isEmpty() && !numberOfKeys.isBlank()) { try { final long sizeLong = Long.parseLong(size); + final long numberOfKeysLong = Long.parseLong(numberOfKeys); final String totalSstFilesSize = rocksdb.getProperty(cfHandle, "rocksdb.total-sst-files-size"); final long totalSstFilesSizeLong = !totalSstFilesSize.isEmpty() && !totalSstFilesSize.isBlank() ? Long.parseLong(totalSstFilesSize) : 0; - if (sizeLong == 0) { + if (sizeLong == 0 && numberOfKeysLong == 0){ emptyColumnFamily = true; } From 7401b5947cc1d997b0da183c9102b55721de6f14 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Fri, 15 Dec 2023 15:42:21 +1000 Subject: [PATCH 08/37] spotless Signed-off-by: Gabriel Fukushima --- .../storage/RocksDbUsageHelper.java | 2 +- .../subcommands/storage/TrieLogHelper.java | 44 +++++++++---------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbUsageHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbUsageHelper.java index 33f551d17f8..4068d5b7a87 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbUsageHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbUsageHelper.java @@ -46,7 +46,7 @@ static void printUsageForColumnFamily( !totalSstFilesSize.isEmpty() && !totalSstFilesSize.isBlank() ? Long.parseLong(totalSstFilesSize) : 0; - if (sizeLong == 0 && numberOfKeysLong == 0){ + if (sizeLong == 0 && numberOfKeysLong == 0) { emptyColumnFamily = true; } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 31207a399e6..ef9c9f61906 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -18,7 +18,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; -import org.hyperledger.besu.controller.BesuController; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.chain.Blockchain; @@ -57,7 +56,8 @@ static void prune( final PrintWriter out, final DataStorageConfiguration config, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final MutableBlockchain blockchain) throws IOException { + final MutableBlockchain blockchain) + throws IOException { TrieLogHelper.validatePruneConfiguration(config); final long layersToRetain = config.getUnstable().getBonsaiTrieLogRetentionThreshold(); @@ -180,34 +180,34 @@ private static void saveTrieLogsInFile(final Map trieLogs) throw File file = new File(trieLogFile); - try (BufferedWriter bf = new BufferedWriter(new FileWriter(file, StandardCharsets.UTF_8))) { - for (Map.Entry entry : trieLogs.entrySet()) { - bf.write(Bytes.of(entry.getKey()) + ":" + Base64.toBase64String(entry.getValue())); - bf.newLine(); - } - bf.flush(); - } catch (IOException e) { - LOG.error(e.getMessage()); - throw e; + try (BufferedWriter bf = new BufferedWriter(new FileWriter(file, StandardCharsets.UTF_8))) { + for (Map.Entry entry : trieLogs.entrySet()) { + bf.write(Bytes.of(entry.getKey()) + ":" + Base64.toBase64String(entry.getValue())); + bf.newLine(); } + bf.flush(); + } catch (IOException e) { + LOG.error(e.getMessage()); + throw e; + } } private static IdentityHashMap readTrieLogsFromFile() throws IOException { File file = new File(trieLogFile); IdentityHashMap trieLogs = new IdentityHashMap<>(); - try (BufferedReader br = new BufferedReader(new FileReader(file, StandardCharsets.UTF_8))) { - String line; - while ((line = br.readLine()) != null) { - List parts = Splitter.on(':').splitToList(line); - byte[] key = Bytes.fromHexString(parts.get(0)).toArrayUnsafe(); - byte[] value = Base64.decode(parts.get(1)); - trieLogs.put(key, value); - } - } catch (IOException e) { - LOG.error(e.getMessage()); - throw e; + try (BufferedReader br = new BufferedReader(new FileReader(file, StandardCharsets.UTF_8))) { + String line; + while ((line = br.readLine()) != null) { + List parts = Splitter.on(':').splitToList(line); + byte[] key = Bytes.fromHexString(parts.get(0)).toArrayUnsafe(); + byte[] value = Base64.decode(parts.get(1)); + trieLogs.put(key, value); } + } catch (IOException e) { + LOG.error(e.getMessage()); + throw e; + } return trieLogs; } From 1b7fb727b8319c16858be23938fbdd77ff2121c4 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Fri, 15 Dec 2023 16:27:44 +1000 Subject: [PATCH 09/37] Fix classes that changed package Signed-off-by: Gabriel Fukushima --- .../besu/cli/subcommands/storage/TrieLogHelper.java | 9 ++++++++- .../besu/cli/subcommands/storage/TrieLogSubCommand.java | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index ef9c9f61906..538acfbbfec 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -19,10 +19,10 @@ import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import java.io.BufferedReader; @@ -64,6 +64,7 @@ static void prune( final long chainHeight = blockchain.getChainHeadBlockNumber(); final long lastBlockToRetainTrieLogsFor = chainHeight - layersToRetain; final Optional finalizedBlockHash = blockchain.getFinalized(); + if (finalizedBlockHash.isEmpty()) { LOG.error("No finalized block present, skipping pruning"); return; @@ -84,6 +85,8 @@ static void prune( } IdentityHashMap trieLogsToRetain; + + //TODO: maybe stop the method here if we don't find enough hashes to retain if ((long) hashesToRetain.size() == layersToRetain) { trieLogsToRetain = new IdentityHashMap<>(); // save trielogs in a flatfile as a fail-safe @@ -102,6 +105,7 @@ static void prune( } out.println("Clear trielogs..."); // clear trielogs storage + //TODO: Add a check to ensure we have trieLogsToRetain.size() == layersToRetain rootWorldStateStorage.clearTrieLog(); // get an update and insert the trielogs we retained @@ -117,6 +121,9 @@ static void prune( out.println("Prune ran successfully. Deleting file..."); deleteTrieLogFile(); } + else{ + out.println("Prune failed. Please check the logs for more details."); + } out.println("Enjoy some GBs of storage back!..."); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index 65756e30853..d55e373d144 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -19,10 +19,10 @@ import org.hyperledger.besu.cli.util.VersionProvider; import org.hyperledger.besu.controller.BesuController; -import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.storage.StorageProvider; +import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; From 11e6b057c2fe95e882ede58f0afa05c2d1346d3c Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Fri, 15 Dec 2023 16:28:13 +1000 Subject: [PATCH 10/37] spotless Signed-off-by: Gabriel Fukushima --- .../besu/cli/subcommands/storage/TrieLogHelper.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 538acfbbfec..9015d00fb8b 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -86,7 +86,7 @@ static void prune( IdentityHashMap trieLogsToRetain; - //TODO: maybe stop the method here if we don't find enough hashes to retain + // TODO: maybe stop the method here if we don't find enough hashes to retain if ((long) hashesToRetain.size() == layersToRetain) { trieLogsToRetain = new IdentityHashMap<>(); // save trielogs in a flatfile as a fail-safe @@ -105,7 +105,7 @@ static void prune( } out.println("Clear trielogs..."); // clear trielogs storage - //TODO: Add a check to ensure we have trieLogsToRetain.size() == layersToRetain + // TODO: Add a check to ensure we have trieLogsToRetain.size() == layersToRetain rootWorldStateStorage.clearTrieLog(); // get an update and insert the trielogs we retained @@ -120,9 +120,8 @@ static void prune( if (rootWorldStateStorage.streamTrieLogKeys(layersToRetain).count() == layersToRetain) { out.println("Prune ran successfully. Deleting file..."); deleteTrieLogFile(); - } - else{ - out.println("Prune failed. Please check the logs for more details."); + } else { + out.println("Prune failed. Please check the logs for more details."); } out.println("Enjoy some GBs of storage back!..."); } From f2d01e29c1307f1e0b0b081e2b9a2169fcb2be6e Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Fri, 15 Dec 2023 23:04:51 +1000 Subject: [PATCH 11/37] Code review Signed-off-by: Gabriel Fukushima --- .../besu/cli/subcommands/storage/RocksDbUsageHelper.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbUsageHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbUsageHelper.java index 4068d5b7a87..4a11abcdf02 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbUsageHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbUsageHelper.java @@ -36,16 +36,14 @@ static void printUsageForColumnFamily( final String size = rocksdb.getProperty(cfHandle, "rocksdb.estimate-live-data-size"); final String numberOfKeys = rocksdb.getProperty(cfHandle, "rocksdb.estimate-num-keys"); boolean emptyColumnFamily = false; - if (!size.isEmpty() && !size.isBlank() && !numberOfKeys.isEmpty() && !numberOfKeys.isBlank()) { + if (!size.isBlank() && !numberOfKeys.isBlank()) { try { final long sizeLong = Long.parseLong(size); final long numberOfKeysLong = Long.parseLong(numberOfKeys); final String totalSstFilesSize = rocksdb.getProperty(cfHandle, "rocksdb.total-sst-files-size"); final long totalSstFilesSizeLong = - !totalSstFilesSize.isEmpty() && !totalSstFilesSize.isBlank() - ? Long.parseLong(totalSstFilesSize) - : 0; + !totalSstFilesSize.isBlank() ? Long.parseLong(totalSstFilesSize) : 0; if (sizeLong == 0 && numberOfKeysLong == 0) { emptyColumnFamily = true; } From 04f1aaa3e5809cf0d8b155728eb2f664813665d2 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Fri, 15 Dec 2023 23:06:28 +1000 Subject: [PATCH 12/37] Only clear DB when we have the exact amount of trie logs we want in memory Signed-off-by: Gabriel Fukushima --- .../subcommands/storage/TrieLogHelper.java | 50 +++++++++++-------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 9015d00fb8b..4561f040acf 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -77,21 +77,21 @@ static void prune( } // retrieve the layersToRetains hashes from blockchain - final List hashesToRetain = new ArrayList<>(); + final List trieLogKeys = new ArrayList<>(); for (long i = chainHeight; i > lastBlockToRetainTrieLogsFor; i--) { final Optional header = blockchain.getBlockHeader(i); - header.ifPresent(blockHeader -> hashesToRetain.add(blockHeader.getHash())); + header.ifPresent(blockHeader -> trieLogKeys.add(blockHeader.getHash())); } IdentityHashMap trieLogsToRetain; // TODO: maybe stop the method here if we don't find enough hashes to retain - if ((long) hashesToRetain.size() == layersToRetain) { + if ((long) trieLogKeys.size() == layersToRetain) { trieLogsToRetain = new IdentityHashMap<>(); - // save trielogs in a flatfile as a fail-safe + // save trielogs in a flatfile in case something goes wrong out.println("Obtaining trielogs to retain..."); - hashesToRetain.forEach( + trieLogKeys.forEach( hash -> { rootWorldStateStorage .getTrieLog(hash) @@ -100,30 +100,36 @@ static void prune( out.println("Saving trielogs to retain in file..."); saveTrieLogsInFile(trieLogsToRetain); } else { - // try to read the triLogs from the flatfile + // in case something went wrong and we already pruned trielogs + // users can re-un the subcommand and we will read trielogs from file trieLogsToRetain = readTrieLogsFromFile(); } - out.println("Clear trielogs..."); - // clear trielogs storage - // TODO: Add a check to ensure we have trieLogsToRetain.size() == layersToRetain - rootWorldStateStorage.clearTrieLog(); - // get an update and insert the trielogs we retained + if (trieLogsToRetain.size() == layersToRetain) { + out.println("Clear trielogs..."); + rootWorldStateStorage.clearTrieLog(); + out.println("Restoring trielogs retained into db..."); + recreateTrieLogs(rootWorldStateStorage, trieLogsToRetain); + } + if (rootWorldStateStorage.streamTrieLogKeys(layersToRetain).count() == layersToRetain) { + out.println("Prune ran successfully. Deleting file..."); + deleteTrieLogFile(); + out.println("Enjoy some GBs of storage back!..."); + } else { + out.println("Prune failed. Re-run the subcommand to load the trielogs from file."); + } + } + + private static void recreateTrieLogs( + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final IdentityHashMap trieLogsToRetain) { var updater = rootWorldStateStorage.updater(); - out.println("restore trielogs retained into db..."); + trieLogsToRetain.forEach( (key, value) -> { updater.getTrieLogStorageTransaction().put(key, value); }); updater.getTrieLogStorageTransaction().commit(); - - if (rootWorldStateStorage.streamTrieLogKeys(layersToRetain).count() == layersToRetain) { - out.println("Prune ran successfully. Deleting file..."); - deleteTrieLogFile(); - } else { - out.println("Prune failed. Please check the logs for more details."); - } - out.println("Enjoy some GBs of storage back!..."); } private static void validatePruneConfiguration(final DataStorageConfiguration config) { @@ -220,7 +226,9 @@ private static IdentityHashMap readTrieLogsFromFile() throws IOE private static void deleteTrieLogFile() { File file = new File(trieLogFile); - file.delete(); + if (file.exists()) { + file.delete(); + } } static void printCount(final PrintWriter out, final TrieLogCount count) { From 2f01c5a4d03e231f0f018b06d661b60683d2cfd7 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Mon, 18 Dec 2023 13:02:36 +1000 Subject: [PATCH 13/37] Trielogs stream to and from file to avoid possibly OOM Signed-off-by: Gabriel Fukushima --- .../subcommands/storage/TrieLogHelper.java | 88 +++++++++++-------- .../storage/TrieLogSubCommand.java | 14 ++- 2 files changed, 56 insertions(+), 46 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 4561f040acf..a306529a85e 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -25,25 +25,20 @@ import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.io.PrintWriter; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.IdentityHashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; -import com.google.common.base.Splitter; -import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.bouncycastle.util.encoders.Base64; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,13 +51,17 @@ static void prune( final PrintWriter out, final DataStorageConfiguration config, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final MutableBlockchain blockchain) - throws IOException { + final MutableBlockchain blockchain) { TrieLogHelper.validatePruneConfiguration(config); final long layersToRetain = config.getUnstable().getBonsaiTrieLogRetentionThreshold(); final long chainHeight = blockchain.getChainHeadBlockNumber(); - final long lastBlockToRetainTrieLogsFor = chainHeight - layersToRetain; + final long lastBlockNumberToRetainTrieLogsFor = chainHeight - layersToRetain; + if (lastBlockNumberToRetainTrieLogsFor < 0) { + LOG.error( + "Trying to retain more trie logs than chain height ({}), skipping pruning", chainHeight); + return; + } final Optional finalizedBlockHash = blockchain.getFinalized(); if (finalizedBlockHash.isEmpty()) { @@ -70,7 +69,7 @@ static void prune( return; } else { if (blockchain.getBlockHeader(finalizedBlockHash.get()).get().getNumber() - < lastBlockToRetainTrieLogsFor) { + < lastBlockNumberToRetainTrieLogsFor) { LOG.error("Trying to prune more layers than the finalized block height, skipping pruning"); return; } @@ -79,9 +78,11 @@ static void prune( // retrieve the layersToRetains hashes from blockchain final List trieLogKeys = new ArrayList<>(); - for (long i = chainHeight; i > lastBlockToRetainTrieLogsFor; i--) { + for (long i = chainHeight; i > lastBlockNumberToRetainTrieLogsFor; i--) { final Optional header = blockchain.getBlockHeader(i); - header.ifPresent(blockHeader -> trieLogKeys.add(blockHeader.getHash())); + header.ifPresentOrElse( + blockHeader -> trieLogKeys.add(blockHeader.getHash()), + () -> LOG.error("Error retrieving block")); } IdentityHashMap trieLogsToRetain; @@ -98,11 +99,21 @@ static void prune( .ifPresent(trieLog -> trieLogsToRetain.put(hash.toArrayUnsafe(), trieLog)); }); out.println("Saving trielogs to retain in file..."); - saveTrieLogsInFile(trieLogsToRetain); + try { + saveTrieLogsInFile(trieLogsToRetain); + } catch (IOException e) { + LOG.error("Error saving trielogs to file: {}", e.getMessage()); + return; + } } else { // in case something went wrong and we already pruned trielogs // users can re-un the subcommand and we will read trielogs from file - trieLogsToRetain = readTrieLogsFromFile(); + try { + trieLogsToRetain = readTrieLogsFromFile(); + } catch (Exception e) { + LOG.error("Error reading trielogs from file: {}", e.getMessage()); + return; + } } if (trieLogsToRetain.size() == layersToRetain) { @@ -114,7 +125,7 @@ static void prune( if (rootWorldStateStorage.streamTrieLogKeys(layersToRetain).count() == layersToRetain) { out.println("Prune ran successfully. Deleting file..."); deleteTrieLogFile(); - out.println("Enjoy some GBs of storage back!..."); + out.println("Enjoy some GBs of storage back!"); } else { out.println("Prune failed. Re-run the subcommand to load the trielogs from file."); } @@ -188,35 +199,38 @@ static TrieLogCount getCount( return new TrieLogCount(total.get(), canonicalCount.get(), forkCount.get(), orphanCount.get()); } - private static void saveTrieLogsInFile(final Map trieLogs) throws IOException { + private static void saveTrieLogsInFile(final IdentityHashMap trieLogs) + throws IOException { File file = new File(trieLogFile); - - try (BufferedWriter bf = new BufferedWriter(new FileWriter(file, StandardCharsets.UTF_8))) { - for (Map.Entry entry : trieLogs.entrySet()) { - bf.write(Bytes.of(entry.getKey()) + ":" + Base64.toBase64String(entry.getValue())); - bf.newLine(); + if (file.exists()) { + LOG.error("File {} already exists, something went terribly wrong", trieLogFile); + } + try (FileOutputStream fos = new FileOutputStream(trieLogFile)) { + try { + ObjectOutputStream oos = new ObjectOutputStream(fos); + oos.writeObject(trieLogs); + } catch (IOException ex) { + throw new RuntimeException(ex); } - bf.flush(); } catch (IOException e) { LOG.error(e.getMessage()); throw e; } } - private static IdentityHashMap readTrieLogsFromFile() throws IOException { + @SuppressWarnings("unchecked") + private static IdentityHashMap readTrieLogsFromFile() + throws IOException, ClassNotFoundException { + + IdentityHashMap trieLogs; + try (FileInputStream fis = new FileInputStream(trieLogFile); + ObjectInputStream ois = new ObjectInputStream(fis)) { + + trieLogs = (IdentityHashMap) ois.readObject(); + + } catch (IOException | ClassNotFoundException e) { - File file = new File(trieLogFile); - IdentityHashMap trieLogs = new IdentityHashMap<>(); - try (BufferedReader br = new BufferedReader(new FileReader(file, StandardCharsets.UTF_8))) { - String line; - while ((line = br.readLine()) != null) { - List parts = Splitter.on(':').splitToList(line); - byte[] key = Bytes.fromHexString(parts.get(0)).toArrayUnsafe(); - byte[] value = Base64.decode(parts.get(1)); - trieLogs.put(key, value); - } - } catch (IOException e) { LOG.error(e.getMessage()); throw e; } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index d55e373d144..391748c9b04 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -111,15 +111,11 @@ static class PruneTrieLog implements Runnable { public void run() { TrieLogContext context = getTrieLogContext(); - try { - TrieLogHelper.prune( - spec.commandLine().getOut(), - context.config(), - context.rootWorldStateStorage(), - context.blockchain()); - } catch (Exception e) { - throw new RuntimeException(e); - } + TrieLogHelper.prune( + spec.commandLine().getOut(), + context.config(), + context.rootWorldStateStorage(), + context.blockchain()); } } From 56e4c8e873367084194e828622b89ed749a4a9e3 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Mon, 18 Dec 2023 16:46:19 +1000 Subject: [PATCH 14/37] Process trie logs in chunks to avoid OOM Signed-off-by: Gabriel Fukushima --- .../subcommands/storage/TrieLogHelper.java | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index a306529a85e..10ba0563c9b 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -122,6 +122,7 @@ static void prune( out.println("Restoring trielogs retained into db..."); recreateTrieLogs(rootWorldStateStorage, trieLogsToRetain); } + if (rootWorldStateStorage.streamTrieLogKeys(layersToRetain).count() == layersToRetain) { out.println("Prune ran successfully. Deleting file..."); deleteTrieLogFile(); @@ -134,12 +135,32 @@ static void prune( private static void recreateTrieLogs( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final IdentityHashMap trieLogsToRetain) { + // process in chunk to avoid OOM + final int chunkSize = 1000; + List keys = new ArrayList<>(trieLogsToRetain.keySet()); + + for (int startIndex = 0; startIndex < keys.size(); startIndex += chunkSize) { + processChunk(startIndex, chunkSize, keys, trieLogsToRetain, rootWorldStateStorage); + } + } + + private static void processChunk( + final int startIndex, + final int chunkSize, + final List keys, + final IdentityHashMap trieLogsToRetain, + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) { + var updater = rootWorldStateStorage.updater(); + int endIndex = Math.min(startIndex + chunkSize, keys.size()); + + for (int i = startIndex; i < endIndex; i++) { + byte[] key = keys.get(i); + byte[] value = trieLogsToRetain.get(key); + updater.getTrieLogStorageTransaction().put(key, value); + LOG.info("Key({}): {}", i, Bytes32.wrap(key)); + } - trieLogsToRetain.forEach( - (key, value) -> { - updater.getTrieLogStorageTransaction().put(key, value); - }); updater.getTrieLogStorageTransaction().commit(); } From 78561b0efd420dd39cbbe799c2448dd82807ddff Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Tue, 19 Dec 2023 17:34:31 +1000 Subject: [PATCH 15/37] save and read in batches to handle edge cases Signed-off-by: Gabriel Fukushima --- .../subcommands/storage/TrieLogHelper.java | 225 ++++++++++++------ .../storage/TrieLogSubCommand.java | 6 +- 2 files changed, 156 insertions(+), 75 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 10ba0563c9b..9b44b5a633a 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -44,107 +44,175 @@ /** Helper class for counting and pruning trie logs */ public class TrieLogHelper { - private static final String trieLogFile = "trieLogsToRetain.txt"; + private static final String TRIE_LOG_FILE = "trieLogsToRetain"; + private static final long BATCH_SIZE = 20000; + private static final int ROCKSDB_MAX_INSERTS_PER_TRANSACTION = 1000; private static final Logger LOG = LoggerFactory.getLogger(TrieLogHelper.class); static void prune( - final PrintWriter out, final DataStorageConfiguration config, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final MutableBlockchain blockchain) { TrieLogHelper.validatePruneConfiguration(config); + final long layersToRetain = config.getUnstable().getBonsaiTrieLogRetentionThreshold(); + final long chainHeight = blockchain.getChainHeadBlockNumber(); + final long lastBlockNumberToRetainTrieLogsFor = chainHeight - layersToRetain; - if (lastBlockNumberToRetainTrieLogsFor < 0) { - LOG.error( - "Trying to retain more trie logs than chain height ({}), skipping pruning", chainHeight); + + if (!validPruneRequirements(blockchain, chainHeight, lastBlockNumberToRetainTrieLogsFor)) { return; } - final Optional finalizedBlockHash = blockchain.getFinalized(); - if (finalizedBlockHash.isEmpty()) { - LOG.error("No finalized block present, skipping pruning"); - return; + final long numberOfBatches = calculateNumberofBatches(layersToRetain); + + processTrieLogBatches( + rootWorldStateStorage, + blockchain, + chainHeight, + lastBlockNumberToRetainTrieLogsFor, + numberOfBatches); + + if (rootWorldStateStorage.streamTrieLogKeys(layersToRetain).count() == layersToRetain) { + deleteFiles(numberOfBatches); } else { - if (blockchain.getBlockHeader(finalizedBlockHash.get()).get().getNumber() - < lastBlockNumberToRetainTrieLogsFor) { - LOG.error("Trying to prune more layers than the finalized block height, skipping pruning"); - return; - } + LOG.error("Prune failed. Re-run the subcommand to load the trie logs from file."); + } + } + + private static void processTrieLogBatches( + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final MutableBlockchain blockchain, + final long chainHeight, + final long lastBlockNumberToRetainTrieLogsFor, + final long numberOfBatches) { + + for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { + saveTrieLogBatches( + rootWorldStateStorage, + blockchain, + chainHeight, + lastBlockNumberToRetainTrieLogsFor, + batchNumber); } - // retrieve the layersToRetains hashes from blockchain + LOG.info("Clear trie logs..."); + rootWorldStateStorage.clearTrieLog(); + + for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { + restoreTrieLogBatches(rootWorldStateStorage, batchNumber); + } + } + + private static void saveTrieLogBatches( + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final MutableBlockchain blockchain, + final long chainHeight, + final long lastBlockNumberToRetainTrieLogsFor, + final long batchNumber) { + + final long firstBlockOfBatch = chainHeight - ((batchNumber - 1) * BATCH_SIZE); + + final long lastBlockOfBatch = + Math.max(chainHeight - (batchNumber * BATCH_SIZE), lastBlockNumberToRetainTrieLogsFor); + final List trieLogKeys = new ArrayList<>(); - for (long i = chainHeight; i > lastBlockNumberToRetainTrieLogsFor; i--) { + getTrieLogKeysForBlocks(blockchain, firstBlockOfBatch, lastBlockOfBatch, trieLogKeys); + + LOG.info("Saving trie logs to retain in file (batch {})...", batchNumber); + + try { + saveTrieLogsInFile(trieLogKeys, rootWorldStateStorage, batchNumber); + } catch (IOException e) { + LOG.error("Error saving trie logs to file: {}", e.getMessage()); + } + } + + private static void restoreTrieLogBatches( + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final long batchNumber) { + + try { + LOG.info("Restoring trie logs retained from batch {}...", batchNumber); + recreateTrieLogs(rootWorldStateStorage, batchNumber); + } catch (IOException | ClassNotFoundException e) { + LOG.error("Error recreating trie logs from batch {}: {}", batchNumber, e.getMessage()); + } + } + + private static void deleteFiles(final long numberOfBatches) { + + LOG.info("Deleting files..."); + + for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { + + deleteTrieLogFile(batchNumber); + } + + LOG.info("Prune ran successfully. Enjoy some disk space back! \uD83D\uDE80"); + } + + private static void getTrieLogKeysForBlocks( + final MutableBlockchain blockchain, + final long firstBlockOfBatch, + final long lastBlockOfBatch, + final List trieLogKeys) { + for (long i = firstBlockOfBatch; i > lastBlockOfBatch; i--) { final Optional header = blockchain.getBlockHeader(i); header.ifPresentOrElse( blockHeader -> trieLogKeys.add(blockHeader.getHash()), () -> LOG.error("Error retrieving block")); } + } - IdentityHashMap trieLogsToRetain; + private static long calculateNumberofBatches(final long layersToRetain) { + return layersToRetain / BATCH_SIZE + ((layersToRetain % BATCH_SIZE == 0) ? 0 : 1); + } - // TODO: maybe stop the method here if we don't find enough hashes to retain - if ((long) trieLogKeys.size() == layersToRetain) { - trieLogsToRetain = new IdentityHashMap<>(); - // save trielogs in a flatfile in case something goes wrong - out.println("Obtaining trielogs to retain..."); - trieLogKeys.forEach( - hash -> { - rootWorldStateStorage - .getTrieLog(hash) - .ifPresent(trieLog -> trieLogsToRetain.put(hash.toArrayUnsafe(), trieLog)); - }); - out.println("Saving trielogs to retain in file..."); - try { - saveTrieLogsInFile(trieLogsToRetain); - } catch (IOException e) { - LOG.error("Error saving trielogs to file: {}", e.getMessage()); - return; - } - } else { - // in case something went wrong and we already pruned trielogs - // users can re-un the subcommand and we will read trielogs from file - try { - trieLogsToRetain = readTrieLogsFromFile(); - } catch (Exception e) { - LOG.error("Error reading trielogs from file: {}", e.getMessage()); - return; - } + private static boolean validPruneRequirements( + final MutableBlockchain blockchain, + final long chainHeight, + final long lastBlockNumberToRetainTrieLogsFor) { + if (lastBlockNumberToRetainTrieLogsFor < 0) { + LOG.error( + "Trying to retain more trie logs than chain height ({}), skipping pruning", chainHeight); + return false; } - if (trieLogsToRetain.size() == layersToRetain) { - out.println("Clear trielogs..."); - rootWorldStateStorage.clearTrieLog(); - out.println("Restoring trielogs retained into db..."); - recreateTrieLogs(rootWorldStateStorage, trieLogsToRetain); - } + final Optional finalizedBlockHash = blockchain.getFinalized(); - if (rootWorldStateStorage.streamTrieLogKeys(layersToRetain).count() == layersToRetain) { - out.println("Prune ran successfully. Deleting file..."); - deleteTrieLogFile(); - out.println("Enjoy some GBs of storage back!"); + if (finalizedBlockHash.isEmpty()) { + LOG.error("No finalized block present, skipping pruning"); + return false; } else { - out.println("Prune failed. Re-run the subcommand to load the trielogs from file."); + final Hash finalizedHash = finalizedBlockHash.get(); + if (blockchain.getBlockHeader(finalizedHash).isPresent() + && blockchain.getBlockHeader(finalizedHash).get().getNumber() + < lastBlockNumberToRetainTrieLogsFor) { + LOG.error("Trying to prune more layers than the finalized block height, skipping pruning"); + return false; + } } + return true; } private static void recreateTrieLogs( - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final IdentityHashMap trieLogsToRetain) { + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final long batchNumber) + throws IOException, ClassNotFoundException { // process in chunk to avoid OOM - final int chunkSize = 1000; + + IdentityHashMap trieLogsToRetain = readTrieLogsFromFile(batchNumber); + final int chunkSize = ROCKSDB_MAX_INSERTS_PER_TRANSACTION; List keys = new ArrayList<>(trieLogsToRetain.keySet()); for (int startIndex = 0; startIndex < keys.size(); startIndex += chunkSize) { - processChunk(startIndex, chunkSize, keys, trieLogsToRetain, rootWorldStateStorage); + processTransactionChunk(startIndex, chunkSize, keys, trieLogsToRetain, rootWorldStateStorage); } } - private static void processChunk( + private static void processTransactionChunk( final int startIndex, final int chunkSize, final List keys, @@ -158,7 +226,7 @@ private static void processChunk( byte[] key = keys.get(i); byte[] value = trieLogsToRetain.get(key); updater.getTrieLogStorageTransaction().put(key, value); - LOG.info("Key({}): {}", i, Bytes32.wrap(key)); + LOG.info("Key({}): {}", i, Bytes32.wrap(key).toShortHexString()); } updater.getTrieLogStorageTransaction().commit(); @@ -220,17 +288,21 @@ static TrieLogCount getCount( return new TrieLogCount(total.get(), canonicalCount.get(), forkCount.get(), orphanCount.get()); } - private static void saveTrieLogsInFile(final IdentityHashMap trieLogs) + private static void saveTrieLogsInFile( + final List trieLogs, + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final long batchNumber) throws IOException { - File file = new File(trieLogFile); + File file = new File(TRIE_LOG_FILE + "-" + batchNumber); if (file.exists()) { - LOG.error("File {} already exists, something went terribly wrong", trieLogFile); + LOG.error("File already exists, skipping file creation"); + return; } - try (FileOutputStream fos = new FileOutputStream(trieLogFile)) { + try (FileOutputStream fos = new FileOutputStream(file)) { try { ObjectOutputStream oos = new ObjectOutputStream(fos); - oos.writeObject(trieLogs); + oos.writeObject(getTrieLogs(trieLogs, rootWorldStateStorage)); } catch (IOException ex) { throw new RuntimeException(ex); } @@ -241,11 +313,11 @@ private static void saveTrieLogsInFile(final IdentityHashMap tri } @SuppressWarnings("unchecked") - private static IdentityHashMap readTrieLogsFromFile() + private static IdentityHashMap readTrieLogsFromFile(final long batchNumber) throws IOException, ClassNotFoundException { IdentityHashMap trieLogs; - try (FileInputStream fis = new FileInputStream(trieLogFile); + try (FileInputStream fis = new FileInputStream(TRIE_LOG_FILE + "-" + batchNumber); ObjectInputStream ois = new ObjectInputStream(fis)) { trieLogs = (IdentityHashMap) ois.readObject(); @@ -259,8 +331,21 @@ private static IdentityHashMap readTrieLogsFromFile() return trieLogs; } - private static void deleteTrieLogFile() { - File file = new File(trieLogFile); + private static IdentityHashMap getTrieLogs( + final List trieLogKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) { + IdentityHashMap trieLogsToRetain = new IdentityHashMap<>(); + + LOG.info("Obtaining trielogs from db, this may take a few minutes..."); + trieLogKeys.forEach( + hash -> + rootWorldStateStorage + .getTrieLog(hash) + .ifPresent(trieLog -> trieLogsToRetain.put(hash.toArrayUnsafe(), trieLog))); + return trieLogsToRetain; + } + + private static void deleteTrieLogFile(final long batchNumber) { + File file = new File(TRIE_LOG_FILE + "-" + batchNumber); if (file.exists()) { file.delete(); } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index 391748c9b04..acbdee39f1a 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -111,11 +111,7 @@ static class PruneTrieLog implements Runnable { public void run() { TrieLogContext context = getTrieLogContext(); - TrieLogHelper.prune( - spec.commandLine().getOut(), - context.config(), - context.rootWorldStateStorage(), - context.blockchain()); + TrieLogHelper.prune(context.config(), context.rootWorldStateStorage(), context.blockchain()); } } From 42c72cf39119a376f6aa09b9ffcf9f274bf8f0e3 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Wed, 20 Dec 2023 14:42:24 +1000 Subject: [PATCH 16/37] save and read files to/from database dir Signed-off-by: Gabriel Fukushima --- .../subcommands/storage/TrieLogHelper.java | 68 +++++++++++-------- .../storage/TrieLogSubCommand.java | 12 +++- 2 files changed, 48 insertions(+), 32 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 9b44b5a633a..52e16f2ec31 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -16,6 +16,7 @@ package org.hyperledger.besu.cli.subcommands.storage; import static com.google.common.base.Preconditions.checkArgument; +import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; import org.hyperledger.besu.datatypes.Hash; @@ -32,6 +33,7 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.PrintWriter; +import java.nio.file.Path; import java.util.ArrayList; import java.util.IdentityHashMap; import java.util.List; @@ -45,15 +47,16 @@ /** Helper class for counting and pruning trie logs */ public class TrieLogHelper { private static final String TRIE_LOG_FILE = "trieLogsToRetain"; - private static final long BATCH_SIZE = 20000; + private static final long BATCH_SIZE = 20_000; private static final int ROCKSDB_MAX_INSERTS_PER_TRANSACTION = 1000; private static final Logger LOG = LoggerFactory.getLogger(TrieLogHelper.class); static void prune( final DataStorageConfiguration config, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final MutableBlockchain blockchain) { - + final MutableBlockchain blockchain, + final Path dataDirectoryPath) { + final String batchFileNameBase = dataDirectoryPath.resolve(DATABASE_PATH) + "/" + TRIE_LOG_FILE; TrieLogHelper.validatePruneConfiguration(config); final long layersToRetain = config.getUnstable().getBonsaiTrieLogRetentionThreshold(); @@ -73,10 +76,12 @@ static void prune( blockchain, chainHeight, lastBlockNumberToRetainTrieLogsFor, - numberOfBatches); + numberOfBatches, + batchFileNameBase); if (rootWorldStateStorage.streamTrieLogKeys(layersToRetain).count() == layersToRetain) { - deleteFiles(numberOfBatches); + deleteFiles(batchFileNameBase, numberOfBatches); + LOG.info("Prune ran successfully. Enjoy some disk space back! \uD83D\uDE80"); } else { LOG.error("Prune failed. Re-run the subcommand to load the trie logs from file."); } @@ -87,10 +92,12 @@ private static void processTrieLogBatches( final MutableBlockchain blockchain, final long chainHeight, final long lastBlockNumberToRetainTrieLogsFor, - final long numberOfBatches) { + final long numberOfBatches, + final String batchFileNameBase) { for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { saveTrieLogBatches( + batchFileNameBase, rootWorldStateStorage, blockchain, chainHeight, @@ -102,11 +109,12 @@ private static void processTrieLogBatches( rootWorldStateStorage.clearTrieLog(); for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { - restoreTrieLogBatches(rootWorldStateStorage, batchNumber); + restoreTrieLogBatches(rootWorldStateStorage, batchNumber, batchFileNameBase); } } private static void saveTrieLogBatches( + final String batchFileNameBase, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final MutableBlockchain blockchain, final long chainHeight, @@ -125,33 +133,35 @@ private static void saveTrieLogBatches( LOG.info("Saving trie logs to retain in file (batch {})...", batchNumber); try { - saveTrieLogsInFile(trieLogKeys, rootWorldStateStorage, batchNumber); + saveTrieLogsInFile(trieLogKeys, rootWorldStateStorage, batchNumber, batchFileNameBase); } catch (IOException e) { LOG.error("Error saving trie logs to file: {}", e.getMessage()); } } private static void restoreTrieLogBatches( - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final long batchNumber) { + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final long batchNumber, + final String batchFileNameBase) { try { LOG.info("Restoring trie logs retained from batch {}...", batchNumber); - recreateTrieLogs(rootWorldStateStorage, batchNumber); + recreateTrieLogs(rootWorldStateStorage, batchNumber, batchFileNameBase); } catch (IOException | ClassNotFoundException e) { LOG.error("Error recreating trie logs from batch {}: {}", batchNumber, e.getMessage()); } } - private static void deleteFiles(final long numberOfBatches) { + private static void deleteFiles(final String batchFileNameBase, final long numberOfBatches) { LOG.info("Deleting files..."); for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { - - deleteTrieLogFile(batchNumber); + File file = new File(batchFileNameBase + "-" + batchNumber); + if (file.exists()) { + file.delete(); + } } - - LOG.info("Prune ran successfully. Enjoy some disk space back! \uD83D\uDE80"); } private static void getTrieLogKeysForBlocks( @@ -199,11 +209,14 @@ private static boolean validPruneRequirements( } private static void recreateTrieLogs( - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final long batchNumber) + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final long batchNumber, + final String batchFileNameBase) throws IOException, ClassNotFoundException { // process in chunk to avoid OOM - IdentityHashMap trieLogsToRetain = readTrieLogsFromFile(batchNumber); + IdentityHashMap trieLogsToRetain = + readTrieLogsFromFile(batchFileNameBase, batchNumber); final int chunkSize = ROCKSDB_MAX_INSERTS_PER_TRANSACTION; List keys = new ArrayList<>(trieLogsToRetain.keySet()); @@ -289,12 +302,13 @@ static TrieLogCount getCount( } private static void saveTrieLogsInFile( - final List trieLogs, + final List trieLogsKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final long batchNumber) + final long batchNumber, + final String batchFileNameBase) throws IOException { - File file = new File(TRIE_LOG_FILE + "-" + batchNumber); + File file = new File(batchFileNameBase + "-" + batchNumber); if (file.exists()) { LOG.error("File already exists, skipping file creation"); return; @@ -302,7 +316,7 @@ private static void saveTrieLogsInFile( try (FileOutputStream fos = new FileOutputStream(file)) { try { ObjectOutputStream oos = new ObjectOutputStream(fos); - oos.writeObject(getTrieLogs(trieLogs, rootWorldStateStorage)); + oos.writeObject(getTrieLogs(trieLogsKeys, rootWorldStateStorage)); } catch (IOException ex) { throw new RuntimeException(ex); } @@ -313,11 +327,12 @@ private static void saveTrieLogsInFile( } @SuppressWarnings("unchecked") - private static IdentityHashMap readTrieLogsFromFile(final long batchNumber) + private static IdentityHashMap readTrieLogsFromFile( + final String batchFileNameBase, final long batchNumber) throws IOException, ClassNotFoundException { IdentityHashMap trieLogs; - try (FileInputStream fis = new FileInputStream(TRIE_LOG_FILE + "-" + batchNumber); + try (FileInputStream fis = new FileInputStream(batchFileNameBase + "-" + batchNumber); ObjectInputStream ois = new ObjectInputStream(fis)) { trieLogs = (IdentityHashMap) ois.readObject(); @@ -344,13 +359,6 @@ private static IdentityHashMap getTrieLogs( return trieLogsToRetain; } - private static void deleteTrieLogFile(final long batchNumber) { - File file = new File(TRIE_LOG_FILE + "-" + batchNumber); - if (file.exists()) { - file.delete(); - } - } - static void printCount(final PrintWriter out, final TrieLogCount count) { out.printf( "trieLog count: %s\n - canonical count: %s\n - fork count: %s\n - orphaned count: %s\n", diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index acbdee39f1a..bf75fd6eb9a 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -27,6 +27,8 @@ import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import java.io.PrintWriter; +import java.nio.file.Path; +import java.nio.file.Paths; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; @@ -110,8 +112,14 @@ static class PruneTrieLog implements Runnable { @Override public void run() { TrieLogContext context = getTrieLogContext(); - - TrieLogHelper.prune(context.config(), context.rootWorldStateStorage(), context.blockchain()); + final Path dataDirectoryPath = + Paths.get( + TrieLogSubCommand.parentCommand.parentCommand.dataDir().toAbsolutePath().toString()); + TrieLogHelper.prune( + context.config(), + context.rootWorldStateStorage(), + context.blockchain(), + dataDirectoryPath); } } From 9389540335b6f2aa3493d36e4bf23d2adaedb9b2 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Thu, 21 Dec 2023 15:20:40 +1000 Subject: [PATCH 17/37] add unit tests and PR review fixes Signed-off-by: Gabriel Fukushima --- .../subcommands/storage/TrieLogHelper.java | 160 ++++++------ .../storage/TrieLogHelperTest.java | 239 ++++++++++++++++++ 2 files changed, 314 insertions(+), 85 deletions(-) create mode 100644 besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 52e16f2ec31..0878a17024a 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -17,7 +17,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH; -import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; @@ -63,7 +62,7 @@ static void prune( final long chainHeight = blockchain.getChainHeadBlockNumber(); - final long lastBlockNumberToRetainTrieLogsFor = chainHeight - layersToRetain; + final long lastBlockNumberToRetainTrieLogsFor = chainHeight - layersToRetain + 1; if (!validPruneRequirements(blockchain, chainHeight, lastBlockNumberToRetainTrieLogsFor)) { return; @@ -96,13 +95,16 @@ private static void processTrieLogBatches( final String batchFileNameBase) { for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { - saveTrieLogBatches( - batchFileNameBase, - rootWorldStateStorage, - blockchain, - chainHeight, - lastBlockNumberToRetainTrieLogsFor, - batchNumber); + + final long firstBlockOfBatch = chainHeight - ((batchNumber - 1) * BATCH_SIZE); + + final long lastBlockOfBatch = + Math.max(chainHeight - (batchNumber * BATCH_SIZE), lastBlockNumberToRetainTrieLogsFor); + + final List trieLogKeys = + getTrieLogKeysForBlocks(blockchain, firstBlockOfBatch, lastBlockOfBatch); + + saveTrieLogBatches(batchFileNameBase, rootWorldStateStorage, batchNumber, trieLogKeys); } LOG.info("Clear trie logs..."); @@ -116,19 +118,8 @@ private static void processTrieLogBatches( private static void saveTrieLogBatches( final String batchFileNameBase, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final MutableBlockchain blockchain, - final long chainHeight, - final long lastBlockNumberToRetainTrieLogsFor, - final long batchNumber) { - - final long firstBlockOfBatch = chainHeight - ((batchNumber - 1) * BATCH_SIZE); - - final long lastBlockOfBatch = - Math.max(chainHeight - (batchNumber * BATCH_SIZE), lastBlockNumberToRetainTrieLogsFor); - - final List trieLogKeys = new ArrayList<>(); - - getTrieLogKeysForBlocks(blockchain, firstBlockOfBatch, lastBlockOfBatch, trieLogKeys); + final long batchNumber, + final List trieLogKeys) { LOG.info("Saving trie logs to retain in file (batch {})...", batchNumber); @@ -136,6 +127,7 @@ private static void saveTrieLogBatches( saveTrieLogsInFile(trieLogKeys, rootWorldStateStorage, batchNumber, batchFileNameBase); } catch (IOException e) { LOG.error("Error saving trie logs to file: {}", e.getMessage()); + throw new RuntimeException(e); } } @@ -147,8 +139,9 @@ private static void restoreTrieLogBatches( try { LOG.info("Restoring trie logs retained from batch {}...", batchNumber); recreateTrieLogs(rootWorldStateStorage, batchNumber, batchFileNameBase); - } catch (IOException | ClassNotFoundException e) { + } catch (IOException e) { LOG.error("Error recreating trie logs from batch {}: {}", batchNumber, e.getMessage()); + throw new RuntimeException(e); } } @@ -164,17 +157,18 @@ private static void deleteFiles(final String batchFileNameBase, final long numbe } } - private static void getTrieLogKeysForBlocks( + private static List getTrieLogKeysForBlocks( final MutableBlockchain blockchain, final long firstBlockOfBatch, - final long lastBlockOfBatch, - final List trieLogKeys) { - for (long i = firstBlockOfBatch; i > lastBlockOfBatch; i--) { + final long lastBlockOfBatch) { + final List trieLogKeys = new ArrayList<>(); + for (long i = firstBlockOfBatch; i >= lastBlockOfBatch; i--) { final Optional header = blockchain.getBlockHeader(i); header.ifPresentOrElse( blockHeader -> trieLogKeys.add(blockHeader.getHash()), () -> LOG.error("Error retrieving block")); } + return trieLogKeys; } private static long calculateNumberofBatches(final long layersToRetain) { @@ -186,23 +180,23 @@ private static boolean validPruneRequirements( final long chainHeight, final long lastBlockNumberToRetainTrieLogsFor) { if (lastBlockNumberToRetainTrieLogsFor < 0) { - LOG.error( - "Trying to retain more trie logs than chain height ({}), skipping pruning", chainHeight); - return false; + throw new IllegalArgumentException( + "Trying to retain more trie logs than chain length (" + + chainHeight + + "), skipping pruning"); } final Optional finalizedBlockHash = blockchain.getFinalized(); if (finalizedBlockHash.isEmpty()) { - LOG.error("No finalized block present, skipping pruning"); - return false; + throw new RuntimeException("No finalized block present, can't safely run trie log prune"); } else { final Hash finalizedHash = finalizedBlockHash.get(); - if (blockchain.getBlockHeader(finalizedHash).isPresent() - && blockchain.getBlockHeader(finalizedHash).get().getNumber() - < lastBlockNumberToRetainTrieLogsFor) { - LOG.error("Trying to prune more layers than the finalized block height, skipping pruning"); - return false; + final Optional finalizedBlockHeader = blockchain.getBlockHeader(finalizedHash); + if (finalizedBlockHeader.isPresent() + && finalizedBlockHeader.get().getNumber() < lastBlockNumberToRetainTrieLogsFor) { + throw new IllegalArgumentException( + "Trying to prune more layers than the finalized block height, skipping pruning"); } } return true; @@ -212,7 +206,7 @@ private static void recreateTrieLogs( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final long batchNumber, final String batchFileNameBase) - throws IOException, ClassNotFoundException { + throws IOException { // process in chunk to avoid OOM IdentityHashMap trieLogsToRetain = @@ -248,10 +242,10 @@ private static void processTransactionChunk( private static void validatePruneConfiguration(final DataStorageConfiguration config) { checkArgument( config.getUnstable().getBonsaiTrieLogRetentionThreshold() - >= MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD, + >= config.getBonsaiMaxLayersToLoad(), String.format( "--Xbonsai-trie-log-retention-threshold minimum value is %d", - MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD)); + config.getBonsaiMaxLayersToLoad())); checkArgument( config.getUnstable().getBonsaiTrieLogPruningLimit() > 0, String.format( @@ -266,41 +260,6 @@ private static void validatePruneConfiguration(final DataStorageConfiguration co config.getUnstable().getBonsaiTrieLogRetentionThreshold())); } - static TrieLogCount getCount( - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final int limit, - final Blockchain blockchain) { - final AtomicInteger total = new AtomicInteger(); - final AtomicInteger canonicalCount = new AtomicInteger(); - final AtomicInteger forkCount = new AtomicInteger(); - final AtomicInteger orphanCount = new AtomicInteger(); - rootWorldStateStorage - .streamTrieLogKeys(limit) - .map(Bytes32::wrap) - .map(Hash::wrap) - .forEach( - hash -> { - total.getAndIncrement(); - blockchain - .getBlockHeader(hash) - .ifPresentOrElse( - (header) -> { - long number = header.getNumber(); - final Optional headerByNumber = - blockchain.getBlockHeader(number); - if (headerByNumber.isPresent() - && headerByNumber.get().getHash().equals(hash)) { - canonicalCount.getAndIncrement(); - } else { - forkCount.getAndIncrement(); - } - }, - orphanCount::getAndIncrement); - }); - - return new TrieLogCount(total.get(), canonicalCount.get(), forkCount.get(), orphanCount.get()); - } - private static void saveTrieLogsInFile( final List trieLogsKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, @@ -313,23 +272,19 @@ private static void saveTrieLogsInFile( LOG.error("File already exists, skipping file creation"); return; } + try (FileOutputStream fos = new FileOutputStream(file)) { - try { - ObjectOutputStream oos = new ObjectOutputStream(fos); - oos.writeObject(getTrieLogs(trieLogsKeys, rootWorldStateStorage)); - } catch (IOException ex) { - throw new RuntimeException(ex); - } + ObjectOutputStream oos = new ObjectOutputStream(fos); + oos.writeObject(getTrieLogs(trieLogsKeys, rootWorldStateStorage)); } catch (IOException e) { LOG.error(e.getMessage()); - throw e; + throw new RuntimeException(e); } } @SuppressWarnings("unchecked") private static IdentityHashMap readTrieLogsFromFile( - final String batchFileNameBase, final long batchNumber) - throws IOException, ClassNotFoundException { + final String batchFileNameBase, final long batchNumber) { IdentityHashMap trieLogs; try (FileInputStream fis = new FileInputStream(batchFileNameBase + "-" + batchNumber); @@ -340,7 +295,7 @@ private static IdentityHashMap readTrieLogsFromFile( } catch (IOException | ClassNotFoundException e) { LOG.error(e.getMessage()); - throw e; + throw new RuntimeException(e); } return trieLogs; @@ -359,6 +314,41 @@ private static IdentityHashMap getTrieLogs( return trieLogsToRetain; } + static TrieLogCount getCount( + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final int limit, + final Blockchain blockchain) { + final AtomicInteger total = new AtomicInteger(); + final AtomicInteger canonicalCount = new AtomicInteger(); + final AtomicInteger forkCount = new AtomicInteger(); + final AtomicInteger orphanCount = new AtomicInteger(); + rootWorldStateStorage + .streamTrieLogKeys(limit) + .map(Bytes32::wrap) + .map(Hash::wrap) + .forEach( + hash -> { + total.getAndIncrement(); + blockchain + .getBlockHeader(hash) + .ifPresentOrElse( + (header) -> { + long number = header.getNumber(); + final Optional headerByNumber = + blockchain.getBlockHeader(number); + if (headerByNumber.isPresent() + && headerByNumber.get().getHash().equals(hash)) { + canonicalCount.getAndIncrement(); + } else { + forkCount.getAndIncrement(); + } + }, + orphanCount::getAndIncrement); + }); + + return new TrieLogCount(total.get(), canonicalCount.get(), forkCount.get(), orphanCount.get()); + } + static void printCount(final PrintWriter out, final TrieLogCount count) { out.printf( "trieLog count: %s\n - canonical count: %s\n - fork count: %s\n - orphaned count: %s\n", diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java new file mode 100644 index 00000000000..f00dbf864f1 --- /dev/null +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -0,0 +1,239 @@ +package org.hyperledger.besu.cli.subcommands.storage; + +import static com.ibm.icu.impl.locale.KeyTypeData.ValueType.any; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageFormat.BONSAI; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.chain.MutableBlockchain; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; +import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; +import org.hyperledger.besu.ethereum.storage.StorageProvider; +import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class TrieLogHelperTest { + + private static final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); + private static BonsaiWorldStateKeyValueStorage inMemoryWorldState; + + @Mock private MutableBlockchain blockchain; + + @TempDir static Path dataDir; + + Path test; + static BlockHeader blockHeader1; + static BlockHeader blockHeader2; + static BlockHeader blockHeader3; + static BlockHeader blockHeader4; + static BlockHeader blockHeader5; + + @BeforeAll + public static void setup() throws IOException { + Files.createDirectories(dataDir.resolve("database")); + blockHeader1 = new BlockHeaderTestFixture().number(1).buildHeader(); + blockHeader2 = new BlockHeaderTestFixture().number(2).buildHeader(); + blockHeader3 = new BlockHeaderTestFixture().number(3).buildHeader(); + blockHeader4 = new BlockHeaderTestFixture().number(4).buildHeader(); + blockHeader5 = new BlockHeaderTestFixture().number(5).buildHeader(); + + inMemoryWorldState = + new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + + var updater = inMemoryWorldState.updater(); + updater + .getTrieLogStorageTransaction() + .put(blockHeader1.getHash().toArrayUnsafe(), Bytes.fromHexString("0x01").toArrayUnsafe()); + updater + .getTrieLogStorageTransaction() + .put(blockHeader2.getHash().toArrayUnsafe(), Bytes.fromHexString("0x02").toArrayUnsafe()); + updater + .getTrieLogStorageTransaction() + .put(blockHeader3.getHash().toArrayUnsafe(), Bytes.fromHexString("0x03").toArrayUnsafe()); + updater + .getTrieLogStorageTransaction() + .put(blockHeader4.getHash().toArrayUnsafe(), Bytes.fromHexString("0x04").toArrayUnsafe()); + updater + .getTrieLogStorageTransaction() + .put(blockHeader5.getHash().toArrayUnsafe(), Bytes.fromHexString("0x05").toArrayUnsafe()); + updater.getTrieLogStorageTransaction().commit(); + } + + void mockBlockchainBase() { + when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); + when(blockchain.getFinalized()).thenReturn(Optional.of(blockHeader3.getBlockHash())); + when(blockchain.getBlockHeader(any(Hash.class))).thenReturn(Optional.of(blockHeader3)); + } + + @Test + public void prune() { + + DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(BONSAI) + .bonsaiMaxLayersToLoad(2L) + .unstable( + ImmutableDataStorageConfiguration.Unstable.builder() + .bonsaiTrieLogRetentionThreshold(3) + .build() + .withBonsaiTrieLogRetentionThreshold(3)) + .build(); + + mockBlockchainBase(); + when(blockchain.getBlockHeader(5)).thenReturn(Optional.of(blockHeader5)); + when(blockchain.getBlockHeader(4)).thenReturn(Optional.of(blockHeader4)); + when(blockchain.getBlockHeader(3)).thenReturn(Optional.of(blockHeader3)); + + // assert trie logs that will be pruned exist before prune call + assertArrayEquals( + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), + Bytes.fromHexString("0x01").toArrayUnsafe()); + assertArrayEquals( + inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), + Bytes.fromHexString("0x02").toArrayUnsafe()); + assertArrayEquals( + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), + Bytes.fromHexString("0x03").toArrayUnsafe()); + + TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir); + + // assert pruned trie logs are not in the DB + assertEquals(inMemoryWorldState.getTrieLog(blockHeader1.getHash()), Optional.empty()); + assertEquals(inMemoryWorldState.getTrieLog(blockHeader2.getHash()), Optional.empty()); + + // assert retained trie logs are in the DB + assertArrayEquals( + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), + Bytes.fromHexString("0x03").toArrayUnsafe()); + assertArrayEquals( + inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), + Bytes.fromHexString("0x04").toArrayUnsafe()); + assertArrayEquals( + inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), + Bytes.fromHexString("0x05").toArrayUnsafe()); + } + + @Test + public void cantPruneIfNoFinalizedIsFound() { + DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(BONSAI) + .bonsaiMaxLayersToLoad(2L) + .unstable( + ImmutableDataStorageConfiguration.Unstable.builder() + .bonsaiTrieLogRetentionThreshold(2) + .build() + .withBonsaiTrieLogRetentionThreshold(2)) + .build(); + + when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); + when(blockchain.getFinalized()).thenReturn(Optional.empty()); + + assertThrows( + RuntimeException.class, + () -> + TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); + } + + @Test + public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength() { + DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(BONSAI) + .bonsaiMaxLayersToLoad(2L) + .unstable( + ImmutableDataStorageConfiguration.Unstable.builder() + .bonsaiTrieLogRetentionThreshold(10) + .build() + .withBonsaiTrieLogRetentionThreshold(10)) + .build(); + + when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); + + assertThrows( + IllegalArgumentException.class, + () -> + TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); + } + + @Test + public void cantPruneIfUserRequiredFurtherThanFinalized() { + + DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(BONSAI) + .bonsaiMaxLayersToLoad(2L) + .unstable( + ImmutableDataStorageConfiguration.Unstable.builder() + .bonsaiTrieLogRetentionThreshold(2) + .build() + .withBonsaiTrieLogRetentionThreshold(2)) + .build(); + + mockBlockchainBase(); + + assertThrows( + IllegalArgumentException.class, + () -> + TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); + } + + @Test + public void exceptionWhileSavingFileStopsPruneProcess() throws IOException { + Files.delete(dataDir.resolve("database")); + + DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(BONSAI) + .bonsaiMaxLayersToLoad(2L) + .unstable( + ImmutableDataStorageConfiguration.Unstable.builder() + .bonsaiTrieLogRetentionThreshold(2) + .build() + .withBonsaiTrieLogRetentionThreshold(2)) + .build(); + + assertThrows( + RuntimeException.class, + () -> + TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); + + // assert all trie logs are still in the DB + assertArrayEquals( + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), + Bytes.fromHexString("0x01").toArrayUnsafe()); + assertArrayEquals( + inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), + Bytes.fromHexString("0x02").toArrayUnsafe()); + assertArrayEquals( + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), + Bytes.fromHexString("0x03").toArrayUnsafe()); + assertArrayEquals( + inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), + Bytes.fromHexString("0x04").toArrayUnsafe()); + assertArrayEquals( + inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), + Bytes.fromHexString("0x05").toArrayUnsafe()); + } +} From c7144fe4d1d12d49b087327eaf7f8d7a3147b4ae Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Thu, 21 Dec 2023 17:16:47 +1000 Subject: [PATCH 18/37] spdx Signed-off-by: Gabriel Fukushima --- .../subcommands/storage/TrieLogHelperTest.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index f00dbf864f1..a51185678f6 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -1,6 +1,20 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + package org.hyperledger.besu.cli.subcommands.storage; -import static com.ibm.icu.impl.locale.KeyTypeData.ValueType.any; import static org.hyperledger.besu.ethereum.worldstate.DataStorageFormat.BONSAI; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; From 20b0ba549b3d7b4f794901b5a4bafc593f64054e Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Thu, 21 Dec 2023 20:52:30 +1000 Subject: [PATCH 19/37] Fix unit tests directory creation and deletion Signed-off-by: Gabriel Fukushima --- .../cli/subcommands/storage/TrieLogHelperTest.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index a51185678f6..82659701b1c 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -39,7 +39,9 @@ import java.util.Optional; import org.apache.tuweni.bytes.Bytes; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; @@ -65,7 +67,7 @@ class TrieLogHelperTest { @BeforeAll public static void setup() throws IOException { - Files.createDirectories(dataDir.resolve("database")); + blockHeader1 = new BlockHeaderTestFixture().number(1).buildHeader(); blockHeader2 = new BlockHeaderTestFixture().number(2).buildHeader(); blockHeader3 = new BlockHeaderTestFixture().number(3).buildHeader(); @@ -94,6 +96,16 @@ public static void setup() throws IOException { updater.getTrieLogStorageTransaction().commit(); } + @BeforeEach + void createDirectory() throws IOException { + Files.createDirectories(dataDir.resolve("database")); + } + + @AfterEach + void deleteDirectory() throws IOException { + Files.deleteIfExists(dataDir.resolve("database")); + } + void mockBlockchainBase() { when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); when(blockchain.getFinalized()).thenReturn(Optional.of(blockHeader3.getBlockHash())); From 586ab254375cf195f59ce4892d22c98f7d78c0c2 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Thu, 4 Jan 2024 14:52:59 +1000 Subject: [PATCH 20/37] rename Xbonsai-trie-log-pruning-enabled to Xbonsai-limit-trie-logs-enabled Signed-off-by: Gabriel Fukushima --- .../besu/cli/options/stable/DataStorageOptions.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java index e0b19735683..8686fc608b1 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java @@ -42,6 +42,12 @@ public class DataStorageOptions implements CLIOptions private static final String BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD = "--bonsai-historical-block-limit"; + private static final String BONSAI_TRIE_LOG_PRUNING_ENABLED = + "--Xbonsai-trie-log-pruning-enabled"; + + private static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED = + "--Xbonsai-limit-trie-logs-enabled"; + // Use Bonsai DB @Option( names = {DATA_STORAGE_FORMAT}, @@ -65,7 +71,7 @@ static class Unstable { @CommandLine.Option( hidden = true, - names = {"--Xbonsai-trie-log-pruning-enabled"}, + names = {BONSAI_LIMIT_TRIE_LOGS_ENABLED, BONSAI_TRIE_LOG_PRUNING_ENABLED}, description = "Enable trie log pruning. (default: ${DEFAULT-VALUE})") private boolean bonsaiTrieLogPruningEnabled = DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED; From 67e6f3d6818d35a98431e742092ae4959dc6f192 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Thu, 4 Jan 2024 14:53:16 +1000 Subject: [PATCH 21/37] Import and export trie log subcommands Signed-off-by: Gabriel Fukushima --- .../subcommands/storage/TrieLogHelper.java | 53 ++++++++----- .../storage/TrieLogSubCommand.java | 75 +++++++++++++++++++ 2 files changed, 110 insertions(+), 18 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 0878a17024a..efecb8e2a83 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -16,6 +16,7 @@ package org.hyperledger.besu.cli.subcommands.storage; import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Collections.singletonList; import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH; import org.hyperledger.besu.datatypes.Hash; @@ -95,16 +96,15 @@ private static void processTrieLogBatches( final String batchFileNameBase) { for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { - + final String batchFileName = batchFileNameBase + "-" + batchNumber; final long firstBlockOfBatch = chainHeight - ((batchNumber - 1) * BATCH_SIZE); - final long lastBlockOfBatch = Math.max(chainHeight - (batchNumber * BATCH_SIZE), lastBlockNumberToRetainTrieLogsFor); - final List trieLogKeys = getTrieLogKeysForBlocks(blockchain, firstBlockOfBatch, lastBlockOfBatch); - saveTrieLogBatches(batchFileNameBase, rootWorldStateStorage, batchNumber, trieLogKeys); + LOG.info("Saving trie logs to retain in file (batch {})...", batchNumber); + saveTrieLogBatches(batchFileName, rootWorldStateStorage, trieLogKeys); } LOG.info("Clear trie logs..."); @@ -116,15 +116,12 @@ private static void processTrieLogBatches( } private static void saveTrieLogBatches( - final String batchFileNameBase, + final String batchFileName, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final long batchNumber, final List trieLogKeys) { - LOG.info("Saving trie logs to retain in file (batch {})...", batchNumber); - try { - saveTrieLogsInFile(trieLogKeys, rootWorldStateStorage, batchNumber, batchFileNameBase); + saveTrieLogsInFile(trieLogKeys, rootWorldStateStorage, batchFileName); } catch (IOException e) { LOG.error("Error saving trie logs to file: {}", e.getMessage()); throw new RuntimeException(e); @@ -208,9 +205,8 @@ private static void recreateTrieLogs( final String batchFileNameBase) throws IOException { // process in chunk to avoid OOM - - IdentityHashMap trieLogsToRetain = - readTrieLogsFromFile(batchFileNameBase, batchNumber); + final String batchFileName = batchFileNameBase + "-" + batchNumber; + IdentityHashMap trieLogsToRetain = readTrieLogsFromFile(batchFileName); final int chunkSize = ROCKSDB_MAX_INSERTS_PER_TRANSACTION; List keys = new ArrayList<>(trieLogsToRetain.keySet()); @@ -263,11 +259,10 @@ private static void validatePruneConfiguration(final DataStorageConfiguration co private static void saveTrieLogsInFile( final List trieLogsKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final long batchNumber, - final String batchFileNameBase) + final String batchFileName) throws IOException { - File file = new File(batchFileNameBase + "-" + batchNumber); + File file = new File(batchFileName); if (file.exists()) { LOG.error("File already exists, skipping file creation"); return; @@ -283,11 +278,10 @@ private static void saveTrieLogsInFile( } @SuppressWarnings("unchecked") - private static IdentityHashMap readTrieLogsFromFile( - final String batchFileNameBase, final long batchNumber) { + private static IdentityHashMap readTrieLogsFromFile(final String batchFileName) { IdentityHashMap trieLogs; - try (FileInputStream fis = new FileInputStream(batchFileNameBase + "-" + batchNumber); + try (FileInputStream fis = new FileInputStream(batchFileName); ObjectInputStream ois = new ObjectInputStream(fis)) { trieLogs = (IdentityHashMap) ois.readObject(); @@ -355,5 +349,28 @@ static void printCount(final PrintWriter out, final TrieLogCount count) { count.total, count.canonicalCount, count.forkCount, count.orphanCount); } + static void importTrieLog( + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final Path dataDirectoryPath, + final Hash trieLogHash) { + final String trieLogFile = dataDirectoryPath.resolve(DATABASE_PATH) + "/" + trieLogHash; + + var trieLog = readTrieLogsFromFile(trieLogFile); + + var updater = rootWorldStateStorage.updater(); + trieLog.forEach((key, value) -> updater.getTrieLogStorageTransaction().put(key, value)); + updater.getTrieLogStorageTransaction().commit(); + } + + static void exportTrieLog( + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final Path dataDirectoryPath, + final Hash trieLogHash) + throws IOException { + final String trieLogFile = dataDirectoryPath.resolve(DATABASE_PATH) + "/" + trieLogHash; + + saveTrieLogsInFile(singletonList(trieLogHash), rootWorldStateStorage, trieLogFile); + } + record TrieLogCount(int total, int canonicalCount, int forkCount, int orphanCount) {} } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index bf75fd6eb9a..59c1f5f2f65 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -17,8 +17,10 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import org.hyperledger.besu.cli.DefaultCommandValues; import org.hyperledger.besu.cli.util.VersionProvider; import org.hyperledger.besu.controller.BesuController; +import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; @@ -26,6 +28,7 @@ import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import java.io.IOException; import java.io.PrintWriter; import java.nio.file.Path; import java.nio.file.Paths; @@ -123,6 +126,78 @@ public void run() { } } + @Command( + name = "export", + description = + "This command prunes all trie log layers below the retention threshold, including orphaned trie logs.", + mixinStandardHelpOptions = true, + versionProvider = VersionProvider.class) + static class ExportTrieLog implements Runnable { + + @SuppressWarnings("unused") + @ParentCommand + private TrieLogSubCommand parentCommand; + + @SuppressWarnings("unused") + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec + + @CommandLine.Option( + names = "--trie-log-hash", + paramLabel = DefaultCommandValues.MANDATORY_LONG_FORMAT_HELP, + description = "The of the block you want the trie log for.", + arity = "1..1") + private String trieLogHash; + + @Override + public void run() { + TrieLogContext context = getTrieLogContext(); + final Path dataDirectoryPath = + Paths.get( + TrieLogSubCommand.parentCommand.parentCommand.dataDir().toAbsolutePath().toString()); + try { + TrieLogHelper.exportTrieLog( + context.rootWorldStateStorage(), dataDirectoryPath, Hash.fromHexString(trieLogHash)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + @Command( + name = "import", + description = + "This command prunes all trie log layers below the retention threshold, including orphaned trie logs.", + mixinStandardHelpOptions = true, + versionProvider = VersionProvider.class) + static class ImportTrieLog implements Runnable { + + @SuppressWarnings("unused") + @ParentCommand + private TrieLogSubCommand parentCommand; + + @SuppressWarnings("unused") + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec + + @CommandLine.Option( + names = "--trie-log-key-hash", + paramLabel = DefaultCommandValues.MANDATORY_LONG_FORMAT_HELP, + description = "The of the block you want the trie log for.", + arity = "1..1") + private String trieLogHash; + + @Override + public void run() { + TrieLogContext context = getTrieLogContext(); + final Path dataDirectoryPath = + Paths.get( + TrieLogSubCommand.parentCommand.parentCommand.dataDir().toAbsolutePath().toString()); + TrieLogHelper.importTrieLog( + context.rootWorldStateStorage(), dataDirectoryPath, Hash.fromHexString(trieLogHash)); + } + } + record TrieLogContext( DataStorageConfiguration config, BonsaiWorldStateKeyValueStorage rootWorldStateStorage, From b9640e5738acf11dffc2b6a6c7dba10859a99548 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Thu, 4 Jan 2024 16:19:23 +1000 Subject: [PATCH 22/37] PR review Signed-off-by: Gabriel Fukushima --- .../besu/cli/subcommands/storage/TrieLogHelper.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index efecb8e2a83..6994f383c24 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -56,8 +56,10 @@ static void prune( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final MutableBlockchain blockchain, final Path dataDirectoryPath) { - final String batchFileNameBase = dataDirectoryPath.resolve(DATABASE_PATH) + "/" + TRIE_LOG_FILE; - TrieLogHelper.validatePruneConfiguration(config); + final String batchFileNameBase = + dataDirectoryPath.resolve(DATABASE_PATH).resolve(TRIE_LOG_FILE).toString(); + + validatePruneConfiguration(config); final long layersToRetain = config.getUnstable().getBonsaiTrieLogRetentionThreshold(); From 3bc1878424b2e91fc7e4c91804e4ed67c6acbe16 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Thu, 4 Jan 2024 16:20:08 +1000 Subject: [PATCH 23/37] spotless Signed-off-by: Gabriel Fukushima --- .../besu/cli/options/stable/DataStorageOptions.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java index 8686fc608b1..e70e4d59f83 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java @@ -45,8 +45,7 @@ public class DataStorageOptions implements CLIOptions private static final String BONSAI_TRIE_LOG_PRUNING_ENABLED = "--Xbonsai-trie-log-pruning-enabled"; - private static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED = - "--Xbonsai-limit-trie-logs-enabled"; + private static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED = "--Xbonsai-limit-trie-logs-enabled"; // Use Bonsai DB @Option( From d47ddf5450df23cc7b1ef5ef58681daca0be30e8 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Mon, 8 Jan 2024 09:59:08 +1000 Subject: [PATCH 24/37] fix path resolver and added unit tests Signed-off-by: Gabriel Fukushima --- .../subcommands/storage/TrieLogHelper.java | 8 ++-- .../storage/TrieLogHelperTest.java | 39 +++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 6994f383c24..6a3aa100a4a 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -280,7 +280,7 @@ private static void saveTrieLogsInFile( } @SuppressWarnings("unchecked") - private static IdentityHashMap readTrieLogsFromFile(final String batchFileName) { + static IdentityHashMap readTrieLogsFromFile(final String batchFileName) { IdentityHashMap trieLogs; try (FileInputStream fis = new FileInputStream(batchFileName); @@ -355,7 +355,8 @@ static void importTrieLog( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final Path dataDirectoryPath, final Hash trieLogHash) { - final String trieLogFile = dataDirectoryPath.resolve(DATABASE_PATH) + "/" + trieLogHash; + final String trieLogFile = + dataDirectoryPath.resolve(DATABASE_PATH).resolve(trieLogHash.toString()).toString(); var trieLog = readTrieLogsFromFile(trieLogFile); @@ -369,7 +370,8 @@ static void exportTrieLog( final Path dataDirectoryPath, final Hash trieLogHash) throws IOException { - final String trieLogFile = dataDirectoryPath.resolve(DATABASE_PATH) + "/" + trieLogHash; + final String trieLogFile = + dataDirectoryPath.resolve(DATABASE_PATH).resolve(trieLogHash.toString()).toString(); saveTrieLogsInFile(singletonList(trieLogHash), rootWorldStateStorage, trieLogFile); } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index 82659701b1c..c86134ae736 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -262,4 +262,43 @@ public void exceptionWhileSavingFileStopsPruneProcess() throws IOException { inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), Bytes.fromHexString("0x05").toArrayUnsafe()); } + + @Test + public void exportedTrieMatchesDbTrieLog() throws IOException { + TrieLogHelper.exportTrieLog(inMemoryWorldState, dataDir, blockHeader1.getHash()); + Path trieLogFile = dataDir.resolve("database").resolve(blockHeader1.getHash().toString()); + + var trieLog = + TrieLogHelper.readTrieLogsFromFile(trieLogFile.toString()).entrySet().stream() + .findFirst() + .get(); + + assertArrayEquals(trieLog.getKey(), blockHeader1.getHash().toArrayUnsafe()); + assertArrayEquals(trieLog.getValue(), Bytes.fromHexString("0x01").toArrayUnsafe()); + + Files.delete(trieLogFile); + } + + @Test + public void importedTrieLogMatchesDbTrieLog() throws IOException { + StorageProvider tempStorageProvider = new InMemoryKeyValueStorageProvider(); + BonsaiWorldStateKeyValueStorage inMemoryWorldState2 = + new BonsaiWorldStateKeyValueStorage(tempStorageProvider, new NoOpMetricsSystem()); + + TrieLogHelper.exportTrieLog(inMemoryWorldState, dataDir, blockHeader1.getHash()); + Path trieLogFile = dataDir.resolve("database").resolve(blockHeader1.getHash().toString()); + + var trieLog = TrieLogHelper.readTrieLogsFromFile(trieLogFile.toString()); + var updater = inMemoryWorldState2.updater(); + + trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v)); + + updater.getTrieLogStorageTransaction().commit(); + + assertArrayEquals( + inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get(), + Bytes.fromHexString("0x01").toArrayUnsafe()); + + Files.delete(trieLogFile); + } } From 1699fe4ca294724192057042d64e63a05a683547 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Mon, 8 Jan 2024 10:31:45 +1000 Subject: [PATCH 25/37] fix unit test Signed-off-by: Gabriel Fukushima --- .../besu/cli/subcommands/storage/TrieLogHelperTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index c86134ae736..85087b61f4a 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -227,7 +227,6 @@ public void cantPruneIfUserRequiredFurtherThanFinalized() { @Test public void exceptionWhileSavingFileStopsPruneProcess() throws IOException { - Files.delete(dataDir.resolve("database")); DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() @@ -243,7 +242,11 @@ public void exceptionWhileSavingFileStopsPruneProcess() throws IOException { assertThrows( RuntimeException.class, () -> - TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); + TrieLogHelper.prune( + dataStorageConfiguration, + inMemoryWorldState, + blockchain, + dataDir.resolve("unknownPath"))); // assert all trie logs are still in the DB assertArrayEquals( From 5d3b4f25b71c70753ff5e18bcafb19f270fcff12 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Mon, 8 Jan 2024 11:08:26 +1000 Subject: [PATCH 26/37] fix unit test Signed-off-by: Gabriel Fukushima --- .../besu/cli/subcommands/storage/TrieLogHelperTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index 85087b61f4a..9d850c6a255 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -40,7 +40,6 @@ import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -65,7 +64,7 @@ class TrieLogHelperTest { static BlockHeader blockHeader4; static BlockHeader blockHeader5; - @BeforeAll + @BeforeEach public static void setup() throws IOException { blockHeader1 = new BlockHeaderTestFixture().number(1).buildHeader(); @@ -94,10 +93,11 @@ public static void setup() throws IOException { .getTrieLogStorageTransaction() .put(blockHeader5.getHash().toArrayUnsafe(), Bytes.fromHexString("0x05").toArrayUnsafe()); updater.getTrieLogStorageTransaction().commit(); + + createDirectory(); } - @BeforeEach - void createDirectory() throws IOException { + static void createDirectory() throws IOException { Files.createDirectories(dataDir.resolve("database")); } From 0caa4cf33a5a55284505c9cd53ca2e6368643c3b Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Mon, 8 Jan 2024 11:23:02 +1000 Subject: [PATCH 27/37] Add import and export to list of subcommands under --x-trie-log Signed-off-by: Gabriel Fukushima --- .../besu/cli/subcommands/storage/TrieLogSubCommand.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index 59c1f5f2f65..22b84f22809 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -46,7 +46,12 @@ description = "Manipulate trie logs", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class, - subcommands = {TrieLogSubCommand.CountTrieLog.class, TrieLogSubCommand.PruneTrieLog.class}) + subcommands = { + TrieLogSubCommand.CountTrieLog.class, + TrieLogSubCommand.PruneTrieLog.class, + TrieLogSubCommand.ExportTrieLog.class, + TrieLogSubCommand.ImportTrieLog.class + }) public class TrieLogSubCommand implements Runnable { @SuppressWarnings("UnusedVariable") From 37df23e1b1f7cf9ebf29537dd7763a778a70a8c1 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Mon, 8 Jan 2024 11:56:28 +1000 Subject: [PATCH 28/37] Remove static from setup method Signed-off-by: Gabriel Fukushima --- .../besu/cli/subcommands/storage/TrieLogHelperTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index 9d850c6a255..ff0784a2cc6 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -65,7 +65,7 @@ class TrieLogHelperTest { static BlockHeader blockHeader5; @BeforeEach - public static void setup() throws IOException { + public void setup() throws IOException { blockHeader1 = new BlockHeaderTestFixture().number(1).buildHeader(); blockHeader2 = new BlockHeaderTestFixture().number(2).buildHeader(); From 5ce18004be6ad761cd17a7befeff36867c5c6f9f Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Mon, 8 Jan 2024 14:10:03 +1000 Subject: [PATCH 29/37] change option name and fix descriptions Signed-off-by: Gabriel Fukushima --- .../besu/cli/subcommands/storage/TrieLogSubCommand.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index 22b84f22809..04aa132f1f3 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -150,7 +150,7 @@ static class ExportTrieLog implements Runnable { @CommandLine.Option( names = "--trie-log-hash", paramLabel = DefaultCommandValues.MANDATORY_LONG_FORMAT_HELP, - description = "The of the block you want the trie log for.", + description = "The hash of the block you want to export the trie log.", arity = "1..1") private String trieLogHash; @@ -186,9 +186,9 @@ static class ImportTrieLog implements Runnable { private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec @CommandLine.Option( - names = "--trie-log-key-hash", + names = "--trie-log-hash", paramLabel = DefaultCommandValues.MANDATORY_LONG_FORMAT_HELP, - description = "The of the block you want the trie log for.", + description = "The hash of the block you want to import the trie log.", arity = "1..1") private String trieLogHash; From cf3a5e642966b4a25540b9f39d23de198a9cb8a8 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Tue, 9 Jan 2024 13:36:57 +1000 Subject: [PATCH 30/37] Fix subcommands descriptions Signed-off-by: Gabriel Fukushima --- .../storage/TrieLogSubCommand.java | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index 04aa132f1f3..7c1dc886910 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -17,7 +17,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import org.hyperledger.besu.cli.DefaultCommandValues; import org.hyperledger.besu.cli.util.VersionProvider; import org.hyperledger.besu.controller.BesuController; import org.hyperledger.besu.datatypes.Hash; @@ -133,8 +132,7 @@ public void run() { @Command( name = "export", - description = - "This command prunes all trie log layers below the retention threshold, including orphaned trie logs.", + description = "This command exports the trie log of a determined block to a binary file", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class) static class ExportTrieLog implements Runnable { @@ -148,11 +146,10 @@ static class ExportTrieLog implements Runnable { private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec @CommandLine.Option( - names = "--trie-log-hash", - paramLabel = DefaultCommandValues.MANDATORY_LONG_FORMAT_HELP, - description = "The hash of the block you want to export the trie log.", + names = "--trie-log-block-hash", + description = "The hash of the block you want to export the trie log of.", arity = "1..1") - private String trieLogHash; + private String trieLogBlockHash; @Override public void run() { @@ -162,7 +159,9 @@ public void run() { TrieLogSubCommand.parentCommand.parentCommand.dataDir().toAbsolutePath().toString()); try { TrieLogHelper.exportTrieLog( - context.rootWorldStateStorage(), dataDirectoryPath, Hash.fromHexString(trieLogHash)); + context.rootWorldStateStorage(), + dataDirectoryPath, + Hash.fromHexString(trieLogBlockHash)); } catch (IOException e) { throw new RuntimeException(e); } @@ -171,8 +170,7 @@ public void run() { @Command( name = "import", - description = - "This command prunes all trie log layers below the retention threshold, including orphaned trie logs.", + description = "This command imports a trie log exported by another besu node", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class) static class ImportTrieLog implements Runnable { @@ -186,11 +184,10 @@ static class ImportTrieLog implements Runnable { private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec @CommandLine.Option( - names = "--trie-log-hash", - paramLabel = DefaultCommandValues.MANDATORY_LONG_FORMAT_HELP, - description = "The hash of the block you want to import the trie log.", + names = "--trie-log-block-hash", + description = "The hash of the block you want to import the trie log of", arity = "1..1") - private String trieLogHash; + private String trieLogBlockHash; @Override public void run() { @@ -199,7 +196,7 @@ public void run() { Paths.get( TrieLogSubCommand.parentCommand.parentCommand.dataDir().toAbsolutePath().toString()); TrieLogHelper.importTrieLog( - context.rootWorldStateStorage(), dataDirectoryPath, Hash.fromHexString(trieLogHash)); + context.rootWorldStateStorage(), dataDirectoryPath, Hash.fromHexString(trieLogBlockHash)); } } From 5fb941392862908cbac8f8a34390d55b96d9fa83 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Fri, 12 Jan 2024 15:28:39 +1000 Subject: [PATCH 31/37] Remove old flag and move commands const into Unstable Signed-off-by: Gabriel Fukushima --- .../cli/options/stable/DataStorageOptions.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java index e70e4d59f83..81e275e9a78 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java @@ -42,10 +42,7 @@ public class DataStorageOptions implements CLIOptions private static final String BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD = "--bonsai-historical-block-limit"; - private static final String BONSAI_TRIE_LOG_PRUNING_ENABLED = - "--Xbonsai-trie-log-pruning-enabled"; - private static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED = "--Xbonsai-limit-trie-logs-enabled"; // Use Bonsai DB @Option( @@ -67,23 +64,26 @@ public class DataStorageOptions implements CLIOptions private final DataStorageOptions.Unstable unstableOptions = new Unstable(); static class Unstable { - + private static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED = "--Xbonsai-limit-trie-logs-enabled"; + private static final String BONSAI_TRIE_LOGS_RETENTION_THRESHOLD = + "--Xbonsai-trie-logs-retention-threshold"; + private static final String BONSAI_TRIE_LOG_PRUNING_LIMIT = "--Xbonsai-trie-logs-pruning-limit"; @CommandLine.Option( hidden = true, - names = {BONSAI_LIMIT_TRIE_LOGS_ENABLED, BONSAI_TRIE_LOG_PRUNING_ENABLED}, + names = {BONSAI_LIMIT_TRIE_LOGS_ENABLED}, description = "Enable trie log pruning. (default: ${DEFAULT-VALUE})") private boolean bonsaiTrieLogPruningEnabled = DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED; @CommandLine.Option( hidden = true, - names = {"--Xbonsai-trie-log-retention-threshold"}, + names = {BONSAI_TRIE_LOGS_RETENTION_THRESHOLD}, description = "The number of blocks for which to retain trie logs. (default: ${DEFAULT-VALUE})") private long bonsaiTrieLogRetentionThreshold = DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; @CommandLine.Option( hidden = true, - names = {"--Xbonsai-trie-log-pruning-limit"}, + names = {BONSAI_TRIE_LOG_PRUNING_LIMIT}, description = "The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})") private int bonsaiTrieLogPruningLimit = DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT; From 087c54b1a01ed6cf7218b80a4b4d20d70f945f30 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Fri, 12 Jan 2024 15:29:59 +1000 Subject: [PATCH 32/37] Allow list of block hashes to passed as well as a file to be generated or imported Signed-off-by: Gabriel Fukushima --- .../options/stable/DataStorageOptions.java | 6 +- .../subcommands/storage/TrieLogHelper.java | 18 ++---- .../storage/TrieLogSubCommand.java | 56 ++++++++++++++----- .../storage/TrieLogHelperTest.java | 38 +++++-------- 4 files changed, 65 insertions(+), 53 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java index 81e275e9a78..5b4cf43eb1e 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java @@ -42,8 +42,6 @@ public class DataStorageOptions implements CLIOptions private static final String BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD = "--bonsai-historical-block-limit"; - - // Use Bonsai DB @Option( names = {DATA_STORAGE_FORMAT}, @@ -64,10 +62,12 @@ public class DataStorageOptions implements CLIOptions private final DataStorageOptions.Unstable unstableOptions = new Unstable(); static class Unstable { - private static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED = "--Xbonsai-limit-trie-logs-enabled"; + private static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED = + "--Xbonsai-limit-trie-logs-enabled"; private static final String BONSAI_TRIE_LOGS_RETENTION_THRESHOLD = "--Xbonsai-trie-logs-retention-threshold"; private static final String BONSAI_TRIE_LOG_PRUNING_LIMIT = "--Xbonsai-trie-logs-pruning-limit"; + @CommandLine.Option( hidden = true, names = {BONSAI_LIMIT_TRIE_LOGS_ENABLED}, diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 6a3aa100a4a..805a9fa2870 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -16,7 +16,6 @@ package org.hyperledger.besu.cli.subcommands.storage; import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Collections.singletonList; import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH; import org.hyperledger.besu.datatypes.Hash; @@ -352,13 +351,9 @@ static void printCount(final PrintWriter out, final TrieLogCount count) { } static void importTrieLog( - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final Path dataDirectoryPath, - final Hash trieLogHash) { - final String trieLogFile = - dataDirectoryPath.resolve(DATABASE_PATH).resolve(trieLogHash.toString()).toString(); + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final Path trieLogFilePath) { - var trieLog = readTrieLogsFromFile(trieLogFile); + var trieLog = readTrieLogsFromFile(trieLogFilePath.toString()); var updater = rootWorldStateStorage.updater(); trieLog.forEach((key, value) -> updater.getTrieLogStorageTransaction().put(key, value)); @@ -367,13 +362,12 @@ static void importTrieLog( static void exportTrieLog( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final Path dataDirectoryPath, - final Hash trieLogHash) + final List trieLogHash, + final Path directoryPath) throws IOException { - final String trieLogFile = - dataDirectoryPath.resolve(DATABASE_PATH).resolve(trieLogHash.toString()).toString(); + final String trieLogFile = directoryPath.toString(); - saveTrieLogsInFile(singletonList(trieLogHash), rootWorldStateStorage, trieLogFile); + saveTrieLogsInFile(trieLogHash, rootWorldStateStorage, trieLogFile); } record TrieLogCount(int total, int canonicalCount, int forkCount, int orphanCount) {} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index 7c1dc886910..edf1a33f4df 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -31,6 +31,7 @@ import java.io.PrintWriter; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.List; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; @@ -147,21 +148,38 @@ static class ExportTrieLog implements Runnable { @CommandLine.Option( names = "--trie-log-block-hash", - description = "The hash of the block you want to export the trie log of.", + description = "The hash of the block you want to import the trie log of", + split = " {0,1}, {0,1}", + arity = "1..*") + private List trieLogBlockHashList; + + @CommandLine.Option( + names = "--trie-log-file-path", + description = "The hash of the block you want to import the trie log of", arity = "1..1") - private String trieLogBlockHash; + private Path trieLogFilePath = null; @Override public void run() { + if (trieLogFilePath == null) { + trieLogFilePath = + Paths.get( + TrieLogSubCommand.parentCommand + .parentCommand + .dataDir() + .resolve("trie-logs.bin") + .toAbsolutePath() + .toString()); + } + TrieLogContext context = getTrieLogContext(); - final Path dataDirectoryPath = - Paths.get( - TrieLogSubCommand.parentCommand.parentCommand.dataDir().toAbsolutePath().toString()); + + final List listOfBlockHashes = + trieLogBlockHashList.stream().map(Hash::fromHexString).toList(); + try { TrieLogHelper.exportTrieLog( - context.rootWorldStateStorage(), - dataDirectoryPath, - Hash.fromHexString(trieLogBlockHash)); + context.rootWorldStateStorage(), listOfBlockHashes, trieLogFilePath); } catch (IOException e) { throw new RuntimeException(e); } @@ -184,19 +202,27 @@ static class ImportTrieLog implements Runnable { private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec @CommandLine.Option( - names = "--trie-log-block-hash", + names = "--trie-log-file-path", description = "The hash of the block you want to import the trie log of", arity = "1..1") - private String trieLogBlockHash; + private Path trieLogFilePath = null; @Override public void run() { + if (trieLogFilePath == null) { + trieLogFilePath = + Paths.get( + TrieLogSubCommand.parentCommand + .parentCommand + .dataDir() + .resolve("trie-logs.bin") + .toAbsolutePath() + .toString()); + } + TrieLogContext context = getTrieLogContext(); - final Path dataDirectoryPath = - Paths.get( - TrieLogSubCommand.parentCommand.parentCommand.dataDir().toAbsolutePath().toString()); - TrieLogHelper.importTrieLog( - context.rootWorldStateStorage(), dataDirectoryPath, Hash.fromHexString(trieLogBlockHash)); + + TrieLogHelper.importTrieLog(context.rootWorldStateStorage(), trieLogFilePath); } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index ff0784a2cc6..f6bff4cd08c 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.cli.subcommands.storage; +import static java.util.Collections.singletonList; import static org.hyperledger.besu.ethereum.worldstate.DataStorageFormat.BONSAI; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -39,7 +40,6 @@ import java.util.Optional; import org.apache.tuweni.bytes.Bytes; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -93,17 +93,6 @@ public void setup() throws IOException { .getTrieLogStorageTransaction() .put(blockHeader5.getHash().toArrayUnsafe(), Bytes.fromHexString("0x05").toArrayUnsafe()); updater.getTrieLogStorageTransaction().commit(); - - createDirectory(); - } - - static void createDirectory() throws IOException { - Files.createDirectories(dataDir.resolve("database")); - } - - @AfterEach - void deleteDirectory() throws IOException { - Files.deleteIfExists(dataDir.resolve("database")); } void mockBlockchainBase() { @@ -113,7 +102,8 @@ void mockBlockchainBase() { } @Test - public void prune() { + public void prune() throws IOException { + Files.createDirectories(dataDir.resolve("database")); DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() @@ -268,18 +258,20 @@ public void exceptionWhileSavingFileStopsPruneProcess() throws IOException { @Test public void exportedTrieMatchesDbTrieLog() throws IOException { - TrieLogHelper.exportTrieLog(inMemoryWorldState, dataDir, blockHeader1.getHash()); - Path trieLogFile = dataDir.resolve("database").resolve(blockHeader1.getHash().toString()); + TrieLogHelper.exportTrieLog( + inMemoryWorldState, + singletonList(blockHeader1.getHash()), + dataDir.resolve("trie-log-dump")); var trieLog = - TrieLogHelper.readTrieLogsFromFile(trieLogFile.toString()).entrySet().stream() + TrieLogHelper.readTrieLogsFromFile(dataDir.resolve("trie-log-dump").toString()) + .entrySet() + .stream() .findFirst() .get(); assertArrayEquals(trieLog.getKey(), blockHeader1.getHash().toArrayUnsafe()); assertArrayEquals(trieLog.getValue(), Bytes.fromHexString("0x01").toArrayUnsafe()); - - Files.delete(trieLogFile); } @Test @@ -288,10 +280,12 @@ public void importedTrieLogMatchesDbTrieLog() throws IOException { BonsaiWorldStateKeyValueStorage inMemoryWorldState2 = new BonsaiWorldStateKeyValueStorage(tempStorageProvider, new NoOpMetricsSystem()); - TrieLogHelper.exportTrieLog(inMemoryWorldState, dataDir, blockHeader1.getHash()); - Path trieLogFile = dataDir.resolve("database").resolve(blockHeader1.getHash().toString()); + TrieLogHelper.exportTrieLog( + inMemoryWorldState, + singletonList(blockHeader1.getHash()), + dataDir.resolve("trie-log-dump")); - var trieLog = TrieLogHelper.readTrieLogsFromFile(trieLogFile.toString()); + var trieLog = TrieLogHelper.readTrieLogsFromFile(dataDir.resolve("trie-log-dump").toString()); var updater = inMemoryWorldState2.updater(); trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v)); @@ -301,7 +295,5 @@ public void importedTrieLogMatchesDbTrieLog() throws IOException { assertArrayEquals( inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get(), Bytes.fromHexString("0x01").toArrayUnsafe()); - - Files.delete(trieLogFile); } } From 3a89ac3d90b23be47936194d5cc26242adebaf6a Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Fri, 12 Jan 2024 16:37:07 +1000 Subject: [PATCH 33/37] Allow list of block hashes to passed as well as a file to be generated or imported Signed-off-by: Gabriel Fukushima --- .../besu/cli/subcommands/storage/TrieLogSubCommand.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index edf1a33f4df..2e1267c1f14 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -148,14 +148,15 @@ static class ExportTrieLog implements Runnable { @CommandLine.Option( names = "--trie-log-block-hash", - description = "The hash of the block you want to import the trie log of", + description = + "Comma separated list of hashes from the blocks you want to export the trie logs of", split = " {0,1}, {0,1}", arity = "1..*") private List trieLogBlockHashList; @CommandLine.Option( names = "--trie-log-file-path", - description = "The hash of the block you want to import the trie log of", + description = "The file you want to export the trie logs to", arity = "1..1") private Path trieLogFilePath = null; @@ -203,7 +204,7 @@ static class ImportTrieLog implements Runnable { @CommandLine.Option( names = "--trie-log-file-path", - description = "The hash of the block you want to import the trie log of", + description = "The file you want to import the trie logs from", arity = "1..1") private Path trieLogFilePath = null; From 9be7d13471d7b535637f56efa5ead71d9262f4c3 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Fri, 12 Jan 2024 16:53:19 +1000 Subject: [PATCH 34/37] Fix broken test when replaced the old option Signed-off-by: Gabriel Fukushima --- .../stable/DataStorageOptionsTest.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java index 437053afd56..2a63901975b 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java @@ -34,8 +34,8 @@ public void bonsaiTrieLogPruningLimitOption() { dataStorageConfiguration -> assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningLimit()) .isEqualTo(1), - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-pruning-limit", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-pruning-limit", "1"); } @@ -43,8 +43,8 @@ public void bonsaiTrieLogPruningLimitOption() { public void bonsaiTrieLogPruningLimitShouldBePositive() { internalTestFailure( "--Xbonsai-trie-log-pruning-limit=0 must be greater than 0", - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-pruning-limit", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-pruning-limit", "0"); } @@ -54,8 +54,8 @@ public void bonsaiTrieLogRetentionThresholdOption() { dataStorageConfiguration -> assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold()) .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD + 1), - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-retention-threshold", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-retention-threshold", "513"); } @@ -65,8 +65,8 @@ public void bonsaiTrieLogRetentionThresholdOption_boundaryTest() { dataStorageConfiguration -> assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold()) .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD), - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-retention-threshold", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-retention-threshold", "512"); } @@ -74,8 +74,8 @@ public void bonsaiTrieLogRetentionThresholdOption_boundaryTest() { public void bonsaiTrieLogRetentionThresholdShouldBeAboveMinimum() { internalTestFailure( "--Xbonsai-trie-log-retention-threshold minimum value is 512", - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-retention-threshold", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-retention-threshold", "511"); } From 75d1c3bfaf097125cdcf2e6bcde717f41eac9ad2 Mon Sep 17 00:00:00 2001 From: Jason Frame Date: Fri, 19 Jan 2024 11:07:03 +1000 Subject: [PATCH 35/37] import and export using rlp Signed-off-by: Jason Frame --- .../subcommands/storage/TrieLogHelper.java | 58 ++++++++++++++-- .../storage/TrieLogHelperTest.java | 67 ++++++++++--------- 2 files changed, 89 insertions(+), 36 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 805a9fa2870..c4e924a8354 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -22,7 +22,11 @@ import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; +import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import java.io.File; @@ -32,6 +36,7 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.PrintWriter; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.IdentityHashMap; @@ -39,6 +44,7 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -286,9 +292,7 @@ static IdentityHashMap readTrieLogsFromFile(final String batchFi ObjectInputStream ois = new ObjectInputStream(fis)) { trieLogs = (IdentityHashMap) ois.readObject(); - } catch (IOException | ClassNotFoundException e) { - LOG.error(e.getMessage()); throw new RuntimeException(e); } @@ -296,6 +300,52 @@ static IdentityHashMap readTrieLogsFromFile(final String batchFi return trieLogs; } + private static void saveTrieLogsAsRlpInFile( + final List trieLogsKeys, + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final String batchFileName) { + File file = new File(batchFileName); + if (file.exists()) { + LOG.error("File already exists, skipping file creation"); + return; + } + + final IdentityHashMap trieLogs = + getTrieLogs(trieLogsKeys, rootWorldStateStorage); + final Bytes rlp = + RLP.encode( + o -> + o.writeList( + trieLogs.entrySet(), (val, out) -> out.writeRaw(Bytes.wrap(val.getValue())))); + try { + Files.write(file.toPath(), rlp.toArrayUnsafe()); + } catch (IOException e) { + LOG.error(e.getMessage()); + throw new RuntimeException(e); + } + } + + static IdentityHashMap readTrieLogsAsRlpFromFile(final String batchFileName) { + try { + final Bytes file = Bytes.wrap(Files.readAllBytes(Path.of(batchFileName))); + final BytesValueRLPInput input = new BytesValueRLPInput(file, false); + + input.enterList(); + final IdentityHashMap trieLogs = new IdentityHashMap<>(); + while (!input.isEndOfCurrentList()) { + final Bytes trieLogBytes = input.currentListAsBytes(); + TrieLogLayer trieLogLayer = + TrieLogFactoryImpl.readFrom(new BytesValueRLPInput(Bytes.wrap(trieLogBytes), false)); + trieLogs.put(trieLogLayer.getBlockHash().toArrayUnsafe(), trieLogBytes.toArrayUnsafe()); + } + input.leaveList(); + + return trieLogs; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + private static IdentityHashMap getTrieLogs( final List trieLogKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) { IdentityHashMap trieLogsToRetain = new IdentityHashMap<>(); @@ -353,7 +403,7 @@ static void printCount(final PrintWriter out, final TrieLogCount count) { static void importTrieLog( final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final Path trieLogFilePath) { - var trieLog = readTrieLogsFromFile(trieLogFilePath.toString()); + var trieLog = readTrieLogsAsRlpFromFile(trieLogFilePath.toString()); var updater = rootWorldStateStorage.updater(); trieLog.forEach((key, value) -> updater.getTrieLogStorageTransaction().put(key, value)); @@ -367,7 +417,7 @@ static void exportTrieLog( throws IOException { final String trieLogFile = directoryPath.toString(); - saveTrieLogsInFile(trieLogHash, rootWorldStateStorage, trieLogFile); + saveTrieLogsAsRlpInFile(trieLogHash, rootWorldStateStorage, trieLogFile); } record TrieLogCount(int total, int canonicalCount, int forkCount, int orphanCount) {} diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index f6bff4cd08c..72b117e1b4f 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -28,8 +28,11 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -39,7 +42,6 @@ import java.nio.file.Path; import java.util.Optional; -import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -76,25 +78,35 @@ public void setup() throws IOException { inMemoryWorldState = new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + createTrieLog(blockHeader1); + var updater = inMemoryWorldState.updater(); updater .getTrieLogStorageTransaction() - .put(blockHeader1.getHash().toArrayUnsafe(), Bytes.fromHexString("0x01").toArrayUnsafe()); + .put(blockHeader1.getHash().toArrayUnsafe(), createTrieLog(blockHeader1)); updater .getTrieLogStorageTransaction() - .put(blockHeader2.getHash().toArrayUnsafe(), Bytes.fromHexString("0x02").toArrayUnsafe()); + .put(blockHeader2.getHash().toArrayUnsafe(), createTrieLog(blockHeader2)); updater .getTrieLogStorageTransaction() - .put(blockHeader3.getHash().toArrayUnsafe(), Bytes.fromHexString("0x03").toArrayUnsafe()); + .put(blockHeader3.getHash().toArrayUnsafe(), createTrieLog(blockHeader3)); updater .getTrieLogStorageTransaction() - .put(blockHeader4.getHash().toArrayUnsafe(), Bytes.fromHexString("0x04").toArrayUnsafe()); + .put(blockHeader4.getHash().toArrayUnsafe(), createTrieLog(blockHeader4)); updater .getTrieLogStorageTransaction() - .put(blockHeader5.getHash().toArrayUnsafe(), Bytes.fromHexString("0x05").toArrayUnsafe()); + .put(blockHeader5.getHash().toArrayUnsafe(), createTrieLog(blockHeader5)); updater.getTrieLogStorageTransaction().commit(); } + private static byte[] createTrieLog(final BlockHeader blockHeader) { + TrieLogLayer trieLogLayer = new TrieLogLayer(); + trieLogLayer.setBlockHash(blockHeader.getBlockHash()); + final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput(); + TrieLogFactoryImpl.writeTo(trieLogLayer, rlpLog); + return rlpLog.encoded().toArrayUnsafe(); + } + void mockBlockchainBase() { when(blockchain.getChainHeadBlockNumber()).thenReturn(5L); when(blockchain.getFinalized()).thenReturn(Optional.of(blockHeader3.getBlockHash())); @@ -123,14 +135,11 @@ public void prune() throws IOException { // assert trie logs that will be pruned exist before prune call assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), - Bytes.fromHexString("0x01").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), createTrieLog(blockHeader1)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), - Bytes.fromHexString("0x02").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), createTrieLog(blockHeader2)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), - Bytes.fromHexString("0x03").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3)); TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir); @@ -140,14 +149,11 @@ public void prune() throws IOException { // assert retained trie logs are in the DB assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), - Bytes.fromHexString("0x03").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), - Bytes.fromHexString("0x04").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), createTrieLog(blockHeader4)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), - Bytes.fromHexString("0x05").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), createTrieLog(blockHeader5)); } @Test @@ -240,20 +246,15 @@ public void exceptionWhileSavingFileStopsPruneProcess() throws IOException { // assert all trie logs are still in the DB assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), - Bytes.fromHexString("0x01").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), createTrieLog(blockHeader1)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), - Bytes.fromHexString("0x02").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), createTrieLog(blockHeader2)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), - Bytes.fromHexString("0x03").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), - Bytes.fromHexString("0x04").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), createTrieLog(blockHeader4)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), - Bytes.fromHexString("0x05").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), createTrieLog(blockHeader5)); } @Test @@ -264,14 +265,15 @@ public void exportedTrieMatchesDbTrieLog() throws IOException { dataDir.resolve("trie-log-dump")); var trieLog = - TrieLogHelper.readTrieLogsFromFile(dataDir.resolve("trie-log-dump").toString()) + TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) .entrySet() .stream() .findFirst() .get(); assertArrayEquals(trieLog.getKey(), blockHeader1.getHash().toArrayUnsafe()); - assertArrayEquals(trieLog.getValue(), Bytes.fromHexString("0x01").toArrayUnsafe()); + assertArrayEquals( + trieLog.getValue(), inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); } @Test @@ -285,7 +287,8 @@ public void importedTrieLogMatchesDbTrieLog() throws IOException { singletonList(blockHeader1.getHash()), dataDir.resolve("trie-log-dump")); - var trieLog = TrieLogHelper.readTrieLogsFromFile(dataDir.resolve("trie-log-dump").toString()); + var trieLog = + TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()); var updater = inMemoryWorldState2.updater(); trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v)); @@ -294,6 +297,6 @@ public void importedTrieLogMatchesDbTrieLog() throws IOException { assertArrayEquals( inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get(), - Bytes.fromHexString("0x01").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); } } From 542507501dcd5c0cb8dbb41fe167ff66854d8d68 Mon Sep 17 00:00:00 2001 From: Jason Frame Date: Fri, 19 Jan 2024 14:06:04 +1000 Subject: [PATCH 36/37] tests for exporting and importing multiple trielogs Signed-off-by: Jason Frame --- .../storage/TrieLogHelperTest.java | 75 ++++++++++++++++--- 1 file changed, 65 insertions(+), 10 deletions(-) diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index 72b117e1b4f..4397bb5d07f 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -40,8 +40,12 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -57,9 +61,6 @@ class TrieLogHelperTest { @Mock private MutableBlockchain blockchain; - @TempDir static Path dataDir; - - Path test; static BlockHeader blockHeader1; static BlockHeader blockHeader2; static BlockHeader blockHeader3; @@ -114,7 +115,7 @@ void mockBlockchainBase() { } @Test - public void prune() throws IOException { + public void prune(final @TempDir Path dataDir) throws IOException { Files.createDirectories(dataDir.resolve("database")); DataStorageConfiguration dataStorageConfiguration = @@ -157,7 +158,7 @@ public void prune() throws IOException { } @Test - public void cantPruneIfNoFinalizedIsFound() { + public void cantPruneIfNoFinalizedIsFound(final @TempDir Path dataDir) { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) @@ -179,7 +180,7 @@ public void cantPruneIfNoFinalizedIsFound() { } @Test - public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength() { + public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength(final @TempDir Path dataDir) { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) @@ -200,7 +201,7 @@ public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength() { } @Test - public void cantPruneIfUserRequiredFurtherThanFinalized() { + public void cantPruneIfUserRequiredFurtherThanFinalized(final @TempDir Path dataDir) { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() @@ -222,7 +223,7 @@ public void cantPruneIfUserRequiredFurtherThanFinalized() { } @Test - public void exceptionWhileSavingFileStopsPruneProcess() throws IOException { + public void exceptionWhileSavingFileStopsPruneProcess(final @TempDir Path dataDir) { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() @@ -258,7 +259,7 @@ public void exceptionWhileSavingFileStopsPruneProcess() throws IOException { } @Test - public void exportedTrieMatchesDbTrieLog() throws IOException { + public void exportedTrieMatchesDbTrieLog(final @TempDir Path dataDir) throws IOException { TrieLogHelper.exportTrieLog( inMemoryWorldState, singletonList(blockHeader1.getHash()), @@ -277,7 +278,31 @@ public void exportedTrieMatchesDbTrieLog() throws IOException { } @Test - public void importedTrieLogMatchesDbTrieLog() throws IOException { + public void exportedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) throws IOException { + TrieLogHelper.exportTrieLog( + inMemoryWorldState, + List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()), + dataDir.resolve("trie-log-dump")); + + var trieLogs = + TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) + .entrySet() + .stream() + .collect(Collectors.toMap(e -> Bytes.wrap(e.getKey()), Map.Entry::getValue)); + + assertArrayEquals( + trieLogs.get(blockHeader1.getHash()), + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); + assertArrayEquals( + trieLogs.get(blockHeader2.getHash()), + inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get()); + assertArrayEquals( + trieLogs.get(blockHeader3.getHash()), + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()); + } + + @Test + public void importedTrieLogMatchesDbTrieLog(final @TempDir Path dataDir) throws IOException { StorageProvider tempStorageProvider = new InMemoryKeyValueStorageProvider(); BonsaiWorldStateKeyValueStorage inMemoryWorldState2 = new BonsaiWorldStateKeyValueStorage(tempStorageProvider, new NoOpMetricsSystem()); @@ -299,4 +324,34 @@ public void importedTrieLogMatchesDbTrieLog() throws IOException { inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get(), inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); } + + @Test + public void importedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) throws IOException { + StorageProvider tempStorageProvider = new InMemoryKeyValueStorageProvider(); + BonsaiWorldStateKeyValueStorage inMemoryWorldState2 = + new BonsaiWorldStateKeyValueStorage(tempStorageProvider, new NoOpMetricsSystem()); + + TrieLogHelper.exportTrieLog( + inMemoryWorldState, + List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()), + dataDir.resolve("trie-log-dump")); + + var trieLog = + TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()); + var updater = inMemoryWorldState2.updater(); + + trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v)); + + updater.getTrieLogStorageTransaction().commit(); + + assertArrayEquals( + inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get(), + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); + assertArrayEquals( + inMemoryWorldState2.getTrieLog(blockHeader2.getHash()).get(), + inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get()); + assertArrayEquals( + inMemoryWorldState2.getTrieLog(blockHeader3.getHash()).get(), + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()); + } } From 55d653ec2cb36d21474e49097976a0bd41349c0c Mon Sep 17 00:00:00 2001 From: Jason Frame Date: Fri, 19 Jan 2024 15:52:53 +1000 Subject: [PATCH 37/37] fix build Signed-off-by: Jason Frame --- .../besu/cli/subcommands/storage/TrieLogHelperTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index 31a96ed20cc..5d4ede7f32e 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -306,7 +306,8 @@ public void exportedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) th public void importedTrieLogMatchesDbTrieLog(final @TempDir Path dataDir) throws IOException { StorageProvider tempStorageProvider = new InMemoryKeyValueStorageProvider(); BonsaiWorldStateKeyValueStorage inMemoryWorldState2 = - new BonsaiWorldStateKeyValueStorage(tempStorageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); TrieLogHelper.exportTrieLog( inMemoryWorldState, @@ -330,7 +331,8 @@ public void importedTrieLogMatchesDbTrieLog(final @TempDir Path dataDir) throws public void importedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) throws IOException { StorageProvider tempStorageProvider = new InMemoryKeyValueStorageProvider(); BonsaiWorldStateKeyValueStorage inMemoryWorldState2 = - new BonsaiWorldStateKeyValueStorage(tempStorageProvider, new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); TrieLogHelper.exportTrieLog( inMemoryWorldState,