diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java index e0b19735683..5b4cf43eb1e 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java @@ -62,23 +62,28 @@ public class DataStorageOptions implements CLIOptions private final DataStorageOptions.Unstable unstableOptions = new Unstable(); static class Unstable { + private static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED = + "--Xbonsai-limit-trie-logs-enabled"; + private static final String BONSAI_TRIE_LOGS_RETENTION_THRESHOLD = + "--Xbonsai-trie-logs-retention-threshold"; + private static final String BONSAI_TRIE_LOG_PRUNING_LIMIT = "--Xbonsai-trie-logs-pruning-limit"; @CommandLine.Option( hidden = true, - names = {"--Xbonsai-trie-log-pruning-enabled"}, + names = {BONSAI_LIMIT_TRIE_LOGS_ENABLED}, description = "Enable trie log pruning. (default: ${DEFAULT-VALUE})") private boolean bonsaiTrieLogPruningEnabled = DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED; @CommandLine.Option( hidden = true, - names = {"--Xbonsai-trie-log-retention-threshold"}, + names = {BONSAI_TRIE_LOGS_RETENTION_THRESHOLD}, description = "The number of blocks for which to retain trie logs. (default: ${DEFAULT-VALUE})") private long bonsaiTrieLogRetentionThreshold = DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD; @CommandLine.Option( hidden = true, - names = {"--Xbonsai-trie-log-pruning-limit"}, + names = {BONSAI_TRIE_LOG_PRUNING_LIMIT}, description = "The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})") private int bonsaiTrieLogPruningLimit = DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT; diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 52dbe559034..c4e924a8354 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -22,7 +22,11 @@ import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; +import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import java.io.File; @@ -32,6 +36,7 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.PrintWriter; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.IdentityHashMap; @@ -39,6 +44,7 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -97,16 +103,15 @@ private static void processTrieLogBatches( final String batchFileNameBase) { for (long batchNumber = 1; batchNumber <= numberOfBatches; batchNumber++) { - + final String batchFileName = batchFileNameBase + "-" + batchNumber; final long firstBlockOfBatch = chainHeight - ((batchNumber - 1) * BATCH_SIZE); - final long lastBlockOfBatch = Math.max(chainHeight - (batchNumber * BATCH_SIZE), lastBlockNumberToRetainTrieLogsFor); - final List trieLogKeys = getTrieLogKeysForBlocks(blockchain, firstBlockOfBatch, lastBlockOfBatch); - saveTrieLogBatches(batchFileNameBase, rootWorldStateStorage, batchNumber, trieLogKeys); + LOG.info("Saving trie logs to retain in file (batch {})...", batchNumber); + saveTrieLogBatches(batchFileName, rootWorldStateStorage, trieLogKeys); } LOG.info("Clear trie logs..."); @@ -118,15 +123,12 @@ private static void processTrieLogBatches( } private static void saveTrieLogBatches( - final String batchFileNameBase, + final String batchFileName, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final long batchNumber, final List trieLogKeys) { - LOG.info("Saving trie logs to retain in file (batch {})...", batchNumber); - try { - saveTrieLogsInFile(trieLogKeys, rootWorldStateStorage, batchNumber, batchFileNameBase); + saveTrieLogsInFile(trieLogKeys, rootWorldStateStorage, batchFileName); } catch (IOException e) { LOG.error("Error saving trie logs to file: {}", e.getMessage()); throw new RuntimeException(e); @@ -210,9 +212,8 @@ private static void recreateTrieLogs( final String batchFileNameBase) throws IOException { // process in chunk to avoid OOM - - IdentityHashMap trieLogsToRetain = - readTrieLogsFromFile(batchFileNameBase, batchNumber); + final String batchFileName = batchFileNameBase + "-" + batchNumber; + IdentityHashMap trieLogsToRetain = readTrieLogsFromFile(batchFileName); final int chunkSize = ROCKSDB_MAX_INSERTS_PER_TRANSACTION; List keys = new ArrayList<>(trieLogsToRetain.keySet()); @@ -265,11 +266,10 @@ private static void validatePruneConfiguration(final DataStorageConfiguration co private static void saveTrieLogsInFile( final List trieLogsKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, - final long batchNumber, - final String batchFileNameBase) + final String batchFileName) throws IOException { - File file = new File(batchFileNameBase + "-" + batchNumber); + File file = new File(batchFileName); if (file.exists()) { LOG.error("File already exists, skipping file creation"); return; @@ -285,17 +285,14 @@ private static void saveTrieLogsInFile( } @SuppressWarnings("unchecked") - private static IdentityHashMap readTrieLogsFromFile( - final String batchFileNameBase, final long batchNumber) { + static IdentityHashMap readTrieLogsFromFile(final String batchFileName) { IdentityHashMap trieLogs; - try (FileInputStream fis = new FileInputStream(batchFileNameBase + "-" + batchNumber); + try (FileInputStream fis = new FileInputStream(batchFileName); ObjectInputStream ois = new ObjectInputStream(fis)) { trieLogs = (IdentityHashMap) ois.readObject(); - } catch (IOException | ClassNotFoundException e) { - LOG.error(e.getMessage()); throw new RuntimeException(e); } @@ -303,6 +300,52 @@ private static IdentityHashMap readTrieLogsFromFile( return trieLogs; } + private static void saveTrieLogsAsRlpInFile( + final List trieLogsKeys, + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final String batchFileName) { + File file = new File(batchFileName); + if (file.exists()) { + LOG.error("File already exists, skipping file creation"); + return; + } + + final IdentityHashMap trieLogs = + getTrieLogs(trieLogsKeys, rootWorldStateStorage); + final Bytes rlp = + RLP.encode( + o -> + o.writeList( + trieLogs.entrySet(), (val, out) -> out.writeRaw(Bytes.wrap(val.getValue())))); + try { + Files.write(file.toPath(), rlp.toArrayUnsafe()); + } catch (IOException e) { + LOG.error(e.getMessage()); + throw new RuntimeException(e); + } + } + + static IdentityHashMap readTrieLogsAsRlpFromFile(final String batchFileName) { + try { + final Bytes file = Bytes.wrap(Files.readAllBytes(Path.of(batchFileName))); + final BytesValueRLPInput input = new BytesValueRLPInput(file, false); + + input.enterList(); + final IdentityHashMap trieLogs = new IdentityHashMap<>(); + while (!input.isEndOfCurrentList()) { + final Bytes trieLogBytes = input.currentListAsBytes(); + TrieLogLayer trieLogLayer = + TrieLogFactoryImpl.readFrom(new BytesValueRLPInput(Bytes.wrap(trieLogBytes), false)); + trieLogs.put(trieLogLayer.getBlockHash().toArrayUnsafe(), trieLogBytes.toArrayUnsafe()); + } + input.leaveList(); + + return trieLogs; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + private static IdentityHashMap getTrieLogs( final List trieLogKeys, final BonsaiWorldStateKeyValueStorage rootWorldStateStorage) { IdentityHashMap trieLogsToRetain = new IdentityHashMap<>(); @@ -357,5 +400,25 @@ static void printCount(final PrintWriter out, final TrieLogCount count) { count.total, count.canonicalCount, count.forkCount, count.orphanCount); } + static void importTrieLog( + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, final Path trieLogFilePath) { + + var trieLog = readTrieLogsAsRlpFromFile(trieLogFilePath.toString()); + + var updater = rootWorldStateStorage.updater(); + trieLog.forEach((key, value) -> updater.getTrieLogStorageTransaction().put(key, value)); + updater.getTrieLogStorageTransaction().commit(); + } + + static void exportTrieLog( + final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final List trieLogHash, + final Path directoryPath) + throws IOException { + final String trieLogFile = directoryPath.toString(); + + saveTrieLogsAsRlpInFile(trieLogHash, rootWorldStateStorage, trieLogFile); + } + record TrieLogCount(int total, int canonicalCount, int forkCount, int orphanCount) {} } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index 74e00197f59..e624b5f3856 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -19,6 +19,7 @@ import org.hyperledger.besu.cli.util.VersionProvider; import org.hyperledger.besu.controller.BesuController; +import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; @@ -26,9 +27,11 @@ import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import java.io.IOException; import java.io.PrintWriter; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.List; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; @@ -43,7 +46,12 @@ description = "Manipulate trie logs", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class, - subcommands = {TrieLogSubCommand.CountTrieLog.class, TrieLogSubCommand.PruneTrieLog.class}) + subcommands = { + TrieLogSubCommand.CountTrieLog.class, + TrieLogSubCommand.PruneTrieLog.class, + TrieLogSubCommand.ExportTrieLog.class, + TrieLogSubCommand.ImportTrieLog.class + }) public class TrieLogSubCommand implements Runnable { @SuppressWarnings("UnusedVariable") @@ -123,6 +131,102 @@ public void run() { } } + @Command( + name = "export", + description = "This command exports the trie log of a determined block to a binary file", + mixinStandardHelpOptions = true, + versionProvider = VersionProvider.class) + static class ExportTrieLog implements Runnable { + + @SuppressWarnings("unused") + @ParentCommand + private TrieLogSubCommand parentCommand; + + @SuppressWarnings("unused") + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec + + @CommandLine.Option( + names = "--trie-log-block-hash", + description = + "Comma separated list of hashes from the blocks you want to export the trie logs of", + split = " {0,1}, {0,1}", + arity = "1..*") + private List trieLogBlockHashList; + + @CommandLine.Option( + names = "--trie-log-file-path", + description = "The file you want to export the trie logs to", + arity = "1..1") + private Path trieLogFilePath = null; + + @Override + public void run() { + if (trieLogFilePath == null) { + trieLogFilePath = + Paths.get( + TrieLogSubCommand.parentCommand + .parentCommand + .dataDir() + .resolve("trie-logs.bin") + .toAbsolutePath() + .toString()); + } + + TrieLogContext context = getTrieLogContext(); + + final List listOfBlockHashes = + trieLogBlockHashList.stream().map(Hash::fromHexString).toList(); + + try { + TrieLogHelper.exportTrieLog( + context.rootWorldStateStorage(), listOfBlockHashes, trieLogFilePath); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + @Command( + name = "import", + description = "This command imports a trie log exported by another besu node", + mixinStandardHelpOptions = true, + versionProvider = VersionProvider.class) + static class ImportTrieLog implements Runnable { + + @SuppressWarnings("unused") + @ParentCommand + private TrieLogSubCommand parentCommand; + + @SuppressWarnings("unused") + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec + + @CommandLine.Option( + names = "--trie-log-file-path", + description = "The file you want to import the trie logs from", + arity = "1..1") + private Path trieLogFilePath = null; + + @Override + public void run() { + if (trieLogFilePath == null) { + trieLogFilePath = + Paths.get( + TrieLogSubCommand.parentCommand + .parentCommand + .dataDir() + .resolve("trie-logs.bin") + .toAbsolutePath() + .toString()); + } + + TrieLogContext context = getTrieLogContext(); + + TrieLogHelper.importTrieLog(context.rootWorldStateStorage(), trieLogFilePath); + } + } + record TrieLogContext( DataStorageConfiguration config, BonsaiWorldStateKeyValueStorage rootWorldStateStorage, diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java index 437053afd56..2a63901975b 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java @@ -34,8 +34,8 @@ public void bonsaiTrieLogPruningLimitOption() { dataStorageConfiguration -> assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningLimit()) .isEqualTo(1), - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-pruning-limit", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-pruning-limit", "1"); } @@ -43,8 +43,8 @@ public void bonsaiTrieLogPruningLimitOption() { public void bonsaiTrieLogPruningLimitShouldBePositive() { internalTestFailure( "--Xbonsai-trie-log-pruning-limit=0 must be greater than 0", - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-pruning-limit", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-pruning-limit", "0"); } @@ -54,8 +54,8 @@ public void bonsaiTrieLogRetentionThresholdOption() { dataStorageConfiguration -> assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold()) .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD + 1), - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-retention-threshold", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-retention-threshold", "513"); } @@ -65,8 +65,8 @@ public void bonsaiTrieLogRetentionThresholdOption_boundaryTest() { dataStorageConfiguration -> assertThat(dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold()) .isEqualTo(MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD), - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-retention-threshold", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-retention-threshold", "512"); } @@ -74,8 +74,8 @@ public void bonsaiTrieLogRetentionThresholdOption_boundaryTest() { public void bonsaiTrieLogRetentionThresholdShouldBeAboveMinimum() { internalTestFailure( "--Xbonsai-trie-log-retention-threshold minimum value is 512", - "--Xbonsai-trie-log-pruning-enabled", - "--Xbonsai-trie-log-retention-threshold", + "--Xbonsai-limit-trie-logs-enabled", + "--Xbonsai-trie-logs-retention-threshold", "511"); } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index 0ba575a9ef1..5d4ede7f32e 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.cli.subcommands.storage; +import static java.util.Collections.singletonList; import static org.hyperledger.besu.ethereum.worldstate.DataStorageFormat.BONSAI; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -27,8 +28,11 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -36,11 +40,12 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import org.apache.tuweni.bytes.Bytes; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -56,17 +61,14 @@ class TrieLogHelperTest { @Mock private MutableBlockchain blockchain; - @TempDir static Path dataDir; - - Path test; static BlockHeader blockHeader1; static BlockHeader blockHeader2; static BlockHeader blockHeader3; static BlockHeader blockHeader4; static BlockHeader blockHeader5; - @BeforeAll - public static void setup() throws IOException { + @BeforeEach + public void setup() throws IOException { blockHeader1 = new BlockHeaderTestFixture().number(1).buildHeader(); blockHeader2 = new BlockHeaderTestFixture().number(2).buildHeader(); @@ -78,33 +80,33 @@ public static void setup() throws IOException { new BonsaiWorldStateKeyValueStorage( storageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); + createTrieLog(blockHeader1); + var updater = inMemoryWorldState.updater(); updater .getTrieLogStorageTransaction() - .put(blockHeader1.getHash().toArrayUnsafe(), Bytes.fromHexString("0x01").toArrayUnsafe()); + .put(blockHeader1.getHash().toArrayUnsafe(), createTrieLog(blockHeader1)); updater .getTrieLogStorageTransaction() - .put(blockHeader2.getHash().toArrayUnsafe(), Bytes.fromHexString("0x02").toArrayUnsafe()); + .put(blockHeader2.getHash().toArrayUnsafe(), createTrieLog(blockHeader2)); updater .getTrieLogStorageTransaction() - .put(blockHeader3.getHash().toArrayUnsafe(), Bytes.fromHexString("0x03").toArrayUnsafe()); + .put(blockHeader3.getHash().toArrayUnsafe(), createTrieLog(blockHeader3)); updater .getTrieLogStorageTransaction() - .put(blockHeader4.getHash().toArrayUnsafe(), Bytes.fromHexString("0x04").toArrayUnsafe()); + .put(blockHeader4.getHash().toArrayUnsafe(), createTrieLog(blockHeader4)); updater .getTrieLogStorageTransaction() - .put(blockHeader5.getHash().toArrayUnsafe(), Bytes.fromHexString("0x05").toArrayUnsafe()); + .put(blockHeader5.getHash().toArrayUnsafe(), createTrieLog(blockHeader5)); updater.getTrieLogStorageTransaction().commit(); } - @BeforeEach - void createDirectory() throws IOException { - Files.createDirectories(dataDir.resolve("database")); - } - - @AfterEach - void deleteDirectory() throws IOException { - Files.deleteIfExists(dataDir.resolve("database")); + private static byte[] createTrieLog(final BlockHeader blockHeader) { + TrieLogLayer trieLogLayer = new TrieLogLayer(); + trieLogLayer.setBlockHash(blockHeader.getBlockHash()); + final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput(); + TrieLogFactoryImpl.writeTo(trieLogLayer, rlpLog); + return rlpLog.encoded().toArrayUnsafe(); } void mockBlockchainBase() { @@ -114,7 +116,8 @@ void mockBlockchainBase() { } @Test - public void prune() { + public void prune(final @TempDir Path dataDir) throws IOException { + Files.createDirectories(dataDir.resolve("database")); DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() @@ -134,14 +137,11 @@ public void prune() { // assert trie logs that will be pruned exist before prune call assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), - Bytes.fromHexString("0x01").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), createTrieLog(blockHeader1)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), - Bytes.fromHexString("0x02").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), createTrieLog(blockHeader2)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), - Bytes.fromHexString("0x03").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3)); TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir); @@ -151,18 +151,15 @@ public void prune() { // assert retained trie logs are in the DB assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), - Bytes.fromHexString("0x03").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), - Bytes.fromHexString("0x04").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), createTrieLog(blockHeader4)); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), - Bytes.fromHexString("0x05").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), createTrieLog(blockHeader5)); } @Test - public void cantPruneIfNoFinalizedIsFound() { + public void cantPruneIfNoFinalizedIsFound(final @TempDir Path dataDir) { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) @@ -184,7 +181,7 @@ public void cantPruneIfNoFinalizedIsFound() { } @Test - public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength() { + public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength(final @TempDir Path dataDir) { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() .dataStorageFormat(BONSAI) @@ -205,7 +202,7 @@ public void cantPruneIfUserRetainsMoreLayerThanExistingChainLength() { } @Test - public void cantPruneIfUserRequiredFurtherThanFinalized() { + public void cantPruneIfUserRequiredFurtherThanFinalized(final @TempDir Path dataDir) { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() @@ -227,8 +224,7 @@ public void cantPruneIfUserRequiredFurtherThanFinalized() { } @Test - public void exceptionWhileSavingFileStopsPruneProcess() throws IOException { - Files.delete(dataDir.resolve("database")); + public void exceptionWhileSavingFileStopsPruneProcess(final @TempDir Path dataDir) { DataStorageConfiguration dataStorageConfiguration = ImmutableDataStorageConfiguration.builder() @@ -244,23 +240,121 @@ public void exceptionWhileSavingFileStopsPruneProcess() throws IOException { assertThrows( RuntimeException.class, () -> - TrieLogHelper.prune(dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir)); + TrieLogHelper.prune( + dataStorageConfiguration, + inMemoryWorldState, + blockchain, + dataDir.resolve("unknownPath"))); // assert all trie logs are still in the DB assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), - Bytes.fromHexString("0x01").toArrayUnsafe()); + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get(), createTrieLog(blockHeader1)); + assertArrayEquals( + inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), createTrieLog(blockHeader2)); + assertArrayEquals( + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), createTrieLog(blockHeader3)); + assertArrayEquals( + inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), createTrieLog(blockHeader4)); + assertArrayEquals( + inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), createTrieLog(blockHeader5)); + } + + @Test + public void exportedTrieMatchesDbTrieLog(final @TempDir Path dataDir) throws IOException { + TrieLogHelper.exportTrieLog( + inMemoryWorldState, + singletonList(blockHeader1.getHash()), + dataDir.resolve("trie-log-dump")); + + var trieLog = + TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) + .entrySet() + .stream() + .findFirst() + .get(); + + assertArrayEquals(trieLog.getKey(), blockHeader1.getHash().toArrayUnsafe()); + assertArrayEquals( + trieLog.getValue(), inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); + } + + @Test + public void exportedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) throws IOException { + TrieLogHelper.exportTrieLog( + inMemoryWorldState, + List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()), + dataDir.resolve("trie-log-dump")); + + var trieLogs = + TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()) + .entrySet() + .stream() + .collect(Collectors.toMap(e -> Bytes.wrap(e.getKey()), Map.Entry::getValue)); + + assertArrayEquals( + trieLogs.get(blockHeader1.getHash()), + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get(), - Bytes.fromHexString("0x02").toArrayUnsafe()); + trieLogs.get(blockHeader2.getHash()), + inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get()); + assertArrayEquals( + trieLogs.get(blockHeader3.getHash()), + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()); + } + + @Test + public void importedTrieLogMatchesDbTrieLog(final @TempDir Path dataDir) throws IOException { + StorageProvider tempStorageProvider = new InMemoryKeyValueStorageProvider(); + BonsaiWorldStateKeyValueStorage inMemoryWorldState2 = + new BonsaiWorldStateKeyValueStorage( + tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); + + TrieLogHelper.exportTrieLog( + inMemoryWorldState, + singletonList(blockHeader1.getHash()), + dataDir.resolve("trie-log-dump")); + + var trieLog = + TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()); + var updater = inMemoryWorldState2.updater(); + + trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v)); + + updater.getTrieLogStorageTransaction().commit(); + + assertArrayEquals( + inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get(), + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); + } + + @Test + public void importedMultipleTriesMatchDbTrieLogs(final @TempDir Path dataDir) throws IOException { + StorageProvider tempStorageProvider = new InMemoryKeyValueStorageProvider(); + BonsaiWorldStateKeyValueStorage inMemoryWorldState2 = + new BonsaiWorldStateKeyValueStorage( + tempStorageProvider, new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); + + TrieLogHelper.exportTrieLog( + inMemoryWorldState, + List.of(blockHeader1.getHash(), blockHeader2.getHash(), blockHeader3.getHash()), + dataDir.resolve("trie-log-dump")); + + var trieLog = + TrieLogHelper.readTrieLogsAsRlpFromFile(dataDir.resolve("trie-log-dump").toString()); + var updater = inMemoryWorldState2.updater(); + + trieLog.forEach((k, v) -> updater.getTrieLogStorageTransaction().put(k, v)); + + updater.getTrieLogStorageTransaction().commit(); + assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get(), - Bytes.fromHexString("0x03").toArrayUnsafe()); + inMemoryWorldState2.getTrieLog(blockHeader1.getHash()).get(), + inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get()); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader4.getHash()).get(), - Bytes.fromHexString("0x04").toArrayUnsafe()); + inMemoryWorldState2.getTrieLog(blockHeader2.getHash()).get(), + inMemoryWorldState.getTrieLog(blockHeader2.getHash()).get()); assertArrayEquals( - inMemoryWorldState.getTrieLog(blockHeader5.getHash()).get(), - Bytes.fromHexString("0x05").toArrayUnsafe()); + inMemoryWorldState2.getTrieLog(blockHeader3.getHash()).get(), + inMemoryWorldState.getTrieLog(blockHeader3.getHash()).get()); } }