From e7ac9e5309aa045c93b42afd0e9483b30a8f9d2e Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 7 Mar 2023 11:48:15 +0100 Subject: [PATCH 1/9] Fix for backward sync wrongly thinking it is done after a Besu restart Signed-off-by: Fabio Di Fabio --- .../eth/sync/backwardsync/BackwardSyncContext.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java index 6574d48e67a..434940f0a02 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java @@ -116,9 +116,7 @@ public synchronized void updateHead(final Hash headHash) { } public synchronized CompletableFuture syncBackwardsUntil(final Hash newBlockHash) { - if (!isTrusted(newBlockHash)) { - backwardChain.addNewHash(newBlockHash); - } + backwardChain.addNewHash(newBlockHash); final Status status = getOrStartSyncSession(); backwardChain @@ -133,9 +131,7 @@ public synchronized CompletableFuture syncBackwardsUntil(final Block newPi backwardChain.appendTrustedBlock(newPivot); } - final Status status = getOrStartSyncSession(); - status.updateTargetHeight(newPivot.getHeader().getNumber()); - return status.currentFuture; + return syncBackwardsUntil(newPivot.getHash()); } private Status getOrStartSyncSession() { @@ -153,7 +149,7 @@ private boolean isTrusted(final Hash hash) { if (backwardChain.isTrusted(hash)) { LOG.atDebug() .setMessage( - "not fetching or appending hash {} to backwards sync since it is present in successors") + "Not fetching or appending hash {} to backwards sync since it is present in successors") .addArgument(hash::toHexString) .log(); return true; From dae939e91841abf92b1eb7431e424fb0df30cb1c Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 7 Mar 2023 12:05:30 +0100 Subject: [PATCH 2/9] Add CHANGELOG entry Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 004e9fa6f33..9adeaeb66ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## Next release + +### Additions and Improvements + +### Bug Fixes +- Fix for backward sync wrongly thinking it is done after a restart [#5182](https://github.com/hyperledger/besu/pull/5182) + ## 23.1.1 This update is required for the Goerli Shanghai/Capella upgrade and recommended for all Mainnet users. If you use Besu on Goerli, update to 23.1.1. If you previously used 23.1.1-RC1, update to test 23.1.1 on Goerli. From c2c88b142d644f8fa87cabec04241168e7658242 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 7 Mar 2023 16:59:12 +0100 Subject: [PATCH 3/9] Remember backward sync status accross restarts Signed-off-by: Fabio Di Fabio --- .../eth/sync/backwardsync/BackwardChain.java | 79 +++++++++++++------ ...orithm.java => BackwardSyncAlgorithm.java} | 8 +- .../backwardsync/BackwardSyncContext.java | 4 +- .../sync/backwardsync/BackwardSyncStep.java | 2 +- .../sync/backwardsync/StringConvertor.java | 36 +++++++++ .../backwardsync/BackwardSyncAlgSpec.java | 7 +- .../InMemoryBackwardChainTest.java | 6 +- 7 files changed, 106 insertions(+), 36 deletions(-) rename ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/{BackwardsSyncAlgorithm.java => BackwardSyncAlgorithm.java} (95%) create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/StringConvertor.java diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java index 7e2588ad65f..5df91d06526 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java @@ -24,6 +24,7 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; +import java.nio.charset.StandardCharsets; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.List; @@ -35,20 +36,28 @@ public class BackwardChain { private static final Logger LOG = getLogger(BackwardChain.class); + private final static String FIRST_STORED_ANCESTOR_KEY = "firstStoredAncestor"; + private final static String LAST_STORED_PIVOT_KEY = "lastStoredPivot"; + private final GenericKeyValueStorageFacade headers; private final GenericKeyValueStorageFacade blocks; private final GenericKeyValueStorageFacade chainStorage; - private Optional firstStoredAncestor = Optional.empty(); - private Optional lastStoredPivot = Optional.empty(); + private final GenericKeyValueStorageFacade variablesStorage; + private Optional firstStoredAncestor; + private Optional lastStoredPivot; private final Queue hashesToAppend = new ArrayDeque<>(); public BackwardChain( final GenericKeyValueStorageFacade headersStorage, final GenericKeyValueStorageFacade blocksStorage, - final GenericKeyValueStorageFacade chainStorage) { + final GenericKeyValueStorageFacade chainStorage, + final GenericKeyValueStorageFacade variablesStorage) { this.headers = headersStorage; this.blocks = blocksStorage; this.chainStorage = chainStorage; + this.variablesStorage = variablesStorage; + firstStoredAncestor = variablesStorage.get(FIRST_STORED_ANCESTOR_KEY); + lastStoredPivot = variablesStorage.get(LAST_STORED_PIVOT_KEY); } public static BackwardChain from( @@ -68,7 +77,13 @@ public static BackwardChain from( Hash::toArrayUnsafe, new HashConvertor(), storageProvider.getStorageBySegmentIdentifier( - KeyValueSegmentIdentifier.BACKWARD_SYNC_CHAIN))); + KeyValueSegmentIdentifier.BACKWARD_SYNC_CHAIN)), + new GenericKeyValueStorageFacade<>( + key -> key.getBytes(StandardCharsets.UTF_8), + BlocksHeadersConvertor.of(blockHeaderFunctions), + storageProvider.getStorageBySegmentIdentifier( + KeyValueSegmentIdentifier.BACKWARD_SYNC_CHAIN)) + ); } public synchronized Optional getFirstAncestorHeader() { @@ -86,22 +101,39 @@ public synchronized List getFirstNAncestorHeaders(final int size) { } public synchronized void prependAncestorsHeader(final BlockHeader blockHeader) { - if (firstStoredAncestor.isEmpty()) { - firstStoredAncestor = Optional.of(blockHeader); - lastStoredPivot = Optional.of(blockHeader); + prependAncestorsHeader(blockHeader, false); + } + public synchronized void prependAncestorsHeader(final BlockHeader blockHeader, final boolean alreadyStored) { + if(!alreadyStored) { headers.put(blockHeader.getHash(), blockHeader); - return; } - final BlockHeader firstHeader = firstStoredAncestor.get(); - headers.put(blockHeader.getHash(), blockHeader); - chainStorage.put(blockHeader.getHash(), firstHeader.getHash()); - firstStoredAncestor = Optional.of(blockHeader); - LOG.atDebug() - .setMessage("Added header {} to backward chain led by pivot {} on height {}") - .addArgument(blockHeader::toLogString) - .addArgument(() -> lastStoredPivot.orElseThrow().toLogString()) - .addArgument(firstHeader::getNumber) - .log(); + + if (firstStoredAncestor.isEmpty()) { + updateLastStorePivot(Optional.of(blockHeader)); + } else { + final BlockHeader firstHeader = firstStoredAncestor.get(); + chainStorage.put(blockHeader.getHash(), firstHeader.getHash()); + LOG.atDebug() + .setMessage("Added header {} to backward chain led by pivot {} on height {}") + .addArgument(blockHeader::toLogString) + .addArgument(() -> lastStoredPivot.orElseThrow().toLogString()) + .addArgument(firstHeader::getNumber) + .log(); + } + + updateFirstStoredAncestor(Optional.of(blockHeader)); + } + + private void updateFirstStoredAncestor(final Optional maybeHeader) { + maybeHeader.ifPresentOrElse(header -> + variablesStorage.put(FIRST_STORED_ANCESTOR_KEY, header), () -> variablesStorage.drop(FIRST_STORED_ANCESTOR_KEY)); + firstStoredAncestor = maybeHeader; + } + + private void updateLastStorePivot(final Optional maybeHeader) { + maybeHeader.ifPresentOrElse(header -> + variablesStorage.put(LAST_STORED_PIVOT_KEY, header), () -> variablesStorage.drop(LAST_STORED_PIVOT_KEY)); + lastStoredPivot = maybeHeader; } public synchronized Optional getPivot() { @@ -118,9 +150,9 @@ public synchronized void dropFirstHeader() { headers.drop(firstStoredAncestor.get().getHash()); final Optional hash = chainStorage.get(firstStoredAncestor.get().getHash()); chainStorage.drop(firstStoredAncestor.get().getHash()); - firstStoredAncestor = hash.flatMap(headers::get); + updateFirstStoredAncestor(hash.flatMap(headers::get)); if (firstStoredAncestor.isEmpty()) { - lastStoredPivot = Optional.empty(); + updateLastStorePivot(Optional.empty()); } } @@ -129,7 +161,7 @@ public synchronized void appendTrustedBlock(final Block newPivot) { headers.put(newPivot.getHash(), newPivot.getHeader()); blocks.put(newPivot.getHash(), newPivot); if (lastStoredPivot.isEmpty()) { - firstStoredAncestor = Optional.of(newPivot.getHeader()); + updateFirstStoredAncestor(Optional.of(newPivot.getHeader())); } else { if (newPivot.getHeader().getParentHash().equals(lastStoredPivot.get().getHash())) { LOG.atDebug() @@ -140,14 +172,14 @@ public synchronized void appendTrustedBlock(final Block newPivot) { .log(); chainStorage.put(lastStoredPivot.get().getHash(), newPivot.getHash()); } else { - firstStoredAncestor = Optional.of(newPivot.getHeader()); + updateFirstStoredAncestor(Optional.of(newPivot.getHeader())); LOG.atDebug() .setMessage("Re-pivoting to new target block {}") .addArgument(newPivot::toLogString) .log(); } } - lastStoredPivot = Optional.of(newPivot.getHeader()); + updateLastStorePivot(Optional.of(newPivot.getHeader())); } public synchronized boolean isTrusted(final Hash hash) { @@ -162,6 +194,7 @@ public synchronized void clear() { blocks.clear(); headers.clear(); chainStorage.clear(); + variablesStorage.clear(); firstStoredAncestor = Optional.empty(); lastStoredPivot = Optional.empty(); hashesToAppend.clear(); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardsSyncAlgorithm.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgorithm.java similarity index 95% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardsSyncAlgorithm.java rename to ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgorithm.java index 3b06eb4424c..f569af20643 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardsSyncAlgorithm.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgorithm.java @@ -35,8 +35,8 @@ import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; -public class BackwardsSyncAlgorithm implements BesuEvents.InitialSyncCompletionListener { - private static final Logger LOG = getLogger(BackwardsSyncAlgorithm.class); +public class BackwardSyncAlgorithm implements BesuEvents.InitialSyncCompletionListener { + private static final Logger LOG = getLogger(BackwardSyncAlgorithm.class); private final BackwardSyncContext context; private final FinalBlockConfirmation finalBlockConfirmation; @@ -44,7 +44,7 @@ public class BackwardsSyncAlgorithm implements BesuEvents.InitialSyncCompletionL new AtomicReference<>(new CountDownLatch(1)); private volatile boolean finished = false; - public BackwardsSyncAlgorithm( + public BackwardSyncAlgorithm( final BackwardSyncContext context, final FinalBlockConfirmation finalBlockConfirmation) { this.context = context; this.finalBlockConfirmation = finalBlockConfirmation; @@ -64,7 +64,7 @@ public CompletableFuture pickNextStep() { return executeSyncStep(firstHash.get()) .thenAccept( result -> { - LOG.info("Backward sync target block is {}", result.toLogString()); + LOG.atDebug().setMessage("Backward sync target block is {}").addArgument(result::toLogString).log(); context.getBackwardChain().removeFromHashToAppend(firstHash.get()); context.getStatus().updateTargetHeight(result.getHeader().getNumber()); }); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java index 434940f0a02..e774398b452 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java @@ -131,7 +131,7 @@ public synchronized CompletableFuture syncBackwardsUntil(final Block newPi backwardChain.appendTrustedBlock(newPivot); } - return syncBackwardsUntil(newPivot.getHash()); + return syncBackwardsUntil(newPivot.getHeader().getParentHash()); } private Status getOrStartSyncSession() { @@ -233,7 +233,7 @@ private Optional extractBackwardSyncException(final Throw @VisibleForTesting CompletableFuture prepareBackwardSyncFuture() { final MutableBlockchain blockchain = getProtocolContext().getBlockchain(); - return new BackwardsSyncAlgorithm( + return new BackwardSyncAlgorithm( this, FinalBlockConfirmation.confirmationChain( FinalBlockConfirmation.genesisConfirmation(blockchain), diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStep.java index 16478366f2b..10315a60102 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStep.java @@ -48,7 +48,7 @@ protected Hash possibleRestoreOldNodes(final BlockHeader firstAncestor) { Hash lastHash = firstAncestor.getParentHash(); Optional iterator = backwardChain.getHeader(lastHash); while (iterator.isPresent()) { - backwardChain.prependAncestorsHeader(iterator.get()); + backwardChain.prependAncestorsHeader(iterator.get(), true); lastHash = iterator.get().getParentHash(); iterator = backwardChain.getHeader(lastHash); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/StringConvertor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/StringConvertor.java new file mode 100644 index 00000000000..45aa3c0afe0 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/StringConvertor.java @@ -0,0 +1,36 @@ +/* + * + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.eth.sync.backwardsync; + +import org.apache.tuweni.bytes.Bytes32; +import org.hyperledger.besu.datatypes.Hash; + +import java.nio.charset.StandardCharsets; + +public class StringConvertor implements ValueConvertor { + @Override + public String fromBytes(final byte[] bytes) { + + return new String(bytes, StandardCharsets.UTF_8); + } + + @Override + public byte[] toBytes(final String value) { + return value.getBytes(StandardCharsets.UTF_8); + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java index 69ab5470f3a..8b607f68a55 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java @@ -66,7 +66,8 @@ public class BackwardSyncAlgSpec { @Captor ArgumentCaptor ttdCaptor; @Captor ArgumentCaptor completionCaptor; - @InjectMocks BackwardsSyncAlgorithm algorithm; + @InjectMocks + BackwardSyncAlgorithm algorithm; @Mock private Hash hash; private static final BlockDataGenerator blockDataGenerator = new BlockDataGenerator(); @@ -95,7 +96,7 @@ public void setUp() throws Exception { algorithm = Mockito.spy( - new BackwardsSyncAlgorithm( + new BackwardSyncAlgorithm( context, FinalBlockConfirmation.confirmationChain( FinalBlockConfirmation.genesisConfirmation(localBlockchain), @@ -292,7 +293,7 @@ public void shouldFailIfADifferentGenesisIsReached() { doReturn(backwardChain).when(context).getBackwardChain(); algorithm = Mockito.spy( - new BackwardsSyncAlgorithm( + new BackwardSyncAlgorithm( context, FinalBlockConfirmation.genesisConfirmation(otherLocalBlockchain))); assertThatThrownBy(() -> algorithm.pickNextStep()) .isInstanceOf(BackwardSyncException.class) diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java index 3fabb7bad83..e8464642a95 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java @@ -138,15 +138,15 @@ public void shouldAddHeaderToQueue() { firstHash = backwardChain.getFirstHashToAppend(); assertThat(firstHash).isPresent(); assertThat(firstHash.orElseThrow()).isEqualTo(blocks.get(7).getHash()); - backwardChain.removeFromHashToAppend(firstHash.get()); + backwardChain.processing(firstHash.get()); firstHash = backwardChain.getFirstHashToAppend(); assertThat(firstHash).isPresent(); assertThat(firstHash.orElseThrow()).isEqualTo(blocks.get(9).getHash()); - backwardChain.removeFromHashToAppend(firstHash.get()); + backwardChain.processing(firstHash.get()); firstHash = backwardChain.getFirstHashToAppend(); assertThat(firstHash).isPresent(); assertThat(firstHash.orElseThrow()).isEqualTo(blocks.get(11).getHash()); - backwardChain.removeFromHashToAppend(firstHash.get()); + backwardChain.processing(firstHash.get()); } @Test From 42f95699599f90ef665cd320de44892ab15afdfc Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 7 Mar 2023 11:48:15 +0100 Subject: [PATCH 4/9] Revert "Fix for backward sync wrongly thinking it is done after a Besu restart" This reverts commit e7ac9e5309aa045c93b42afd0e9483b30a8f9d2e. Signed-off-by: Fabio Di Fabio # Conflicts: # ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java --- .../eth/sync/backwardsync/BackwardSyncContext.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java index e774398b452..7b87a476272 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java @@ -116,7 +116,9 @@ public synchronized void updateHead(final Hash headHash) { } public synchronized CompletableFuture syncBackwardsUntil(final Hash newBlockHash) { - backwardChain.addNewHash(newBlockHash); + if (!isTrusted(newBlockHash)) { + backwardChain.addNewHash(newBlockHash); + } final Status status = getOrStartSyncSession(); backwardChain @@ -131,7 +133,9 @@ public synchronized CompletableFuture syncBackwardsUntil(final Block newPi backwardChain.appendTrustedBlock(newPivot); } - return syncBackwardsUntil(newPivot.getHeader().getParentHash()); + final Status status = getOrStartSyncSession(); + status.updateTargetHeight(newPivot.getHeader().getNumber()); + return status.currentFuture; } private Status getOrStartSyncSession() { From 667e82ad852ee2769285bef90a53fb082d503636 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 7 Mar 2023 17:17:52 +0100 Subject: [PATCH 5/9] Persist backward sync status to support resuming across restarts Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 2 +- .../eth/sync/backwardsync/BackwardChain.java | 65 +++++++++++++------ .../backwardsync/BackwardSyncAlgorithm.java | 5 +- .../sync/backwardsync/StringConvertor.java | 36 ---------- .../backwardsync/BackwardSyncAlgSpec.java | 3 +- 5 files changed, 50 insertions(+), 61 deletions(-) delete mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/StringConvertor.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 9adeaeb66ef..e2419bbadb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Additions and Improvements ### Bug Fixes -- Fix for backward sync wrongly thinking it is done after a restart [#5182](https://github.com/hyperledger/besu/pull/5182) +- Persist backward sync status to support resuming across restarts [#5182](https://github.com/hyperledger/besu/pull/5182) ## 23.1.1 This update is required for the Goerli Shanghai/Capella upgrade and recommended for all Mainnet users. If you use Besu on Goerli, update to 23.1.1. If you previously used 23.1.1-RC1, update to test 23.1.1 on Goerli. diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java index 5df91d06526..466176484ed 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java @@ -36,8 +36,8 @@ public class BackwardChain { private static final Logger LOG = getLogger(BackwardChain.class); - private final static String FIRST_STORED_ANCESTOR_KEY = "firstStoredAncestor"; - private final static String LAST_STORED_PIVOT_KEY = "lastStoredPivot"; + private static final String FIRST_STORED_ANCESTOR_KEY = "firstStoredAncestor"; + private static final String LAST_STORED_PIVOT_KEY = "lastStoredPivot"; private final GenericKeyValueStorageFacade headers; private final GenericKeyValueStorageFacade blocks; @@ -56,8 +56,28 @@ public BackwardChain( this.blocks = blocksStorage; this.chainStorage = chainStorage; this.variablesStorage = variablesStorage; - firstStoredAncestor = variablesStorage.get(FIRST_STORED_ANCESTOR_KEY); - lastStoredPivot = variablesStorage.get(LAST_STORED_PIVOT_KEY); + firstStoredAncestor = + variablesStorage + .get(FIRST_STORED_ANCESTOR_KEY) + .map( + header -> { + LOG.atDebug() + .setMessage(FIRST_STORED_ANCESTOR_KEY + " loaded from storage with value {}") + .addArgument(header::toLogString) + .log(); + return header; + }); + lastStoredPivot = + variablesStorage + .get(LAST_STORED_PIVOT_KEY) + .map( + header -> { + LOG.atDebug() + .setMessage(LAST_STORED_PIVOT_KEY + " loaded from storage with value {}") + .addArgument(header::toLogString) + .log(); + return header; + }); } public static BackwardChain from( @@ -78,12 +98,11 @@ public static BackwardChain from( new HashConvertor(), storageProvider.getStorageBySegmentIdentifier( KeyValueSegmentIdentifier.BACKWARD_SYNC_CHAIN)), - new GenericKeyValueStorageFacade<>( - key -> key.getBytes(StandardCharsets.UTF_8), - BlocksHeadersConvertor.of(blockHeaderFunctions), - storageProvider.getStorageBySegmentIdentifier( - KeyValueSegmentIdentifier.BACKWARD_SYNC_CHAIN)) - ); + new GenericKeyValueStorageFacade<>( + key -> key.getBytes(StandardCharsets.UTF_8), + BlocksHeadersConvertor.of(blockHeaderFunctions), + storageProvider.getStorageBySegmentIdentifier( + KeyValueSegmentIdentifier.BACKWARD_SYNC_CHAIN))); } public synchronized Optional getFirstAncestorHeader() { @@ -103,8 +122,10 @@ public synchronized List getFirstNAncestorHeaders(final int size) { public synchronized void prependAncestorsHeader(final BlockHeader blockHeader) { prependAncestorsHeader(blockHeader, false); } - public synchronized void prependAncestorsHeader(final BlockHeader blockHeader, final boolean alreadyStored) { - if(!alreadyStored) { + + public synchronized void prependAncestorsHeader( + final BlockHeader blockHeader, final boolean alreadyStored) { + if (!alreadyStored) { headers.put(blockHeader.getHash(), blockHeader); } @@ -114,25 +135,27 @@ public synchronized void prependAncestorsHeader(final BlockHeader blockHeader, f final BlockHeader firstHeader = firstStoredAncestor.get(); chainStorage.put(blockHeader.getHash(), firstHeader.getHash()); LOG.atDebug() - .setMessage("Added header {} to backward chain led by pivot {} on height {}") - .addArgument(blockHeader::toLogString) - .addArgument(() -> lastStoredPivot.orElseThrow().toLogString()) - .addArgument(firstHeader::getNumber) - .log(); + .setMessage("Added header {} to backward chain led by pivot {} on height {}") + .addArgument(blockHeader::toLogString) + .addArgument(() -> lastStoredPivot.orElseThrow().toLogString()) + .addArgument(firstHeader::getNumber) + .log(); } updateFirstStoredAncestor(Optional.of(blockHeader)); } private void updateFirstStoredAncestor(final Optional maybeHeader) { - maybeHeader.ifPresentOrElse(header -> - variablesStorage.put(FIRST_STORED_ANCESTOR_KEY, header), () -> variablesStorage.drop(FIRST_STORED_ANCESTOR_KEY)); + maybeHeader.ifPresentOrElse( + header -> variablesStorage.put(FIRST_STORED_ANCESTOR_KEY, header), + () -> variablesStorage.drop(FIRST_STORED_ANCESTOR_KEY)); firstStoredAncestor = maybeHeader; } private void updateLastStorePivot(final Optional maybeHeader) { - maybeHeader.ifPresentOrElse(header -> - variablesStorage.put(LAST_STORED_PIVOT_KEY, header), () -> variablesStorage.drop(LAST_STORED_PIVOT_KEY)); + maybeHeader.ifPresentOrElse( + header -> variablesStorage.put(LAST_STORED_PIVOT_KEY, header), + () -> variablesStorage.drop(LAST_STORED_PIVOT_KEY)); lastStoredPivot = maybeHeader; } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgorithm.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgorithm.java index f569af20643..731291dc1ea 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgorithm.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgorithm.java @@ -64,7 +64,10 @@ public CompletableFuture pickNextStep() { return executeSyncStep(firstHash.get()) .thenAccept( result -> { - LOG.atDebug().setMessage("Backward sync target block is {}").addArgument(result::toLogString).log(); + LOG.atDebug() + .setMessage("Backward sync target block is {}") + .addArgument(result::toLogString) + .log(); context.getBackwardChain().removeFromHashToAppend(firstHash.get()); context.getStatus().updateTargetHeight(result.getHeader().getNumber()); }); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/StringConvertor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/StringConvertor.java deleted file mode 100644 index 45aa3c0afe0..00000000000 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/StringConvertor.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * - * Copyright Hyperledger Besu Contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ - -package org.hyperledger.besu.ethereum.eth.sync.backwardsync; - -import org.apache.tuweni.bytes.Bytes32; -import org.hyperledger.besu.datatypes.Hash; - -import java.nio.charset.StandardCharsets; - -public class StringConvertor implements ValueConvertor { - @Override - public String fromBytes(final byte[] bytes) { - - return new String(bytes, StandardCharsets.UTF_8); - } - - @Override - public byte[] toBytes(final String value) { - return value.getBytes(StandardCharsets.UTF_8); - } -} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java index 8b607f68a55..f53b32cd54f 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java @@ -66,8 +66,7 @@ public class BackwardSyncAlgSpec { @Captor ArgumentCaptor ttdCaptor; @Captor ArgumentCaptor completionCaptor; - @InjectMocks - BackwardSyncAlgorithm algorithm; + @InjectMocks BackwardSyncAlgorithm algorithm; @Mock private Hash hash; private static final BlockDataGenerator blockDataGenerator = new BlockDataGenerator(); From 11957c1943ba7e04d11a723005b75356c104e311 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 7 Mar 2023 18:13:15 +0100 Subject: [PATCH 6/9] Fix tests Signed-off-by: Fabio Di Fabio --- .../MergeBesuControllerBuilderTest.java | 2 ++ .../backwardsync/BackwardSyncContextTest.java | 8 +++++++- .../sync/backwardsync/BackwardSyncStepTest.java | 10 ++++++++-- .../sync/backwardsync/ForwardSyncStepTest.java | 10 ++++++++-- .../backwardsync/InMemoryBackwardChainTest.java | 15 +++++++++++---- 5 files changed, 36 insertions(+), 9 deletions(-) diff --git a/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java index 5a29193e4de..e631e0b57f5 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java @@ -121,6 +121,8 @@ public void setup() { .thenReturn( new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new MainnetBlockHeaderFunctions())); + when(storageProvider.getStorageBySegmentIdentifier(any())) + .thenReturn(new InMemoryKeyValueStorage()); when(synchronizerConfiguration.getDownloaderParallelism()).thenReturn(1); when(synchronizerConfiguration.getTransactionsParallelism()).thenReturn(1); when(synchronizerConfiguration.getComputationParallelism()).thenReturn(1); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java index a85df3cdb2c..2fa03304a29 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java @@ -52,6 +52,7 @@ import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -193,7 +194,12 @@ public static BackwardChain inMemoryBackwardChain() { final GenericKeyValueStorageFacade chainStorage = new GenericKeyValueStorageFacade<>( Hash::toArrayUnsafe, new HashConvertor(), new InMemoryKeyValueStorage()); - return new BackwardChain(headersStorage, blocksStorage, chainStorage); + final GenericKeyValueStorageFacade variablesStorage = + new GenericKeyValueStorageFacade<>( + key -> key.getBytes(StandardCharsets.UTF_8), + BlocksHeadersConvertor.of(new MainnetBlockHeaderFunctions()), + new InMemoryKeyValueStorage()); + return new BackwardChain(headersStorage, blocksStorage, chainStorage, variablesStorage); } @Test diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStepTest.java index d9df79a402c..bb2709873d5 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStepTest.java @@ -40,6 +40,7 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.concurrent.CompletableFuture; import javax.annotation.Nonnull; @@ -73,6 +74,7 @@ public class BackwardSyncStepTest { GenericKeyValueStorageFacade headersStorage; GenericKeyValueStorageFacade blocksStorage; GenericKeyValueStorageFacade chainStorage; + GenericKeyValueStorageFacade variablesStorage; @Before public void setup() { @@ -86,10 +88,14 @@ public void setup() { Hash::toArrayUnsafe, new BlocksConvertor(new MainnetBlockHeaderFunctions()), new InMemoryKeyValueStorage()); - chainStorage = new GenericKeyValueStorageFacade<>( Hash::toArrayUnsafe, new HashConvertor(), new InMemoryKeyValueStorage()); + variablesStorage = + new GenericKeyValueStorageFacade<>( + key -> key.getBytes(StandardCharsets.UTF_8), + new BlocksHeadersConvertor(new MainnetBlockHeaderFunctions()), + new InMemoryKeyValueStorage()); Block genesisBlock = blockDataGenerator.genesisBlock(); remoteBlockchain = createInMemoryBlockchain(genesisBlock); @@ -234,7 +240,7 @@ private BackwardChain createBackwardChain(final int from, final int until) { @Nonnull private BackwardChain createBackwardChain(final int number) { final BackwardChain backwardChain = - new BackwardChain(headersStorage, blocksStorage, chainStorage); + new BackwardChain(headersStorage, blocksStorage, chainStorage, variablesStorage); backwardChain.appendTrustedBlock(remoteBlockchain.getBlockByNumber(number).orElseThrow()); return backwardChain; } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java index 2b6831d71e7..7705d71afb3 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java @@ -41,6 +41,7 @@ import org.hyperledger.besu.ethereum.referencetests.ReferenceTestWorldState; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -73,8 +74,8 @@ public class ForwardSyncStepTest { private MutableBlockchain localBlockchain; GenericKeyValueStorageFacade headersStorage; GenericKeyValueStorageFacade blocksStorage; - GenericKeyValueStorageFacade chainStorage; + GenericKeyValueStorageFacade variablesStorage; @Before public void setup() { @@ -91,6 +92,11 @@ public void setup() { chainStorage = new GenericKeyValueStorageFacade<>( Hash::toArrayUnsafe, new HashConvertor(), new InMemoryKeyValueStorage()); + variablesStorage = + new GenericKeyValueStorageFacade<>( + key -> key.getBytes(StandardCharsets.UTF_8), + new BlocksHeadersConvertor(new MainnetBlockHeaderFunctions()), + new InMemoryKeyValueStorage()); Block genesisBlock = blockDataGenerator.genesisBlock(); remoteBlockchain = createInMemoryBlockchain(genesisBlock); @@ -197,7 +203,7 @@ private BackwardChain createBackwardChain(final int from, final int until) { @Nonnull private BackwardChain backwardChainFromBlock(final int number) { final BackwardChain backwardChain = - new BackwardChain(headersStorage, blocksStorage, chainStorage); + new BackwardChain(headersStorage, blocksStorage, chainStorage, variablesStorage); backwardChain.appendTrustedBlock(remoteBlockchain.getBlockByNumber(number).orElseThrow()); return backwardChain; } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java index e8464642a95..34748b84d32 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Optional; import javax.annotation.Nonnull; @@ -45,6 +46,7 @@ public class InMemoryBackwardChainTest { GenericKeyValueStorageFacade headersStorage; GenericKeyValueStorageFacade blocksStorage; GenericKeyValueStorageFacade chainStorage; + GenericKeyValueStorageFacade variablesStorage; @Before public void prepareData() { @@ -61,6 +63,11 @@ public void prepareData() { chainStorage = new GenericKeyValueStorageFacade<>( Hash::toArrayUnsafe, new HashConvertor(), new InMemoryKeyValueStorage()); + variablesStorage = + new GenericKeyValueStorageFacade<>( + key -> key.getBytes(StandardCharsets.UTF_8), + new BlocksHeadersConvertor(new MainnetBlockHeaderFunctions()), + new InMemoryKeyValueStorage()); blocks = prepareChain(ELEMENTS, HEIGHT); } @@ -78,7 +85,7 @@ public void shouldReturnFirstHeaderCorrectly() { @Nonnull private BackwardChain createChainFromBlock(final Block pivot) { final BackwardChain backwardChain = - new BackwardChain(headersStorage, blocksStorage, chainStorage); + new BackwardChain(headersStorage, blocksStorage, chainStorage, variablesStorage); backwardChain.appendTrustedBlock(pivot); return backwardChain; } @@ -138,15 +145,15 @@ public void shouldAddHeaderToQueue() { firstHash = backwardChain.getFirstHashToAppend(); assertThat(firstHash).isPresent(); assertThat(firstHash.orElseThrow()).isEqualTo(blocks.get(7).getHash()); - backwardChain.processing(firstHash.get()); + backwardChain.removeFromHashToAppend(firstHash.get()); firstHash = backwardChain.getFirstHashToAppend(); assertThat(firstHash).isPresent(); assertThat(firstHash.orElseThrow()).isEqualTo(blocks.get(9).getHash()); - backwardChain.processing(firstHash.get()); + backwardChain.removeFromHashToAppend(firstHash.get()); firstHash = backwardChain.getFirstHashToAppend(); assertThat(firstHash).isPresent(); assertThat(firstHash.orElseThrow()).isEqualTo(blocks.get(11).getHash()); - backwardChain.processing(firstHash.get()); + backwardChain.removeFromHashToAppend(firstHash.get()); } @Test From 9eea27d837141a36407b1a21f31108bcfc70aab2 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Fri, 10 Mar 2023 17:33:21 +0100 Subject: [PATCH 7/9] Fix typo Signed-off-by: Fabio Di Fabio --- .../ethereum/eth/sync/backwardsync/BackwardChain.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java index 466176484ed..cb524e9f46c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java @@ -130,7 +130,7 @@ public synchronized void prependAncestorsHeader( } if (firstStoredAncestor.isEmpty()) { - updateLastStorePivot(Optional.of(blockHeader)); + updateLastStoredPivot(Optional.of(blockHeader)); } else { final BlockHeader firstHeader = firstStoredAncestor.get(); chainStorage.put(blockHeader.getHash(), firstHeader.getHash()); @@ -152,7 +152,7 @@ private void updateFirstStoredAncestor(final Optional maybeHeader) firstStoredAncestor = maybeHeader; } - private void updateLastStorePivot(final Optional maybeHeader) { + private void updateLastStoredPivot(final Optional maybeHeader) { maybeHeader.ifPresentOrElse( header -> variablesStorage.put(LAST_STORED_PIVOT_KEY, header), () -> variablesStorage.drop(LAST_STORED_PIVOT_KEY)); @@ -175,7 +175,7 @@ public synchronized void dropFirstHeader() { chainStorage.drop(firstStoredAncestor.get().getHash()); updateFirstStoredAncestor(hash.flatMap(headers::get)); if (firstStoredAncestor.isEmpty()) { - updateLastStorePivot(Optional.empty()); + updateLastStoredPivot(Optional.empty()); } } @@ -202,7 +202,7 @@ public synchronized void appendTrustedBlock(final Block newPivot) { .log(); } } - updateLastStorePivot(Optional.of(newPivot.getHeader())); + updateLastStoredPivot(Optional.of(newPivot.getHeader())); } public synchronized boolean isTrusted(final Hash hash) { From fc351dd7be1949f77b9f33a0837e35e4f9800000 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Mon, 13 Mar 2023 11:31:51 +0100 Subject: [PATCH 8/9] Renaming and comment from code review Signed-off-by: Fabio Di Fabio --- .../eth/sync/backwardsync/BackwardChain.java | 23 +++++++++++-------- .../backwardsync/BackwardSyncContextTest.java | 4 ++-- .../backwardsync/BackwardSyncStepTest.java | 6 ++--- .../backwardsync/ForwardSyncStepTest.java | 6 ++--- .../InMemoryBackwardChainTest.java | 6 ++--- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java index cb524e9f46c..ee4718b0613 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardChain.java @@ -42,7 +42,7 @@ public class BackwardChain { private final GenericKeyValueStorageFacade headers; private final GenericKeyValueStorageFacade blocks; private final GenericKeyValueStorageFacade chainStorage; - private final GenericKeyValueStorageFacade variablesStorage; + private final GenericKeyValueStorageFacade sessionDataStorage; private Optional firstStoredAncestor; private Optional lastStoredPivot; private final Queue hashesToAppend = new ArrayDeque<>(); @@ -51,13 +51,13 @@ public BackwardChain( final GenericKeyValueStorageFacade headersStorage, final GenericKeyValueStorageFacade blocksStorage, final GenericKeyValueStorageFacade chainStorage, - final GenericKeyValueStorageFacade variablesStorage) { + final GenericKeyValueStorageFacade sessionDataStorage) { this.headers = headersStorage; this.blocks = blocksStorage; this.chainStorage = chainStorage; - this.variablesStorage = variablesStorage; + this.sessionDataStorage = sessionDataStorage; firstStoredAncestor = - variablesStorage + sessionDataStorage .get(FIRST_STORED_ANCESTOR_KEY) .map( header -> { @@ -68,7 +68,7 @@ public BackwardChain( return header; }); lastStoredPivot = - variablesStorage + sessionDataStorage .get(LAST_STORED_PIVOT_KEY) .map( header -> { @@ -98,6 +98,9 @@ public static BackwardChain from( new HashConvertor(), storageProvider.getStorageBySegmentIdentifier( KeyValueSegmentIdentifier.BACKWARD_SYNC_CHAIN)), + // using BACKWARD_SYNC_CHAIN that contains the sequence of the work to do, + // to also store the session data that will be used to resume + // the backward sync from where it was left before the restart new GenericKeyValueStorageFacade<>( key -> key.getBytes(StandardCharsets.UTF_8), BlocksHeadersConvertor.of(blockHeaderFunctions), @@ -147,15 +150,15 @@ public synchronized void prependAncestorsHeader( private void updateFirstStoredAncestor(final Optional maybeHeader) { maybeHeader.ifPresentOrElse( - header -> variablesStorage.put(FIRST_STORED_ANCESTOR_KEY, header), - () -> variablesStorage.drop(FIRST_STORED_ANCESTOR_KEY)); + header -> sessionDataStorage.put(FIRST_STORED_ANCESTOR_KEY, header), + () -> sessionDataStorage.drop(FIRST_STORED_ANCESTOR_KEY)); firstStoredAncestor = maybeHeader; } private void updateLastStoredPivot(final Optional maybeHeader) { maybeHeader.ifPresentOrElse( - header -> variablesStorage.put(LAST_STORED_PIVOT_KEY, header), - () -> variablesStorage.drop(LAST_STORED_PIVOT_KEY)); + header -> sessionDataStorage.put(LAST_STORED_PIVOT_KEY, header), + () -> sessionDataStorage.drop(LAST_STORED_PIVOT_KEY)); lastStoredPivot = maybeHeader; } @@ -217,7 +220,7 @@ public synchronized void clear() { blocks.clear(); headers.clear(); chainStorage.clear(); - variablesStorage.clear(); + sessionDataStorage.clear(); firstStoredAncestor = Optional.empty(); lastStoredPivot = Optional.empty(); hashesToAppend.clear(); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java index 2fa03304a29..ea3d872cdb8 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java @@ -194,12 +194,12 @@ public static BackwardChain inMemoryBackwardChain() { final GenericKeyValueStorageFacade chainStorage = new GenericKeyValueStorageFacade<>( Hash::toArrayUnsafe, new HashConvertor(), new InMemoryKeyValueStorage()); - final GenericKeyValueStorageFacade variablesStorage = + final GenericKeyValueStorageFacade sessionDataStorage = new GenericKeyValueStorageFacade<>( key -> key.getBytes(StandardCharsets.UTF_8), BlocksHeadersConvertor.of(new MainnetBlockHeaderFunctions()), new InMemoryKeyValueStorage()); - return new BackwardChain(headersStorage, blocksStorage, chainStorage, variablesStorage); + return new BackwardChain(headersStorage, blocksStorage, chainStorage, sessionDataStorage); } @Test diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStepTest.java index bb2709873d5..9ddcaf7b8cb 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStepTest.java @@ -74,7 +74,7 @@ public class BackwardSyncStepTest { GenericKeyValueStorageFacade headersStorage; GenericKeyValueStorageFacade blocksStorage; GenericKeyValueStorageFacade chainStorage; - GenericKeyValueStorageFacade variablesStorage; + GenericKeyValueStorageFacade sessionDataStorage; @Before public void setup() { @@ -91,7 +91,7 @@ public void setup() { chainStorage = new GenericKeyValueStorageFacade<>( Hash::toArrayUnsafe, new HashConvertor(), new InMemoryKeyValueStorage()); - variablesStorage = + sessionDataStorage = new GenericKeyValueStorageFacade<>( key -> key.getBytes(StandardCharsets.UTF_8), new BlocksHeadersConvertor(new MainnetBlockHeaderFunctions()), @@ -240,7 +240,7 @@ private BackwardChain createBackwardChain(final int from, final int until) { @Nonnull private BackwardChain createBackwardChain(final int number) { final BackwardChain backwardChain = - new BackwardChain(headersStorage, blocksStorage, chainStorage, variablesStorage); + new BackwardChain(headersStorage, blocksStorage, chainStorage, sessionDataStorage); backwardChain.appendTrustedBlock(remoteBlockchain.getBlockByNumber(number).orElseThrow()); return backwardChain; } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java index 7705d71afb3..cf03c676ad9 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java @@ -75,7 +75,7 @@ public class ForwardSyncStepTest { GenericKeyValueStorageFacade headersStorage; GenericKeyValueStorageFacade blocksStorage; GenericKeyValueStorageFacade chainStorage; - GenericKeyValueStorageFacade variablesStorage; + GenericKeyValueStorageFacade sessionDataStorage; @Before public void setup() { @@ -92,7 +92,7 @@ public void setup() { chainStorage = new GenericKeyValueStorageFacade<>( Hash::toArrayUnsafe, new HashConvertor(), new InMemoryKeyValueStorage()); - variablesStorage = + sessionDataStorage = new GenericKeyValueStorageFacade<>( key -> key.getBytes(StandardCharsets.UTF_8), new BlocksHeadersConvertor(new MainnetBlockHeaderFunctions()), @@ -203,7 +203,7 @@ private BackwardChain createBackwardChain(final int from, final int until) { @Nonnull private BackwardChain backwardChainFromBlock(final int number) { final BackwardChain backwardChain = - new BackwardChain(headersStorage, blocksStorage, chainStorage, variablesStorage); + new BackwardChain(headersStorage, blocksStorage, chainStorage, sessionDataStorage); backwardChain.appendTrustedBlock(remoteBlockchain.getBlockByNumber(number).orElseThrow()); return backwardChain; } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java index 34748b84d32..405e06d1d35 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/InMemoryBackwardChainTest.java @@ -46,7 +46,7 @@ public class InMemoryBackwardChainTest { GenericKeyValueStorageFacade headersStorage; GenericKeyValueStorageFacade blocksStorage; GenericKeyValueStorageFacade chainStorage; - GenericKeyValueStorageFacade variablesStorage; + GenericKeyValueStorageFacade sessionDataStorage; @Before public void prepareData() { @@ -63,7 +63,7 @@ public void prepareData() { chainStorage = new GenericKeyValueStorageFacade<>( Hash::toArrayUnsafe, new HashConvertor(), new InMemoryKeyValueStorage()); - variablesStorage = + sessionDataStorage = new GenericKeyValueStorageFacade<>( key -> key.getBytes(StandardCharsets.UTF_8), new BlocksHeadersConvertor(new MainnetBlockHeaderFunctions()), @@ -85,7 +85,7 @@ public void shouldReturnFirstHeaderCorrectly() { @Nonnull private BackwardChain createChainFromBlock(final Block pivot) { final BackwardChain backwardChain = - new BackwardChain(headersStorage, blocksStorage, chainStorage, variablesStorage); + new BackwardChain(headersStorage, blocksStorage, chainStorage, sessionDataStorage); backwardChain.appendTrustedBlock(pivot); return backwardChain; } From fa37c78c6fc3c9f523b0600f854267020ebe21f0 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Wed, 15 Mar 2023 09:50:36 +0100 Subject: [PATCH 9/9] [skip ci] fix typo Co-authored-by: Sally MacFarlane Signed-off-by: Fabio Di Fabio --- .../ethereum/eth/sync/backwardsync/BackwardSyncContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java index 7b87a476272..bc618e7c133 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java @@ -153,7 +153,7 @@ private boolean isTrusted(final Hash hash) { if (backwardChain.isTrusted(hash)) { LOG.atDebug() .setMessage( - "Not fetching or appending hash {} to backwards sync since it is present in successors") + "Not fetching or appending hash {} to backward sync since it is present in successors") .addArgument(hash::toHexString) .log(); return true;