Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Check network on start and reset db #8642

Merged
merged 31 commits into from
Oct 8, 2024
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
62100c1
Add check for Ephemery and reset db
gconnect Sep 24, 2024
d5dc6c4
Remove unused comment
gconnect Sep 24, 2024
a8f6afa
Add a shouldResetForEphemery method and test
gconnect Sep 24, 2024
a33707b
Merge branch 'master' into check-network-on-start
gconnect Sep 24, 2024
c1149ac
Merge branch 'master' into check-network-on-start
gconnect Sep 25, 2024
34ecc2d
update null check
gconnect Sep 25, 2024
572d91b
Merge branch 'master' into check-network-on-start
gconnect Sep 26, 2024
7b6584b
Merge branch 'master' into check-network-on-start
gconnect Sep 26, 2024
e935764
Merge branch 'master' into check-network-on-start
gconnect Sep 27, 2024
d496675
Merge branch 'master' into check-network-on-start
gconnect Sep 30, 2024
1bbc3c4
optimized the reset code and add a custom EphemeryException
gconnect Sep 30, 2024
8a3d40c
Merge branch 'master' into check-network-on-start
gconnect Sep 30, 2024
d23b3c4
Update the reset method and remove migrate
gconnect Sep 30, 2024
6a5d9fc
Remove private modifier
gconnect Sep 30, 2024
c2aced4
Apply spotlessApply
gconnect Sep 30, 2024
6d2cf43
Apply spotlessApply
gconnect Oct 1, 2024
7b2871c
Create custom Ephemery reset class and test
gconnect Oct 1, 2024
d9d1951
Add fnal and move global variable to local variable
gconnect Oct 1, 2024
0d47959
Remove repeated code
gconnect Oct 1, 2024
4306df8
Merge branch 'master' into check-network-on-start
gconnect Oct 2, 2024
ee57df8
Merge branch 'master' into check-network-on-start
gconnect Oct 2, 2024
f73e491
Ensure reset method delete specific files and update test and change …
gconnect Oct 2, 2024
f95a61a
Remove print statement and add final
gconnect Oct 2, 2024
b2bcefd
Update the recursive delete method and test
gconnect Oct 2, 2024
769c6f8
Merge branch 'master' into check-network-on-start
gconnect Oct 2, 2024
6cdc994
Remove print statement
gconnect Oct 2, 2024
5588f16
Merge branch 'master' into check-network-on-start
gconnect Oct 3, 2024
e96e6e3
Merge branch 'master' into check-network-on-start
gconnect Oct 5, 2024
92f6d5b
Merge branch 'master' into check-network-on-start
rolfyone Oct 7, 2024
c1ad18d
Merge branch 'master' into check-network-on-start
rolfyone Oct 8, 2024
b20eedc
Merge branch 'master' into check-network-on-start
rolfyone Oct 8, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
import static tech.pegasys.teku.infrastructure.async.AsyncRunnerFactory.DEFAULT_MAX_QUEUE_SIZE;
import static tech.pegasys.teku.spec.config.Constants.STORAGE_QUERY_CHANNEL_PARALLELISM;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Optional;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand All @@ -42,6 +45,7 @@
import tech.pegasys.teku.storage.server.RetryingStorageUpdateChannel;
import tech.pegasys.teku.storage.server.StorageConfiguration;
import tech.pegasys.teku.storage.server.VersionedDatabaseFactory;
import tech.pegasys.teku.storage.server.network.EphemeryException;
import tech.pegasys.teku.storage.server.pruner.BlobSidecarPruner;
import tech.pegasys.teku.storage.server.pruner.BlockPruner;
import tech.pegasys.teku.storage.server.pruner.StatePruner;
Expand Down Expand Up @@ -80,14 +84,24 @@ protected SafeFuture<?> doStart() {
1,
DEFAULT_MAX_QUEUE_SIZE,
Thread.NORM_PRIORITY - 1);
final VersionedDatabaseFactory dbFactory =
gconnect marked this conversation as resolved.
Show resolved Hide resolved
VersionedDatabaseFactory dbFactory =
new VersionedDatabaseFactory(
serviceConfig.getMetricsSystem(),
serviceConfig.getDataDirLayout().getBeaconDataDirectory(),
config);
database = dbFactory.createDatabase();

database.migrate();
try {
database = dbFactory.createDatabase();
database.migrate();
} catch (EphemeryException e) {
try {
resetDatabaseDirectories(serviceConfig);
database = dbFactory.createDatabase();
database.migrate();
} catch (Exception ex) {
throw new RuntimeException("Failed to reset and recreate the database.", ex);
}
}
gconnect marked this conversation as resolved.
Show resolved Hide resolved

final SettableLabelledGauge pruningTimingsLabelledGauge =
SettableLabelledGauge.create(
Expand Down Expand Up @@ -222,4 +236,30 @@ protected SafeFuture<?> doStop() {
public ChainStorage getChainStorage() {
return chainStorage;
}

/** This method is called only on Ephemery network when reset is due. */
void resetDatabaseDirectories(final ServiceConfig serviceConfig) throws IOException {
final Path beaconDataDir = serviceConfig.getDataDirLayout().getBeaconDataDirectory();
final Path slashProtectionDir =
serviceConfig.getDataDirLayout().getValidatorDataDirectory().resolve("slashprotection");
deleteDirectoryRecursively(beaconDataDir);
deleteDirectoryRecursively(slashProtectionDir);
}
gconnect marked this conversation as resolved.
Show resolved Hide resolved

private void deleteDirectoryRecursively(final Path path) throws IOException {
if (Files.isDirectory(path)) {
try (var stream = Files.walk(path)) {
stream
.sorted((o1, o2) -> o2.compareTo(o1))
.forEach(
p -> {
try {
Files.delete(p);
} catch (IOException e) {
throw new RuntimeException("Failed to delete file: " + p, e);
}
});
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ public class VersionedDatabaseFactory implements DatabaseFactory {
@VisibleForTesting static final String STORAGE_MODE_PATH = "data-storage-mode.txt";
@VisibleForTesting static final String METADATA_FILENAME = "metadata.yml";
@VisibleForTesting static final String NETWORK_FILENAME = "network.yml";

private final MetricsSystem metricsSystem;
private final File dataDirectory;
private final int maxKnownNodeCacheSize;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ public class DatabaseNetwork {
@VisibleForTesting
final Long depositChainId;

private static final String EPHEMERY_DEPOSIT_CONTRACT_ADDRESS =
"0x4242424242424242424242424242424242424242";

@JsonCreator
DatabaseNetwork(
@JsonProperty(value = "fork_version") final String forkVersion,
Expand Down Expand Up @@ -84,6 +87,20 @@ public static DatabaseNetwork init(
formatMessage(
"deposit contract", depositContractString, databaseNetwork.depositContract));
}
if (databaseNetwork.depositChainId != null
&& !depositContractString.equals(EPHEMERY_DEPOSIT_CONTRACT_ADDRESS)
&& !databaseNetwork.depositChainId.equals(depositChainId)) {
throw DatabaseStorageException.unrecoverable(
formatMessage(
"deposit chain id",
String.valueOf(depositChainId),
String.valueOf(databaseNetwork.depositChainId)));
}
if (databaseNetwork.depositChainId != null
&& depositContractString.equals(EPHEMERY_DEPOSIT_CONTRACT_ADDRESS)
&& !databaseNetwork.depositChainId.equals(depositChainId)) {
throw new EphemeryException();
}
return databaseNetwork;
} else {
DatabaseNetwork databaseNetwork =
Expand All @@ -93,6 +110,10 @@ public static DatabaseNetwork init(
}
}

public Long getDepositChainId() {
return depositChainId;
}

private static String formatMessage(
final String fieldName, final String expected, final String actual) {
return String.format(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
/*
* Copyright Consensys Software Inc., 2024
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/

package tech.pegasys.teku.storage.server.network;

public class EphemeryException extends RuntimeException {}