Skip to content

Commit

Permalink
Merge branch 'master' into ed/etc_atlantis_fork
Browse files Browse the repository at this point in the history
  • Loading branch information
edwardmack authored Nov 11, 2019
2 parents 75ccab8 + 0a27885 commit f98f55c
Show file tree
Hide file tree
Showing 13 changed files with 242 additions and 93 deletions.
47 changes: 17 additions & 30 deletions besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import org.hyperledger.besu.cli.options.EthProtocolOptions;
import org.hyperledger.besu.cli.options.MetricsCLIOptions;
import org.hyperledger.besu.cli.options.NetworkingOptions;
import org.hyperledger.besu.cli.options.PrunerOptions;
import org.hyperledger.besu.cli.options.SynchronizerOptions;
import org.hyperledger.besu.cli.options.TransactionPoolOptions;
import org.hyperledger.besu.cli.subcommands.PasswordSubCommand;
Expand Down Expand Up @@ -86,7 +87,6 @@
import org.hyperledger.besu.ethereum.permissioning.SmartContractPermissioningConfiguration;
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider;
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder;
import org.hyperledger.besu.ethereum.worldstate.PruningConfiguration;
import org.hyperledger.besu.metrics.BesuMetricCategory;
import org.hyperledger.besu.metrics.MetricCategoryRegistryImpl;
import org.hyperledger.besu.metrics.ObservableMetricsSystem;
Expand Down Expand Up @@ -185,6 +185,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
final EthProtocolOptions ethProtocolOptions = EthProtocolOptions.create();
final MetricsCLIOptions metricsCLIOptions = MetricsCLIOptions.create();
final TransactionPoolOptions transactionPoolOptions = TransactionPoolOptions.create();
final PrunerOptions prunerOptions = PrunerOptions.create();
private final RunnerBuilder runnerBuilder;
private final BesuController.Builder controllerBuilderFactory;
private final BesuPluginContextImpl besuPluginContext;
Expand Down Expand Up @@ -588,24 +589,8 @@ void setBannedNodeIds(final List<String> values) {
@Option(
names = {"--pruning-enabled"},
description =
"Enable pruning of world state of blocks older than the retention period (default: ${DEFAULT-VALUE})")
private final Boolean isPruningEnabled = false;

@Option(
names = {"--pruning-blocks-retained"},
hidden = true,
description =
"Minimum number of recent blocks for which to keep entire world state (default: ${DEFAULT-VALUE})",
arity = "1")
private final Long pruningBlocksRetained = DEFAULT_PRUNING_BLOCKS_RETAINED;

@Option(
names = {"--pruning-block-confirmations"},
hidden = true,
description =
"Minimum number of confirmations on a block before marking begins (default: ${DEFAULT-VALUE})",
arity = "1")
private final Long pruningBlockConfirmations = DEFAULT_PRUNING_BLOCK_CONFIRMATIONS;
"Enable disk-space saving optimization that removes old state that is unlikely to be required (default: true if fast sync is enabled, false otherwise)")
private Boolean pruningOverride;

@Option(
names = {"--permissions-nodes-config-file-enabled"},
Expand Down Expand Up @@ -870,6 +855,7 @@ private BesuCommand handleUnstableOptions() {
.put("P2P Network", networkingOptions)
.put("Synchronizer", synchronizerOptions)
.put("TransactionPool", transactionPoolOptions)
.put("Pruner", prunerOptions)
.build();

UnstableOptionsSubCommand.createUnstableOptions(commandLine, unstableOptions);
Expand Down Expand Up @@ -1006,13 +992,6 @@ private void issueOptionWarnings() {
"--sync-mode",
!SyncMode.FAST.equals(syncMode),
singletonList("--fast-sync-min-peers"));

CommandLineUtils.checkOptionDependencies(
logger,
commandLine,
"--pruning-enabled",
!isPruningEnabled,
asList("--pruning-block-confirmations", "--pruning-blocks-retained"));
}

private BesuCommand configure() throws Exception {
Expand Down Expand Up @@ -1087,8 +1066,8 @@ public BesuControllerBuilder<?> getControllerBuilder() {
.clock(Clock.systemUTC())
.isRevertReasonEnabled(isRevertReasonEnabled)
.storageProvider(keyStorageProvider(keyValueStorageName))
.isPruningEnabled(isPruningEnabled)
.pruningConfiguration(buildPruningConfiguration())
.isPruningEnabled(isPruningEnabled())
.pruningConfiguration(prunerOptions.toDomainObject())
.genesisConfigOverrides(genesisConfigOverrides)
.targetGasLimit(targetGasLimit == null ? Optional.empty() : Optional.of(targetGasLimit))
.requiredBlocks(requiredBlocks);
Expand Down Expand Up @@ -1325,6 +1304,14 @@ private PrivacyParameters privacyParameters() throws IOException {

final PrivacyParameters.Builder privacyParametersBuilder = new PrivacyParameters.Builder();
if (isPrivacyEnabled) {
final String errorSuffix = "cannot be enabled with privacy.";
if (syncMode == SyncMode.FAST) {
throw new ParameterException(commandLine, String.format("%s %s", "Fast sync", errorSuffix));
}
if (isPruningEnabled()) {
throw new ParameterException(commandLine, String.format("%s %s", "Pruning", errorSuffix));
}

privacyParametersBuilder.setEnabled(true);
privacyParametersBuilder.setEnclaveUrl(privacyUrl);
if (privacyPublicKeyFile() != null) {
Expand Down Expand Up @@ -1370,8 +1357,8 @@ private TransactionPoolConfiguration buildTransactionPoolConfiguration() {
.build();
}

private PruningConfiguration buildPruningConfiguration() {
return new PruningConfiguration(pruningBlockConfirmations, pruningBlocksRetained);
private boolean isPruningEnabled() {
return Optional.ofNullable(pruningOverride).orElse(syncMode == SyncMode.FAST);
}

// Blockchain synchronisation from peers.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,6 @@ public interface DefaultCommandValues {
String MANDATORY_NETWORK_FORMAT_HELP = "<NETWORK>";
String MANDATORY_NODE_ID_FORMAT_HELP = "<NODEID>";
Wei DEFAULT_MIN_TRANSACTION_GAS_PRICE = Wei.of(1000);
long DEFAULT_PRUNING_BLOCKS_RETAINED = 1024;
long DEFAULT_PRUNING_BLOCK_CONFIRMATIONS = 10;
BytesValue DEFAULT_EXTRA_DATA = BytesValue.EMPTY;
long DEFAULT_MAX_REFRESH_DELAY = 3600000;
long DEFAULT_MIN_REFRESH_DELAY = 1;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
/*
* Copyright ConsenSys AG.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.cli.options;

import org.hyperledger.besu.ethereum.worldstate.PrunerConfiguration;

import java.util.Arrays;
import java.util.List;

import picocli.CommandLine;

public class PrunerOptions implements CLIOptions<PrunerConfiguration> {
private static final String BLOCKS_RETAINED_FLAG = "--Xpruning-blocks-retained";
private static final String BLOCK_CONFIRMATIONS_FLAG = "--Xpruning-block-confirmations";

@CommandLine.Option(
names = {BLOCKS_RETAINED_FLAG},
hidden = true,
defaultValue = "1024",
paramLabel = "<INTEGER>",
description =
"Minimum number of recent blocks for which to keep entire world state (default: ${DEFAULT-VALUE})",
arity = "1")
private long pruningBlocksRetained = PrunerConfiguration.DEFAULT_PRUNING_BLOCKS_RETAINED;

@CommandLine.Option(
names = {BLOCK_CONFIRMATIONS_FLAG},
defaultValue = "10",
hidden = true,
paramLabel = "<INTEGER>",
description =
"Minimum number of confirmations on a block before marking begins (default: ${DEFAULT-VALUE})",
arity = "1")
private long pruningBlockConfirmations = PrunerConfiguration.DEFAULT_PRUNING_BLOCK_CONFIRMATIONS;

public static PrunerOptions create() {
return new PrunerOptions();
}

@Override
public PrunerConfiguration toDomainObject() {
return new PrunerConfiguration(pruningBlockConfirmations, pruningBlocksRetained);
}

public static PrunerOptions fromDomainObject(final PrunerConfiguration prunerConfiguration) {
final PrunerOptions prunerOptions = new PrunerOptions();
prunerOptions.pruningBlockConfirmations = prunerConfiguration.getBlockConfirmations();
prunerOptions.pruningBlocksRetained = prunerConfiguration.getBlocksRetained();
return prunerOptions;
}

@Override
public List<String> getCLIOptions() {
return Arrays.asList(
BLOCKS_RETAINED_FLAG,
String.valueOf(pruningBlocksRetained),
BLOCK_CONFIRMATIONS_FLAG,
String.valueOf(pruningBlockConfirmations));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
package org.hyperledger.besu.controller;

import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static org.hyperledger.besu.controller.KeyPairUtil.loadKeyPair;

import org.hyperledger.besu.config.GenesisConfigFile;
Expand Down Expand Up @@ -49,7 +48,7 @@
import org.hyperledger.besu.ethereum.storage.StorageProvider;
import org.hyperledger.besu.ethereum.worldstate.MarkSweepPruner;
import org.hyperledger.besu.ethereum.worldstate.Pruner;
import org.hyperledger.besu.ethereum.worldstate.PruningConfiguration;
import org.hyperledger.besu.ethereum.worldstate.PrunerConfiguration;
import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
import org.hyperledger.besu.metrics.ObservableMetricsSystem;

Expand All @@ -66,7 +65,12 @@
import java.util.Optional;
import java.util.OptionalLong;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

public abstract class BesuControllerBuilder<C> {
private static final Logger LOG = LogManager.getLogger();

protected GenesisConfigFile genesisConfig;
SynchronizerConfiguration syncConfig;
EthProtocolConfiguration ethereumWireProtocolConfiguration;
Expand All @@ -82,7 +86,7 @@ public abstract class BesuControllerBuilder<C> {
GasLimitCalculator gasLimitCalculator;
private StorageProvider storageProvider;
private boolean isPruningEnabled;
private PruningConfiguration pruningConfiguration;
private PrunerConfiguration prunerConfiguration;
Map<String, String> genesisConfigOverrides;
private Map<Long, Hash> requiredBlocks = Collections.emptyMap();

Expand Down Expand Up @@ -160,14 +164,14 @@ public BesuControllerBuilder<C> isRevertReasonEnabled(final boolean isRevertReas
return this;
}

public BesuControllerBuilder<C> isPruningEnabled(final boolean pruningEnabled) {
this.isPruningEnabled = pruningEnabled;
public BesuControllerBuilder<C> isPruningEnabled(final boolean isPruningEnabled) {
this.isPruningEnabled = isPruningEnabled;
return this;
}

public BesuControllerBuilder<C> pruningConfiguration(
final PruningConfiguration pruningConfiguration) {
this.pruningConfiguration = pruningConfiguration;
final PrunerConfiguration prunerConfiguration) {
this.prunerConfiguration = prunerConfiguration;
return this;
}

Expand Down Expand Up @@ -219,19 +223,21 @@ public BesuController<C> build() {

Optional<Pruner> maybePruner = Optional.empty();
if (isPruningEnabled) {
checkState(
storageProvider.isWorldStateIterable(),
"Cannot enable pruning with current database version. Resync to get the latest version.");
maybePruner =
Optional.of(
new Pruner(
new MarkSweepPruner(
protocolContext.getWorldStateArchive().getWorldStateStorage(),
blockchain,
storageProvider.createPruningStorage(),
metricsSystem),
blockchain,
pruningConfiguration));
if (!storageProvider.isWorldStateIterable()) {
LOG.warn(
"Cannot enable pruning with current database version. Disabling. Resync to get the latest database version or disable pruning explicitly on the command line to remove this warning.");
} else {
maybePruner =
Optional.of(
new Pruner(
new MarkSweepPruner(
protocolContext.getWorldStateArchive().getWorldStateStorage(),
blockchain,
storageProvider.createPruningStorage(),
metricsSystem),
blockchain,
prunerConfiguration));
}
}

final boolean fastSyncEnabled = syncConfig.getSyncMode().equals(SyncMode.FAST);
Expand Down
57 changes: 47 additions & 10 deletions besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@
import org.hyperledger.besu.ethereum.permissioning.LocalPermissioningConfiguration;
import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration;
import org.hyperledger.besu.ethereum.permissioning.SmartContractPermissioningConfiguration;
import org.hyperledger.besu.ethereum.worldstate.PruningConfiguration;
import org.hyperledger.besu.ethereum.worldstate.PrunerConfiguration;
import org.hyperledger.besu.metrics.StandardMetricCategory;
import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
import org.hyperledger.besu.nat.NatMethod;
Expand Down Expand Up @@ -2329,8 +2329,8 @@ public void miningParametersAreCaptured() throws Exception {
}

@Test
public void pruningIsEnabledWhenSpecified() throws Exception {
parseCommand("--pruning-enabled");
public void pruningIsEnabledIfSyncModeIsFast() {
parseCommand("--sync-mode", "FAST");

verify(mockControllerBuilder).isPruningEnabled(true);
verify(mockControllerBuilder).build();
Expand All @@ -2340,12 +2340,33 @@ public void pruningIsEnabledWhenSpecified() throws Exception {
}

@Test
public void pruningOptionsRequiresServiceToBeEnabled() {
public void pruningIsDisabledIfSyncModeIsFull() {
parseCommand("--sync-mode", "FULL");

parseCommand("--pruning-blocks-retained", "4", "--pruning-block-confirmations", "1");
verify(mockControllerBuilder).isPruningEnabled(false);
verify(mockControllerBuilder).build();

verifyOptionsConstraintLoggerCall(
"--pruning-enabled", "--pruning-blocks-retained", "--pruning-block-confirmations");
assertThat(commandOutput.toString()).isEmpty();
assertThat(commandErrorOutput.toString()).isEmpty();
}

@Test
public void pruningEnabledExplicitly() {
parseCommand("--pruning-enabled", "--sync-mode=FULL");

verify(mockControllerBuilder).isPruningEnabled(true);
verify(mockControllerBuilder).build();

assertThat(commandOutput.toString()).isEmpty();
assertThat(commandErrorOutput.toString()).isEmpty();
}

@Test
public void pruningDisabledExplicitly() {
parseCommand("--pruning-enabled=false", "--sync-mode=FAST");

verify(mockControllerBuilder).isPruningEnabled(false);
verify(mockControllerBuilder).build();

assertThat(commandOutput.toString()).isEmpty();
assertThat(commandErrorOutput.toString()).isEmpty();
Expand All @@ -2354,10 +2375,10 @@ public void pruningOptionsRequiresServiceToBeEnabled() {
@Test
public void pruningParametersAreCaptured() throws Exception {
parseCommand(
"--pruning-enabled", "--pruning-blocks-retained=15", "--pruning-block-confirmations=4");
"--pruning-enabled", "--Xpruning-blocks-retained=15", "--Xpruning-block-confirmations=4");

final ArgumentCaptor<PruningConfiguration> pruningArg =
ArgumentCaptor.forClass(PruningConfiguration.class);
final ArgumentCaptor<PrunerConfiguration> pruningArg =
ArgumentCaptor.forClass(PrunerConfiguration.class);

verify(mockControllerBuilder).pruningConfiguration(pruningArg.capture());
verify(mockControllerBuilder).build();
Expand Down Expand Up @@ -2682,6 +2703,22 @@ public void privacyPublicKeyFileOptionDisabledUnderDocker() {
assertThat(commandOutput.toString()).isEmpty();
}

@Test
public void privacyWithFastSyncMustError() {
parseCommand("--sync-mode=FAST", "--privacy-enabled");

assertThat(commandErrorOutput.toString()).contains("Fast sync cannot be enabled with privacy.");
assertThat(commandOutput.toString()).isEmpty();
}

@Test
public void privacyWithPruningMustError() {
parseCommand("--pruning-enabled", "--privacy-enabled");

assertThat(commandErrorOutput.toString()).contains("Pruning cannot be enabled with privacy.");
assertThat(commandOutput.toString()).isEmpty();
}

@Test
public void rpcHttpAuthCredentialsFileOptionDisabledUnderDocker() {
System.setProperty("besu.docker", "true");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.hyperledger.besu.cli.options.EthProtocolOptions;
import org.hyperledger.besu.cli.options.MetricsCLIOptions;
import org.hyperledger.besu.cli.options.NetworkingOptions;
import org.hyperledger.besu.cli.options.PrunerOptions;
import org.hyperledger.besu.cli.options.SynchronizerOptions;
import org.hyperledger.besu.cli.options.TransactionPoolOptions;
import org.hyperledger.besu.cli.subcommands.PublicKeySubCommand;
Expand Down Expand Up @@ -332,6 +333,10 @@ public SynchronizerOptions getSynchronizerOptions() {
return synchronizerOptions;
}

public PrunerOptions getPrunerOptions() {
return prunerOptions;
}

public EthProtocolOptions getEthProtocolOptions() {
return ethProtocolOptions;
}
Expand Down
Loading

0 comments on commit f98f55c

Please sign in to comment.