Merge branch 'master' into issue1414b

CHANGELOG.md

@@ -1,11 +1,25 @@
 # Changelog
 
+## 20.10.3
+
+### Additions and Improvements
+* Added `memory` as an option to `--key-value-storage`.  This ephemeral storage is intended for sync testing and debugging.  [\#1617](https://github.com/hyperledger/besu/pull/1617)
+* Fixed gasPrice parameter not always respected when passed to `eth_estimateGas` endpoint [#1636](https://github.com/hyperledger/besu/pull/1636)
+
+### Bug Fixes
+
+
+#### Previously identified known issues
+
+- [Fast sync when running Besu on cloud providers](KNOWN_ISSUES.md#fast-sync-when-running-besu-on-cloud-providers)
+- [Privacy users with private transactions created using v1.3.4 or earlier](KNOWN_ISSUES.md#privacy-users-with-private-transactions-created-using-v134-or-earlier)
+
 ## 20.10.2
 
 ### Additions and Improvements
 * Added support for batched requests in WebSockets. [#1583](https://github.com/hyperledger/besu/pull/1583)
-* Added a protocols section to `admin_peers` to provide info about peer health. [\#1582](https://github.com/hyperledger/besu/pull/1582)
-* Added CLI option `--goquorum-compatibility-enabled` to enable GoQuorum compatibility mode. [#1598](https://github.com/hyperledger/besu/pull/1598)
+* Added protocols section to `admin_peers` to provide info about peer health. [\#1582](https://github.com/hyperledger/besu/pull/1582)
+* Added CLI option `--goquorum-compatibility-enabled` to enable GoQuorum compatibility mode. [#1598](https://github.com/hyperledger/besu/pull/1598). Note that this mode is incompatible with Mainnet.
 
 ### Bug Fixes
 
@@ -19,6 +33,11 @@
 - [Fast sync when running Besu on cloud providers](KNOWN_ISSUES.md#fast-sync-when-running-besu-on-cloud-providers)
 - [Privacy users with private transactions created using v1.3.4 or earlier](KNOWN_ISSUES.md#privacy-users-with-private-transactions-created-using-v134-or-earlier)
 
+### Download Link
+
+https://dl.bintray.com/hyperledger-org/besu-repo/besu-20.10.2.zip
+sha256: `710aed228dcbe9b8103aef39e4431b0c63e73c3a708ce88bcd1ecfa1722ad307`
+
 ## 20.10.1
 
 ### Additions and Improvements

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java

@@ -414,14 +414,15 @@ public boolean isActive(final String nodeName) {
   private void killBesuProcess(final String name) {
     final Process process = besuProcesses.remove(name);
     if (process == null) {
-      LOG.error("Process {} wasn't in our list", name);
+      LOG.error("Process {} wasn't in our list, pid {}", name, process.pid());
+      return;
     }
     if (!process.isAlive()) {
-      LOG.info("Process {} already exited", name);
+      LOG.info("Process {} already exited, pid {}", name, process.pid());
       return;
     }
 
-    LOG.info("Killing {} process", name);
+    LOG.info("Killing {} process, pid {}", name, process.pid());
 
     process.destroy();
     try {
@@ -431,7 +432,7 @@ private void killBesuProcess(final String name) {
     }
 
     if (process.isAlive()) {
-      LOG.warn("Process {} still alive, destroying forcibly now", name);
+      LOG.warn("Process {} still alive, destroying forcibly now, pid {}", name, process.pid());
       try {
         process.destroyForcibly().waitFor(30, TimeUnit.SECONDS);
       } catch (final Exception e) {

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/cluster/Cluster.java

@@ -87,12 +87,19 @@ public void start(final List<? extends RunnableNode> nodes) {
 
     nodes
         .parallelStream()
-        .filter(node -> bootnode.map(boot -> boot != node).orElse(true))
+        .filter(
+            node -> {
+              LOG.info("starting non-bootnode {}", node.getName());
+              return bootnode.map(boot -> boot != node).orElse(true);
+            })
         .forEach(this::startNode);
 
     if (clusterConfiguration.isAwaitPeerDiscovery()) {
       for (final RunnableNode node : nodes) {
-        LOG.info("Awaiting peer discovery for node {}", node.getName());
+        LOG.info(
+            "Awaiting peer discovery for node {}, expecting {} peers",
+            node.getName(),
+            nodes.size() - 1);
         node.awaitPeerDiscovery(net.awaitPeerCount(nodes.size() - 1));
       }
     }

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -48,6 +48,7 @@
 import org.hyperledger.besu.cli.custom.JsonRPCAllowlistHostsProperty;
 import org.hyperledger.besu.cli.custom.RpcAuthFileValidator;
 import org.hyperledger.besu.cli.error.BesuExceptionHandler;
+import org.hyperledger.besu.cli.options.unstable.DataStorageOptions;
 import org.hyperledger.besu.cli.options.unstable.DnsOptions;
 import org.hyperledger.besu.cli.options.unstable.EthProtocolOptions;
 import org.hyperledger.besu.cli.options.unstable.EthstatsOptions;
@@ -73,6 +74,7 @@
 import org.hyperledger.besu.cli.util.VersionProvider;
 import org.hyperledger.besu.config.GenesisConfigFile;
 import org.hyperledger.besu.config.GenesisConfigOptions;
+import org.hyperledger.besu.config.GoQuorumOptions;
 import org.hyperledger.besu.config.experimental.ExperimentalEIPs;
 import org.hyperledger.besu.controller.BesuController;
 import org.hyperledger.besu.controller.BesuControllerBuilder;
@@ -143,6 +145,7 @@
 import org.hyperledger.besu.services.PicoCLIOptionsImpl;
 import org.hyperledger.besu.services.SecurityModuleServiceImpl;
 import org.hyperledger.besu.services.StorageServiceImpl;
+import org.hyperledger.besu.services.kvstore.InMemoryStoragePlugin;
 import org.hyperledger.besu.util.NetworkUtility;
 import org.hyperledger.besu.util.PermissioningConfigurationValidator;
 import org.hyperledger.besu.util.number.Fraction;
@@ -228,6 +231,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
   final MetricsCLIOptions unstableMetricsCLIOptions = MetricsCLIOptions.create();
   final TransactionPoolOptions unstableTransactionPoolOptions = TransactionPoolOptions.create();
   private final EthstatsOptions unstableEthstatsOptions = EthstatsOptions.create();
+  private final DataStorageOptions unstableDataStorageOptions = DataStorageOptions.create();
   private final DnsOptions unstableDnsOptions = DnsOptions.create();
   private final MiningOptions unstableMiningOptions = MiningOptions.create();
   private final NatOptions unstableNatOptions = NatOptions.create();
@@ -1219,6 +1223,7 @@ private void handleUnstableOptions() {
             .put("Ethstats", unstableEthstatsOptions)
             .put("Mining", unstableMiningOptions)
             .put("Native Library", unstableNativeLibraryOptions)
+            .put("Data Storage Options", unstableDataStorageOptions)
             .build();
 
     UnstableOptionsSubCommand.createUnstableOptions(commandLine, unstableOptions);
@@ -1232,6 +1237,7 @@ private void preparePlugins() {
 
     // register built-in plugins
     new RocksDBPlugin().register(besuPluginContext);
+    new InMemoryStoragePlugin().register(besuPluginContext);
 
     besuPluginContext.registerPlugins(pluginsDir());
 
@@ -1333,7 +1339,6 @@ private BesuCommand validateOptions() {
     validateNatParams();
     validateNetStatsParams();
     validateDnsOptionsParams();
-    validateGoQuorumCompatibilityModeParam();
 
     return this;
   }
@@ -1403,17 +1408,6 @@ private void validateDnsOptionsParams() {
     }
   }
 
-  private void validateGoQuorumCompatibilityModeParam() {
-    if (isGoQuorumCompatibilityMode) {
-      final GenesisConfigOptions genesisConfigOptions = readGenesisConfigOptions();
-
-      if (!genesisConfigOptions.isQuorum()) {
-        throw new IllegalStateException(
-            "GoQuorum compatibility mode (enabled) can only be used if genesis file has 'isQuorum' flag set to true.");
-      }
-    }
-  }
-
   private GenesisConfigOptions readGenesisConfigOptions() {
     final GenesisConfigOptions genesisConfigOptions;
     try {
@@ -1475,10 +1469,6 @@ private void issueOptionWarnings() {
   private BesuCommand configure() throws Exception {
     checkPortClash();
 
-    if (isGoQuorumCompatibilityMode) {
-      checkGoQuorumCompatibilityConfig();
-    }
-
     syncMode =
         Optional.ofNullable(syncMode)
             .orElse(
@@ -1487,6 +1477,9 @@ private BesuCommand configure() throws Exception {
                     : SyncMode.FULL);
 
     ethNetworkConfig = updateNetworkConfig(getNetwork());
+    if (isGoQuorumCompatibilityMode) {
+      checkGoQuorumCompatibilityConfig(ethNetworkConfig);
+    }
     jsonRpcConfiguration = jsonRpcConfiguration();
     graphQLConfiguration = graphQLConfiguration();
     webSocketConfiguration = webSocketConfiguration();
@@ -1586,7 +1579,8 @@ public BesuControllerBuilder getControllerBuilder() {
                 .<GasLimitCalculator>map(TargetingGasLimitCalculator::new)
                 .orElse(GasLimitCalculator.constant()))
         .requiredBlocks(requiredBlocks)
-        .reorgLoggingThreshold(reorgLoggingThreshold);
+        .reorgLoggingThreshold(reorgLoggingThreshold)
+        .dataStorageConfiguration(unstableDataStorageOptions.toDomainObject());
   }
 
   private GraphQLConfiguration graphQLConfiguration() {
@@ -2264,6 +2258,7 @@ private EthNetworkConfig updateNetworkConfig(final NetworkName network) {
         // than a useless one that may make user think that it can work when it can't.
         builder.setBootNodes(new ArrayList<>());
       }
+      builder.setDnsDiscoveryUrl(null);
     }
 
     if (networkId != null) {
@@ -2284,7 +2279,6 @@ private EthNetworkConfig updateNetworkConfig(final NetworkName network) {
         throw new ParameterException(commandLine, e.getMessage());
       }
     }
-
     return builder.build();
   }
 
@@ -2384,21 +2378,12 @@ public EnodeDnsConfiguration getEnodeDnsConfiguration() {
   }
 
   private void checkPortClash() {
-    // List of port parameters
-    final List<Integer> ports =
-        asList(
-            p2pPort,
-            graphQLHttpPort,
-            rpcHttpPort,
-            rpcWsPort,
-            metricsPort,
-            metricsPushPort,
-            stratumPort);
-    ports.stream()
+    getEffectivePorts().stream()
         .filter(Objects::nonNull)
+        .filter(port -> port > 0)
         .forEach(
             port -> {
-              if (port != 0 && !allocatedPorts.add(port)) {
+              if (!allocatedPorts.add(port)) {
                 throw new ParameterException(
                     commandLine,
                     "Port number '"
@@ -2408,13 +2393,73 @@ private void checkPortClash() {
             });
   }
 
-  private void checkGoQuorumCompatibilityConfig() {
-    if (genesisFile != null
-        && getGenesisConfigFile().getConfigOptions().isQuorum()
-        && !minTransactionGasPrice.isZero()) {
-      throw new ParameterException(
-          this.commandLine,
-          "--min-gas-price must be set to zero if GoQuorum compatibility is enabled in the genesis config.");
+  /**
+   * * Gets the list of effective ports (ports that are enabled).
+   *
+   * @return The list of effective ports
+   */
+  private List<Integer> getEffectivePorts() {
+    final List<Integer> effectivePorts = new ArrayList<>();
+    addPortIfEnabled(effectivePorts, p2pPort, p2pEnabled);
+    addPortIfEnabled(effectivePorts, graphQLHttpPort, isGraphQLHttpEnabled);
+    addPortIfEnabled(effectivePorts, rpcHttpPort, isRpcHttpEnabled);
+    addPortIfEnabled(effectivePorts, rpcWsPort, isRpcWsEnabled);
+    addPortIfEnabled(effectivePorts, metricsPort, isMetricsEnabled);
+    addPortIfEnabled(effectivePorts, metricsPushPort, isMetricsPushEnabled);
+    addPortIfEnabled(effectivePorts, stratumPort, iStratumMiningEnabled);
+    return effectivePorts;
+  }
+
+  /**
+   * Adds port in the passed list only if enabled.
+   *
+   * @param ports The list of ports
+   * @param port The port value
+   * @param enabled true if enabled, false otherwise
+   */
+  private void addPortIfEnabled(
+      final List<Integer> ports, final Integer port, final boolean enabled) {
+    if (enabled) {
+      ports.add(port);
+    }
+  }
+
+  private void checkGoQuorumCompatibilityConfig(final EthNetworkConfig ethNetworkConfig) {
+    if (isGoQuorumCompatibilityMode) {
+      final GenesisConfigOptions genesisConfigOptions = readGenesisConfigOptions();
+      // this static flag is read by the RLP decoder
+      GoQuorumOptions.goquorumCompatibilityMode = true;
+
+      if (!genesisConfigOptions.isQuorum()) {
+        throw new IllegalStateException(
+            "GoQuorum compatibility mode (enabled) can only be used if genesis file has 'isQuorum' flag set to true.");
+      }
+      genesisConfigOptions
+          .getChainId()
+          .ifPresent(
+              chainId ->
+                  ensureGoQuorumCompatibilityModeNotUsedOnMainnet(
+                      chainId, isGoQuorumCompatibilityMode));
+
+      if (genesisFile != null
+          && getGenesisConfigFile().getConfigOptions().isQuorum()
+          && !minTransactionGasPrice.isZero()) {
+        throw new ParameterException(
+            this.commandLine,
+            "--min-gas-price must be set to zero if GoQuorum compatibility is enabled in the genesis config.");
+      }
+      if (ethNetworkConfig.getNetworkId().equals(EthNetworkConfig.MAINNET_NETWORK_ID)) {
+        throw new ParameterException(
+            this.commandLine, "GoQuorum compatibility mode (enabled) cannot be used on Mainnet.");
+      }
+    }
+  }
+
+  private void ensureGoQuorumCompatibilityModeNotUsedOnMainnet(
+      final BigInteger chainId, final boolean isGoQuorumCompatibilityMode) {
+    if (isGoQuorumCompatibilityMode && chainId.equals(EthNetworkConfig.MAINNET_NETWORK_ID)) {
+      throw new IllegalStateException(
+          "GoQuorum compatibility mode (enabled) cannot be used on Mainnet.");
     }
   }
 
@@ -2434,5 +2479,13 @@ public Path getStoragePath() {
     public Path getDataPath() {
       return dataDir();
     }
+
+    @Override
+    public int getDatabaseVersion() {
+      return unstableDataStorageOptions
+          .toDomainObject()
+          .getDataStorageFormat()
+          .getDatabaseVersion();
+    }
   }
 }

besu/src/main/java/org/hyperledger/besu/cli/config/EthNetworkConfig.java

@@ -25,6 +25,7 @@
 import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.RINKEBY_BOOTSTRAP_NODES;
 import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.RINKEBY_DISCOVERY_URL;
 import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.ROPSTEN_BOOTSTRAP_NODES;
+import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.ROPSTEN_DISCOVERY_URL;
 import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.YOLO_V2_BOOTSTRAP_NODES;
 
 import org.hyperledger.besu.ethereum.p2p.peers.EnodeURL;
@@ -131,7 +132,10 @@ public static EthNetworkConfig getNetworkConfig(final NetworkName networkName) {
     switch (networkName) {
       case ROPSTEN:
         return new EthNetworkConfig(
-            jsonConfig(ROPSTEN_GENESIS), ROPSTEN_NETWORK_ID, ROPSTEN_BOOTSTRAP_NODES, null);
+            jsonConfig(ROPSTEN_GENESIS),
+            ROPSTEN_NETWORK_ID,
+            ROPSTEN_BOOTSTRAP_NODES,
+            ROPSTEN_DISCOVERY_URL);
       case RINKEBY:
         return new EthNetworkConfig(
             jsonConfig(RINKEBY_GENESIS),
@@ -205,7 +209,7 @@ public static String jsonConfig(final NetworkName network) {
 
   public static class Builder {
 
-    private final String dnsDiscoveryUrl;
+    private String dnsDiscoveryUrl;
     private String genesisConfig;
     private BigInteger networkId;
     private List<EnodeURL> bootNodes;
@@ -232,6 +236,11 @@ public Builder setBootNodes(final List<EnodeURL> bootNodes) {
       return this;
     }
 
+    public Builder setDnsDiscoveryUrl(final String dnsDiscoveryUrl) {
+      this.dnsDiscoveryUrl = dnsDiscoveryUrl;
+      return this;
+    }
+
     public EthNetworkConfig build() {
       return new EthNetworkConfig(genesisConfig, networkId, bootNodes, dnsDiscoveryUrl);
     }