Merge branch 'main' into blobdb-for-static-data

# Conflicts:
#	plugin-api/build.gradle

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## 23.4.1
 
 ### Breaking Changes
+- Add request content length limit for the JSON-RPC API (5MB) [#5467](https://github.com/hyperledger/besu/pull/5467)
 
 ### Additions and Improvements
 - Set the retention policy for RocksDB log files to maintain only the logs from the last week [#5428](https://github.com/hyperledger/besu/pull/5428)
@@ -12,6 +13,7 @@
 - New optional feature to save the txpool content to file on shutdown and reloading it on startup [#5434](https://github.com/hyperledger/besu/pull/5434)
 - New option to send SNI header in TLS ClientHello message [#5439](https://github.com/hyperledger/besu/pull/5439)
 - Early access - layered transaction pool implementation [#5290](https://github.com/hyperledger/besu/pull/5290)
+- New RPC method `debug_getRawReceipts`
 
 ### Bug Fixes
 - Fix eth_feeHistory response for the case in which blockCount is higher than highestBlock requested. [#5397](https://github.com/hyperledger/besu/pull/5397)

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java

@@ -386,7 +386,7 @@ public String getHostName() {
     return LOCALHOST;
   }
 
-  private NodeRequests nodeRequests() {
+  public NodeRequests nodeRequests() {
     Optional<WebSocketService> websocketService = Optional.empty();
     if (nodeRequests == null) {
       final Web3jService web3jService;

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java

@@ -45,7 +45,9 @@
 import org.hyperledger.besu.plugin.services.PicoCLIOptions;
 import org.hyperledger.besu.plugin.services.SecurityModuleService;
 import org.hyperledger.besu.plugin.services.StorageService;
+import org.hyperledger.besu.plugin.services.TransactionSelectionService;
 import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBPlugin;
+import org.hyperledger.besu.plugin.services.txselection.TransactionSelectorFactory;
 import org.hyperledger.besu.services.BesuConfigurationImpl;
 import org.hyperledger.besu.services.BesuEventsImpl;
 import org.hyperledger.besu.services.BesuPluginContextImpl;
@@ -54,6 +56,7 @@
 import org.hyperledger.besu.services.RpcEndpointServiceImpl;
 import org.hyperledger.besu.services.SecurityModuleServiceImpl;
 import org.hyperledger.besu.services.StorageServiceImpl;
+import org.hyperledger.besu.services.TransactionSelectionServiceImpl;
 
 import java.io.File;
 import java.nio.file.Path;
@@ -63,6 +66,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
@@ -91,14 +95,22 @@ private BesuPluginContextImpl buildPluginContext(
     besuPluginContext.addService(StorageService.class, storageService);
     besuPluginContext.addService(SecurityModuleService.class, securityModuleService);
     besuPluginContext.addService(PicoCLIOptions.class, new PicoCLIOptionsImpl(commandLine));
-
-    final Path pluginsPath = node.homeDirectory().resolve("plugins");
-    final File pluginsDirFile = pluginsPath.toFile();
-    if (!pluginsDirFile.isDirectory()) {
-      pluginsDirFile.mkdirs();
-      pluginsDirFile.deleteOnExit();
+    besuPluginContext.addService(
+        TransactionSelectionService.class, new TransactionSelectionServiceImpl());
+
+    final Path pluginsPath;
+    final String pluginDirEnv = System.getenv("besu.plugins.dir");
+    if (pluginDirEnv == null || pluginDirEnv.isEmpty()) {
+      pluginsPath = node.homeDirectory().resolve("plugins");
+      final File pluginsDirFile = pluginsPath.toFile();
+      if (!pluginsDirFile.isDirectory()) {
+        pluginsDirFile.mkdirs();
+        pluginsDirFile.deleteOnExit();
+      }
+    } else {
+      pluginsPath = Path.of(pluginDirEnv);
+      System.setProperty("besu.plugins.dir", pluginsPath.toString());
     }
-    System.setProperty("besu.plugins.dir", pluginsPath.toString());
     besuPluginContext.registerPlugins(pluginsPath);
 
     commandLine.parseArgs(node.getConfiguration().getExtraCLIOptions().toArray(new String[0]));
@@ -169,6 +181,9 @@ public void startNode(final BesuNode node) {
 
     final int maxPeers = 25;
 
+    final Optional<TransactionSelectorFactory> transactionSelectorFactory =
+        getTransactionSelectorFactory(besuPluginContext);
+
     builder
         .synchronizerConfiguration(new SynchronizerConfiguration.Builder().build())
         .dataDirectory(node.homeDirectory())
@@ -190,7 +205,8 @@ public void startNode(final BesuNode node) {
         .lowerBoundPeers(maxPeers)
         .maxRemotelyInitiatedPeers(15)
         .networkConfiguration(node.getNetworkingConfiguration())
-        .randomPeerPriority(false);
+        .randomPeerPriority(false)
+        .transactionSelectorFactory(transactionSelectorFactory);
 
     node.getGenesisConfig()
         .map(GenesisConfigFile::fromConfig)
@@ -299,4 +315,11 @@ public void startConsoleCapture() {
   public String getConsoleContents() {
     throw new RuntimeException("Console contents can only be captured in process execution");
   }
+
+  private Optional<TransactionSelectorFactory> getTransactionSelectorFactory(
+      final BesuPluginContextImpl besuPluginContext) {
+    final Optional<TransactionSelectionService> txSelectionService =
+        besuPluginContext.getService(TransactionSelectionService.class);
+    return txSelectionService.isPresent() ? txSelectionService.get().get() : Optional.empty();
+  }
 }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java

@@ -107,10 +107,33 @@ public BesuNode createMinerNode(
     return create(config);
   }
 
+  public BesuNode createMinerNodeWithExtraCliOptions(
+      final String name,
+      final UnaryOperator<BesuNodeConfigurationBuilder> configModifier,
+      final List<String> extraCliOptions)
+      throws IOException {
+    BesuNodeConfigurationBuilder builder =
+        new BesuNodeConfigurationBuilder()
+            .name(name)
+            .miningEnabled()
+            .jsonRpcEnabled()
+            .webSocketEnabled()
+            .extraCLIOptions(extraCliOptions);
+    builder = configModifier.apply(builder);
+    final BesuNodeConfiguration config = builder.build();
+
+    return create(config);
+  }
+
   public BesuNode createMinerNode(final String name) throws IOException {
     return createMinerNode(name, UnaryOperator.identity());
   }
 
+  public BesuNode createMinerNodeWithExtraCliOptions(
+      final String name, final List<String> extraCliOptions) throws IOException {
+    return createMinerNodeWithExtraCliOptions(name, UnaryOperator.identity(), extraCliOptions);
+  }
+
   public BesuNode createMinerNodeWithRevertReasonEnabled(final String name) throws IOException {
     return createMinerNode(name, BesuNodeConfigurationBuilder::revertReasonEnabled);
   }

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -167,12 +167,14 @@
 import org.hyperledger.besu.plugin.services.SecurityModuleService;
 import org.hyperledger.besu.plugin.services.StorageService;
 import org.hyperledger.besu.plugin.services.TraceService;
+import org.hyperledger.besu.plugin.services.TransactionSelectionService;
 import org.hyperledger.besu.plugin.services.exception.StorageException;
 import org.hyperledger.besu.plugin.services.metrics.MetricCategory;
 import org.hyperledger.besu.plugin.services.metrics.MetricCategoryRegistry;
 import org.hyperledger.besu.plugin.services.securitymodule.SecurityModule;
 import org.hyperledger.besu.plugin.services.storage.PrivacyKeyValueStorageFactory;
 import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBPlugin;
+import org.hyperledger.besu.plugin.services.txselection.TransactionSelectorFactory;
 import org.hyperledger.besu.services.BesuEventsImpl;
 import org.hyperledger.besu.services.BesuPluginContextImpl;
 import org.hyperledger.besu.services.BlockchainServiceImpl;
@@ -183,6 +185,7 @@
 import org.hyperledger.besu.services.SecurityModuleServiceImpl;
 import org.hyperledger.besu.services.StorageServiceImpl;
 import org.hyperledger.besu.services.TraceServiceImpl;
+import org.hyperledger.besu.services.TransactionSelectionServiceImpl;
 import org.hyperledger.besu.services.kvstore.InMemoryStoragePlugin;
 import org.hyperledger.besu.util.InvalidConfigurationException;
 import org.hyperledger.besu.util.LogConfigurator;
@@ -234,6 +237,7 @@
 import io.vertx.core.metrics.MetricsOptions;
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.units.bigints.UInt256;
+import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import picocli.AutoComplete;
 import picocli.CommandLine;
@@ -360,6 +364,8 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
   @CommandLine.ArgGroup(validate = false, heading = "@|bold P2P Discovery Options|@%n")
   P2PDiscoveryOptionGroup p2PDiscoveryOptionGroup = new P2PDiscoveryOptionGroup();
 
+  private final TransactionSelectionServiceImpl transactionSelectionServiceImpl;
+
   static class P2PDiscoveryOptionGroup {
 
     // Public IP stored to prevent having to research it each time we need it.
@@ -767,6 +773,12 @@ static class JsonRPCHttpOptionGroup {
         description =
             "Specifies the maximum number of requests in a single RPC batch request via RPC. -1 specifies no limit  (default: ${DEFAULT-VALUE})")
     private final Integer rpcHttpMaxBatchSize = DEFAULT_HTTP_MAX_BATCH_SIZE;
+
+    @CommandLine.Option(
+        names = {"--rpc-http-max-request-content-length"},
+        paramLabel = MANDATORY_LONG_FORMAT_HELP,
+        description = "Specifies the maximum request content length. (default: ${DEFAULT-VALUE})")
+    private final Long rpcHttpMaxRequestContentLength = DEFAULT_MAX_REQUEST_CONTENT_LENGTH;
   }
 
   // JSON-RPC Websocket Options
@@ -1374,7 +1386,8 @@ public BesuCommand(
         new PermissioningServiceImpl(),
         new PrivacyPluginServiceImpl(),
         new PkiBlockCreationConfigurationProvider(),
-        new RpcEndpointServiceImpl());
+        new RpcEndpointServiceImpl(),
+        new TransactionSelectionServiceImpl());
   }
 
   /**
@@ -1394,6 +1407,7 @@ public BesuCommand(
    * @param privacyPluginService instance of PrivacyPluginServiceImpl
    * @param pkiBlockCreationConfigProvider instance of PkiBlockCreationConfigurationProvider
    * @param rpcEndpointServiceImpl instance of RpcEndpointServiceImpl
+   * @param transactionSelectionServiceImpl instance of TransactionSelectionServiceImpl
    */
   @VisibleForTesting
   protected BesuCommand(
@@ -1410,7 +1424,8 @@ protected BesuCommand(
       final PermissioningServiceImpl permissioningService,
       final PrivacyPluginServiceImpl privacyPluginService,
       final PkiBlockCreationConfigurationProvider pkiBlockCreationConfigProvider,
-      final RpcEndpointServiceImpl rpcEndpointServiceImpl) {
+      final RpcEndpointServiceImpl rpcEndpointServiceImpl,
+      final TransactionSelectionServiceImpl transactionSelectionServiceImpl) {
     this.besuComponent = besuComponent;
     this.logger = besuComponent.getBesuCommandLogger();
     this.rlpBlockImporter = rlpBlockImporter;
@@ -1428,6 +1443,7 @@ protected BesuCommand(
     besuPluginContext.addService(BesuConfiguration.class, pluginCommonConfiguration);
     this.pkiBlockCreationConfigProvider = pkiBlockCreationConfigProvider;
     this.rpcEndpointServiceImpl = rpcEndpointServiceImpl;
+    this.transactionSelectionServiceImpl = transactionSelectionServiceImpl;
   }
 
   /**
@@ -1608,6 +1624,8 @@ private void preparePlugins() {
     besuPluginContext.addService(PermissioningService.class, permissioningService);
     besuPluginContext.addService(PrivacyPluginService.class, privacyPluginService);
     besuPluginContext.addService(RpcEndpointService.class, rpcEndpointServiceImpl);
+    besuPluginContext.addService(
+        TransactionSelectionService.class, transactionSelectionServiceImpl);
 
     // register built-in plugins
     rocksDBPlugin = new RocksDBPlugin();
@@ -1709,6 +1727,9 @@ private void startPlugins() {
                 besuController.getProtocolContext().getWorldStateArchive()),
             besuController.getProtocolSchedule()));
 
+    besuPluginContext.addService(
+        TransactionSelectionService.class, new TransactionSelectionServiceImpl());
+
     besuController.getAdditionalPluginServices().appendPluginServices(besuPluginContext);
     besuPluginContext.startPlugins();
   }
@@ -2235,12 +2256,15 @@ public BesuController buildController() {
    */
   public BesuControllerBuilder getControllerBuilder() {
     final KeyValueStorageProvider storageProvider = keyValueStorageProvider(keyValueStorageName);
+    final Optional<TransactionSelectorFactory> transactionSelectorFactory =
+        getTransactionSelectorFactory();
     return controllerBuilderFactory
         .fromEthNetworkConfig(
             updateNetworkConfig(network), genesisConfigOverrides, getDefaultSyncModeIfNotSet())
         .synchronizerConfiguration(buildSyncConfig())
         .ethProtocolConfiguration(unstableEthProtocolOptions.toDomainObject())
         .networkConfiguration(unstableNetworkingOptions.toDomainObject())
+        .transactionSelectorFactory(transactionSelectorFactory)
         .dataDirectory(dataDir())
         .miningParameters(
             new MiningParameters.Builder()
@@ -2290,6 +2314,13 @@ public BesuControllerBuilder getControllerBuilder() {
         .chainPruningConfiguration(unstableChainPruningOptions.toDomainObject());
   }
 
+  @NotNull
+  private Optional<TransactionSelectorFactory> getTransactionSelectorFactory() {
+    final Optional<TransactionSelectionService> txSelectionService =
+        besuPluginContext.getService(TransactionSelectionService.class);
+    return txSelectionService.isPresent() ? txSelectionService.get().get() : Optional.empty();
+  }
+
   private GraphQLConfiguration graphQLConfiguration() {
 
     CommandLineUtils.checkOptionDependencies(
@@ -2379,6 +2410,8 @@ && rpcHttpAuthenticationCredentialsFile() == null
     jsonRpcConfiguration.setTlsConfiguration(rpcHttpTlsConfiguration());
     jsonRpcConfiguration.setHttpTimeoutSec(unstableRPCOptions.getHttpTimeoutSec());
     jsonRpcConfiguration.setMaxBatchSize(jsonRPCHttpOptionGroup.rpcHttpMaxBatchSize);
+    jsonRpcConfiguration.setMaxRequestContentLength(
+        jsonRPCHttpOptionGroup.rpcHttpMaxRequestContentLength);
     return jsonRpcConfiguration;
   }
 

besu/src/main/java/org/hyperledger/besu/cli/DefaultCommandValues.java

@@ -88,6 +88,8 @@ public interface DefaultCommandValues {
   int DEFAULT_HTTP_MAX_CONNECTIONS = 80;
   /** The constant DEFAULT_HTTP_MAX_BATCH_SIZE. */
   int DEFAULT_HTTP_MAX_BATCH_SIZE = 1024;
+  /** The constant DEFAULT_MAX_REQUEST_CONTENT_LENGTH. */
+  long DEFAULT_MAX_REQUEST_CONTENT_LENGTH = 5 * 1024 * 1024; // 5MB
   /** The constant DEFAULT_WS_MAX_CONNECTIONS. */
   int DEFAULT_WS_MAX_CONNECTIONS = 80;
   /** The constant DEFAULT_WS_MAX_FRAME_SIZE. */

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -95,6 +95,7 @@
 import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 import org.hyperledger.besu.plugin.services.permissioning.NodeMessagePermissioningProvider;
+import org.hyperledger.besu.plugin.services.txselection.TransactionSelectorFactory;
 
 import java.io.Closeable;
 import java.math.BigInteger;
@@ -180,6 +181,7 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
 
   private NetworkingConfiguration networkingConfiguration;
   private Boolean randomPeerPriority;
+  private Optional<TransactionSelectorFactory> transactionSelectorFactory = Optional.empty();
   /** the Dagger configured context that can provide dependencies */
   protected Optional<BesuComponent> besuComponent = Optional.empty();
 
@@ -523,6 +525,18 @@ public BesuControllerBuilder randomPeerPriority(final Boolean randomPeerPriority
     return this;
   }
 
+  /**
+   * sets the transactionSelectorFactory in the builder
+   *
+   * @param transactionSelectorFactory the optional transaction selector factory
+   * @return the besu controller builder
+   */
+  public BesuControllerBuilder transactionSelectorFactory(
+      final Optional<TransactionSelectorFactory> transactionSelectorFactory) {
+    this.transactionSelectorFactory = transactionSelectorFactory;
+    return this;
+  }
+
   /**
    * Build besu controller.
    *
@@ -579,7 +593,11 @@ public BesuController build() {
 
     final ProtocolContext protocolContext =
         createProtocolContext(
-            blockchain, worldStateArchive, protocolSchedule, this::createConsensusContext);
+            blockchain,
+            worldStateArchive,
+            protocolSchedule,
+            this::createConsensusContext,
+            transactionSelectorFactory);
     validateContext(protocolContext);
 
     if (chainPrunerConfiguration.getChainPruningEnabled()) {
@@ -995,15 +1013,21 @@ protected EthProtocolManager createEthProtocolManager(
    * @param worldStateArchive the world state archive
    * @param protocolSchedule the protocol schedule
    * @param consensusContextFactory the consensus context factory
+   * @param transactionSelectorFactory optional transaction selector factory
    * @return the protocol context
    */
   protected ProtocolContext createProtocolContext(
       final MutableBlockchain blockchain,
       final WorldStateArchive worldStateArchive,
       final ProtocolSchedule protocolSchedule,
-      final ConsensusContextFactory consensusContextFactory) {
+      final ConsensusContextFactory consensusContextFactory,
+      final Optional<TransactionSelectorFactory> transactionSelectorFactory) {
     return ProtocolContext.init(
-        blockchain, worldStateArchive, protocolSchedule, consensusContextFactory);
+        blockchain,
+        worldStateArchive,
+        protocolSchedule,
+        consensusContextFactory,
+        transactionSelectorFactory);
   }
 
   private Optional<SnapProtocolManager> createSnapProtocolManager(

besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java

@@ -62,6 +62,7 @@
 import org.hyperledger.besu.evm.internal.EvmConfiguration;
 import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 import org.hyperledger.besu.plugin.services.permissioning.NodeMessagePermissioningProvider;
+import org.hyperledger.besu.plugin.services.txselection.TransactionSelectorFactory;
 
 import java.math.BigInteger;
 import java.nio.file.Path;
@@ -174,9 +175,14 @@ protected ProtocolContext createProtocolContext(
       final MutableBlockchain blockchain,
       final WorldStateArchive worldStateArchive,
       final ProtocolSchedule protocolSchedule,
-      final ConsensusContextFactory consensusContextFactory) {
+      final ConsensusContextFactory consensusContextFactory,
+      final Optional<TransactionSelectorFactory> transactionSelectorFactory) {
     return MigratingProtocolContext.init(
-        blockchain, worldStateArchive, protocolSchedule, consensusContextFactory);
+        blockchain,
+        worldStateArchive,
+        protocolSchedule,
+        consensusContextFactory,
+        transactionSelectorFactory);
   }
 
   @Override