updated from main, incorporates txpool refactoring

Signed-off-by: Justin Florentine <justin+github@florentine.us>

datatypes/src/main/java/org/hyperledger/besu/datatypes/VersionedHash.java

@@ -36,6 +36,9 @@ public class VersionedHash {
   /** The version id for sha256 hashes. */
   public static final byte SHA256_VERSION_ID = 1;
 
+  public static VersionedHash DEFAULT_VERSIONED_HASH =
+      new VersionedHash(SHA256_VERSION_ID, Hash.ZERO);
+
   /**
    * Construct a VersionedHash from a Bytes32 value.
    *

ethereum/api/build.gradle

@@ -95,6 +95,10 @@ dependencies {
   testImplementation 'io.vertx:vertx-junit5'
   testImplementation 'io.vertx:vertx-unit'
   testImplementation 'io.vertx:vertx-web-client'
+  testImplementation 'io.tmio:tuweni-bytes'
+  testImplementation 'io.tmio:tuweni-net'
+  testImplementation 'io.tmio:tuweni-toml'
+  testImplementation 'io.tmio:tuweni-units'
   testImplementation 'junit:junit'
   testImplementation 'org.apache.logging.log4j:log4j-core'
   testImplementation 'org.assertj:assertj-core'

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractEngineNewPayload.java

@@ -460,7 +460,7 @@ private boolean validateDataGasUsed(
       final List<VersionedHash> maybeVersionedHashes,
       final ProtocolSpec protocolSpec) {
     var calculatedDataGas =
-        protocolSpec.getGasCalculator().dataGasUsed(maybeVersionedHashes.size());
+        protocolSpec.getGasCalculator().dataGasCost(maybeVersionedHashes.size());
     return header.getDataGasUsed().orElse(0L).equals(calculatedDataGas);
   }
 

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthJsonRpcMethods.java

@@ -141,7 +141,7 @@ protected Map<String, JsonRpcMethod> create() {
         new EthGetTransactionByHash(blockchainQueries, transactionPool),
         new EthGetTransactionByBlockHashAndIndex(blockchainQueries),
         new EthGetTransactionByBlockNumberAndIndex(blockchainQueries),
-        new EthGetTransactionCount(blockchainQueries, transactionPool.getPendingTransactions()),
+        new EthGetTransactionCount(blockchainQueries, transactionPool),
         new EthGetTransactionReceipt(blockchainQueries, protocolSchedule),
         new EthUninstallFilter(filterManager),
         new EthGetFilterChanges(filterManager),

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/BlockchainQueries.java

@@ -673,7 +673,7 @@ public Optional<TransactionReceiptWithMetadata> transactionReceiptByTransactionH
   private Optional<Long> getDataGasUsed(
       final Transaction transaction, final ProtocolSpec protocolSpec) {
     return transaction.getType().supportsBlob()
-        ? Optional.of(protocolSpec.getGasCalculator().dataGasUsed(transaction.getBlobCount()))
+        ? Optional.of(protocolSpec.getGasCalculator().dataGasCost(transaction.getBlobCount()))
         : Optional.empty();
   }
 

ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractEngineNewPayloadTest.java

@@ -72,7 +72,6 @@
 import java.util.concurrent.CompletableFuture;
 
 import io.vertx.core.Vertx;
-import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
 import org.jetbrains.annotations.NotNull;
 import org.junit.Before;
@@ -104,8 +103,6 @@ AbstractEngineNewPayload create(
 
   private final MethodFactory methodFactory;
   protected AbstractEngineNewPayload method;
-  private final VersionedHash DEFAULT_VERSIONED_HASH =
-      new VersionedHash(Bytes32.wrap(Bytes.repeat((byte) 01, 32)));
 
   public AbstractEngineNewPayloadTest(final MethodFactory methodFactory) {
     this.methodFactory = methodFactory;
@@ -169,7 +166,11 @@ protected EnginePayloadParameter mockPayload(
       final List<WithdrawalParameter> withdrawals,
       final List<DepositParameter> deposits) {
     return mockPayload(
-        header, txs, withdrawals, deposits, List.of(DEFAULT_VERSIONED_HASH.toBytes()));
+        header,
+        txs,
+        withdrawals,
+        deposits,
+        List.of(VersionedHash.DEFAULT_VERSIONED_HASH.toBytes()));
   }
 
   protected EnginePayloadParameter mockPayload(

ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV3Test.java

@@ -70,7 +70,7 @@ public void before() {
 
   @Test
   public void shouldInvalidPayloadOnShortVersionedHash() {
-    Bytes shortHash = Bytes.repeat((byte) 0x69, 31);
+    Bytes shortHash = Bytes.fromHexString("0x" + "69".repeat(31));
     EnginePayloadParameter payload = mock(EnginePayloadParameter.class);
     JsonRpcResponse badParam =
         method.response(

ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/TransactionTracerTest.java

@@ -16,8 +16,10 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.lenient;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -116,7 +118,7 @@ public void setUp() throws Exception {
     when(blockchain.getChainHeadHeader()).thenReturn(blockHeader);
     when(protocolSpec.getBadBlocksManager()).thenReturn(new BadBlockManager());
     when(protocolSpec.getGasCalculator()).thenReturn(gasCalculator);
-    when(gasCalculator.computeExcessDataGas(anyLong(), anyLong())).thenReturn(0L);
+    lenient().when(gasCalculator.computeExcessDataGas(anyLong(), anyInt())).thenReturn(0L);
   }
 
   @Test

ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockCreator.java

@@ -230,30 +230,11 @@ protected BlockCreationResult createBlock(
 
       throwIfStopped();
 
-      DataGas newExcessDataGas = null;
-      Long newDataGasUsed = null;
-      if (newProtocolSpec.getFeeMarket().implementsDataFee()) {
-        final var gasCalculator = newProtocolSpec.getGasCalculator();
-        newExcessDataGas =
-            DataGas.of(
-                gasCalculator.computeExcessDataGas(
-                    // casting parent excess data gas to long since for the moment it should be well
-                    // below that limit
-                    parentHeader.getExcessDataGas().map(DataGas::toLong).orElse(0L),
-                    parentHeader.getDataGasUsed().orElse(0L)));
-
-        final int newBlobsCount =
-            transactionResults.getTransactionsByType(TransactionType.BLOB).stream()
-                .map(tx -> tx.getVersionedHashes().orElseThrow())
-                .mapToInt(List::size)
-                .sum();
-
-        newDataGasUsed = gasCalculator.dataGasUsed(newBlobsCount);
-      }
+      final GasUsage usage = computeExcessDataGas(transactionResults, newProtocolSpec);
 
       throwIfStopped();
 
-      final SealableBlockHeader sealableBlockHeader =
+      BlockHeaderBuilder builder =
           BlockHeaderBuilder.create()
               .populateFrom(processableBlockHeader)
               .ommersHash(BodyValidation.ommersHash(ommers))
@@ -268,10 +249,12 @@ protected BlockCreationResult createBlock(
                   withdrawalsCanBeProcessed
                       ? BodyValidation.withdrawalsRoot(maybeWithdrawals.get())
                       : null)
-              .depositsRoot(maybeDeposits.map(BodyValidation::depositsRoot).orElse(null))
-              .dataGasUsed(newDataGasUsed)
-              .excessDataGas(newExcessDataGas)
-              .buildSealableBlockHeader();
+              .depositsRoot(maybeDeposits.map(BodyValidation::depositsRoot).orElse(null));
+      if (usage != null) {
+        builder.dataGasUsed(usage.used.toLong()).excessDataGas(usage.excessDataGas);
+      }
+
+      final SealableBlockHeader sealableBlockHeader = builder.buildSealableBlockHeader();
 
       final BlockHeader blockHeader = createFinalBlockHeader(sealableBlockHeader);
 
@@ -302,6 +285,31 @@ List<Deposit> findDepositsFromReceipts(final TransactionSelectionResults transac
         .toList();
   }
 
+  record GasUsage(DataGas excessDataGas, DataGas used) {}
+  ;
+
+  private GasUsage computeExcessDataGas(
+      final TransactionSelectionResults transactionResults, final ProtocolSpec newProtocolSpec) {
+
+    if (newProtocolSpec.getFeeMarket().implementsDataFee()) {
+      final var gasCalculator = newProtocolSpec.getGasCalculator();
+      final int newBlobsCount =
+          transactionResults.getTransactionsByType(TransactionType.BLOB).stream()
+              .map(tx -> tx.getVersionedHashes().orElseThrow())
+              .mapToInt(List::size)
+              .sum();
+      // casting parent excess data gas to long since for the moment it should be well below that
+      // limit
+      DataGas ecessDataGas =
+          DataGas.of(
+              gasCalculator.computeExcessDataGas(
+                  parentHeader.getExcessDataGas().map(DataGas::toLong).orElse(0L), newBlobsCount));
+      DataGas used = DataGas.of(gasCalculator.dataGasCost(newBlobsCount));
+      return new GasUsage(ecessDataGas, used);
+    }
+    return null;
+  }
+
   private TransactionSelectionResults selectTransactions(
       final ProcessableBlockHeader processableBlockHeader,
       final MutableWorldState disposableWorldState,

ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/BlockTransactionSelector.java

@@ -79,15 +79,12 @@
  */
 public class BlockTransactionSelector {
 
-  public record TransactionValidationResult(
-      Transaction transaction, ValidationResult<TransactionInvalidReason> validationResult) {}
-
   public static class TransactionSelectionResults {
     private final List<Transaction> transactions = Lists.newArrayList();
     private final Map<TransactionType, List<Transaction>> transactionsByType =
         new EnumMap<>(TransactionType.class);
     private final List<TransactionReceipt> receipts = Lists.newArrayList();
-    private final List<TransactionValidationResult> invalidTransactions = Lists.newArrayList();
+    private final Map<Transaction, TransactionInvalidReason> invalidTransactions = new HashMap<>();
     private final List<TransactionSelectionResult> selectionResults = Lists.newArrayList();
     private long cumulativeGasUsed = 0;
     private long cumulativeDataGasUsed = 0;
@@ -115,9 +112,8 @@ private void update(
     }
 
     private void updateWithInvalidTransaction(
-        final Transaction transaction,
-        final ValidationResult<TransactionInvalidReason> validationResult) {
-      invalidTransactions.add(new TransactionValidationResult(transaction, validationResult));
+        final Transaction transaction, final TransactionInvalidReason invalidReason) {
+      invalidTransactions.put(transaction, invalidReason);
     }
 
     public List<Transaction> getTransactions() {
@@ -140,7 +136,7 @@ public long getCumulativeDataGasUsed() {
       return cumulativeDataGasUsed;
     }
 
-    public List<TransactionValidationResult> getInvalidTransactions() {
+    public Map<Transaction, TransactionInvalidReason> getInvalidTransactions() {
       return invalidTransactions;
     }
 
@@ -382,7 +378,7 @@ private TransactionSelectionResult evaluateTransaction(final Transaction transac
             transactionReceiptFactory.create(
                 transaction.getType(), effectiveResult, worldState, cumulativeGasUsed);
 
-        final long dataGasUsed = gasCalculator.dataGasUsed(transaction.getBlobCount());
+        final long dataGasUsed = gasCalculator.dataGasCost(transaction.getBlobCount());
 
         transactionSelectionResults.update(transaction, receipt, gasUsedByTransaction, dataGasUsed);
 
@@ -469,12 +465,8 @@ private boolean isTransientValidationError(final TransactionInvalidReason invali
         || invalidReason.equals(TransactionInvalidReason.NONCE_TOO_HIGH);
   }
 
-  private boolean isIncorrectNonce(final ValidationResult<TransactionInvalidReason> result) {
-    return result.getInvalidReason().equals(TransactionInvalidReason.NONCE_TOO_HIGH);
-  }
-
   private boolean transactionTooLargeForBlock(final Transaction transaction) {
-    final long dataGasUsed = gasCalculator.dataGasUsed(transaction.getBlobCount());
+    final long dataGasUsed = gasCalculator.dataGasCost(transaction.getBlobCount());
 
     if (dataGasUsed
         > gasLimitCalculator.currentDataGasLimit()

ethereum/core/build.gradle

@@ -27,8 +27,6 @@ jar {
   }
 }
 
-
-
 dependencies {
   api 'org.slf4j:slf4j-api'
   api 'org.web3j:core'
@@ -56,10 +54,10 @@ dependencies {
 
   implementation 'net.java.dev.jna:jna'
   implementation 'org.apache.commons:commons-lang3'
-  implementation 'org.apache.tuweni:tuweni-bytes'
-  implementation 'org.apache.tuweni:tuweni-concurrent'
-  implementation 'org.apache.tuweni:tuweni-units'
-  implementation 'org.apache.tuweni:tuweni-rlp'
+  implementation 'io.tmio:tuweni-bytes'
+  implementation 'io.tmio:tuweni-concurrent'
+  implementation 'io.tmio:tuweni-units'
+  implementation 'io.tmio:tuweni-rlp'
   implementation 'org.hyperledger.besu:bls12-381'
   implementation 'org.immutables:value-annotations'
   implementation 'tech.pegasys:jc-kzg-4844'
@@ -100,6 +98,7 @@ dependencies {
   integrationTestImplementation 'org.junit.jupiter:junit-jupiter-api'
   integrationTestImplementation 'org.mockito:mockito-core'
   integrationTestImplementation 'org.testcontainers:testcontainers'
+  integrationTestImplementation 'io.tmio:tuweni-bytes'
 
   integrationTestRuntimeOnly 'org.junit.jupiter:junit-jupiter'
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/DefaultProtocolSchedule.java

@@ -131,7 +131,7 @@ public Optional<ScheduledProtocolSpec.Hardfork> hardforkFor(
   }
 
   @Override
-  public void setTransactionFilter(final TransactionFilter transactionFilter) {
+  public void setTransactionFilter(final PermissionTransactionFilter permissionTransactionFilter) {
     protocolSpecs.forEach(
         spec ->
             spec.spec()

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java

@@ -305,7 +305,7 @@ public TransactionProcessingResult processTransaction(
       final Wei transactionGasPrice =
           feeMarket.getTransactionPriceCalculator().price(transaction, blockHeader.getBaseFee());
 
-      final long dataGas = gasCalculator.dataGasUsed(transaction.getBlobCount());
+      final long dataGas = gasCalculator.dataGasCost(transaction.getBlobCount());
 
       final Wei upfrontGasCost =
           transaction.getUpfrontGasCost(transactionGasPrice, dataGasPrice, dataGas);

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionValidator.java

@@ -28,7 +28,6 @@
 import org.hyperledger.besu.ethereum.GasLimitCalculator;
 import org.hyperledger.besu.ethereum.core.PermissionTransactionFilter;
 import org.hyperledger.besu.ethereum.core.Transaction;
-import org.hyperledger.besu.ethereum.core.TransactionFilter;
 import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket;
 import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason;
 import org.hyperledger.besu.evm.account.Account;
@@ -187,7 +186,7 @@ private ValidationResult<TransactionInvalidReason> validateCostAndFee(
     }
 
     if (transaction.getType().supportsBlob()) {
-      final long txTotalDataGas = gasCalculator.dataGasUsed(transaction.getBlobCount());
+      final long txTotalDataGas = gasCalculator.dataGasCost(transaction.getBlobCount());
       if (txTotalDataGas > gasLimitCalculator.currentDataGasLimit()) {
         return ValidationResult.invalid(
             TransactionInvalidReason.TOTAL_DATA_GAS_TOO_HIGH,
@@ -198,7 +197,7 @@ private ValidationResult<TransactionInvalidReason> validateCostAndFee(
     }
 
     if (transaction.getType().supportsBlob()) {
-      final long txTotalDataGas = gasCalculator.dataGasUsed(transaction.getBlobCount());
+      final long txTotalDataGas = gasCalculator.dataGasCost(transaction.getBlobCount());
       if (txTotalDataGas > gasLimitCalculator.currentDataGasLimit()) {
         return ValidationResult.invalid(
             TransactionInvalidReason.TOTAL_DATA_GAS_TOO_HIGH,
@@ -239,7 +238,7 @@ public ValidationResult<TransactionInvalidReason> validateForSender(
     }
 
     final Wei upfrontCost =
-        transaction.getUpfrontCost(gasCalculator.dataGasUsed(transaction.getBlobCount()));
+        transaction.getUpfrontCost(gasCalculator.dataGasCost(transaction.getBlobCount()));
     if (upfrontCost.compareTo(senderBalance) > 0) {
       return ValidationResult.invalid(
           TransactionInvalidReason.UPFRONT_COST_EXCEEDS_BALANCE,

ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockDataGenerator.java

@@ -79,9 +79,6 @@ public class BlockDataGenerator {
   private final SignatureAlgorithm signatureAlgorithm = SignatureAlgorithmFactory.getInstance();
   private Supplier<BlockOptions> blockOptionsSupplier = BlockOptions::create;
 
-  private final VersionedHash DEFAULT_VERSIONED_HASH =
-      new VersionedHash(Bytes32.wrap(Bytes.repeat((byte) 01, 32)));
-
   public BlockDataGenerator(final int seed) {
     this.random = new Random(seed);
     keyPairGenerator = createKeyPairGenerator(seed);
@@ -446,7 +443,7 @@ private Transaction blobTransaction(final Bytes payload, final Address to) {
         .payload(payload)
         .chainId(BigInteger.ONE)
         .maxFeePerDataGas(Wei.of(1))
-        .versionedHashes(List.of(DEFAULT_VERSIONED_HASH))
+        .versionedHashes(List.of(VersionedHash.DEFAULT_VERSIONED_HASH))
         .signAndBuild(generateKeyPair());
   }
 

ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TransactionTestFixture.java

@@ -27,13 +27,10 @@
 import java.util.Optional;
 
 import org.apache.tuweni.bytes.Bytes;
-import org.apache.tuweni.bytes.Bytes32;
 
 public class TransactionTestFixture {
 
   private TransactionType transactionType = TransactionType.FRONTIER;
-  private final VersionedHash DEFAULT_VERSIONED_HASH =
-      new VersionedHash(Bytes32.wrap(Bytes.repeat((byte) 01, 32)));
 
   private long nonce = 0;
 
@@ -88,7 +85,8 @@ public Transaction createTransaction(final KeyPair keys) {
         builder.maxFeePerGas(maxFeePerGas.orElse(Wei.of(5000)));
         builder.accessList(accessListEntries.orElse(List.of()));
         builder.maxFeePerDataGas(maxFeePerDataGas.orElse(Wei.ONE));
-        builder.versionedHashes(versionedHashes.orElse(List.of(DEFAULT_VERSIONED_HASH)));
+        builder.versionedHashes(
+            versionedHashes.orElse(List.of(VersionedHash.DEFAULT_VERSIONED_HASH)));
         blobs.ifPresent(
             bwc -> {
               builder.kzgBlobs(bwc.getKzgCommitments(), bwc.getBlobs(), bwc.getKzgProofs());