From c01fee7aa289e8e0a39c401539256cb8ed413d71 Mon Sep 17 00:00:00 2001 From: Mehdi AOUADI Date: Mon, 30 Sep 2024 14:52:21 +0200 Subject: [PATCH 01/14] add engine get blobs --- .../ExecutionEngineClient.java | 3 + .../ThrottlingExecutionEngineClient.java | 7 ++ .../methods/EngineApiMethod.java | 3 +- .../methods/EngineGetBlobsV1.java | 86 ++++++++++++++++++ .../MetricRecordingExecutionEngineClient.java | 8 ++ .../schema/BlobAndProofV1.java | 88 +++++++++++++++++++ .../web3j/Web3JExecutionEngineClient.java | 17 ++++ .../ExecutionClientHandler.java | 5 ++ .../ExecutionClientHandlerImpl.java | 22 +++++ .../ExecutionLayerManagerImpl.java | 9 ++ ...toneBasedEngineJsonRpcMethodsResolver.java | 3 + .../execution/BlobAndProof.java | 19 ++++ .../executionlayer/ExecutionLayerChannel.java | 11 +++ .../ExecutionLayerChannelStub.java | 19 ++++ 14 files changed, 299 insertions(+), 1 deletion(-) create mode 100644 ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java create mode 100644 ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/schema/BlobAndProofV1.java create mode 100644 ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/execution/BlobAndProof.java diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/ExecutionEngineClient.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/ExecutionEngineClient.java index 02ee14282b5..d55c149f834 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/ExecutionEngineClient.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/ExecutionEngineClient.java @@ -16,6 +16,7 @@ import java.util.List; import java.util.Optional; import org.apache.tuweni.bytes.Bytes32; +import tech.pegasys.teku.ethereum.executionclient.schema.BlobAndProofV1; import tech.pegasys.teku.ethereum.executionclient.schema.ClientVersionV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV2; @@ -77,4 +78,6 @@ SafeFuture> forkChoiceUpdatedV3( SafeFuture>> exchangeCapabilities(List capabilities); SafeFuture>> getClientVersionV1(ClientVersionV1 clientVersion); + + SafeFuture>> getBlobsV1(List blobVersionedHashes); } diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/ThrottlingExecutionEngineClient.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/ThrottlingExecutionEngineClient.java index 964c527cd4b..b67c57e2cd0 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/ThrottlingExecutionEngineClient.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/ThrottlingExecutionEngineClient.java @@ -17,6 +17,7 @@ import java.util.Optional; import org.apache.tuweni.bytes.Bytes32; import org.hyperledger.besu.plugin.services.MetricsSystem; +import tech.pegasys.teku.ethereum.executionclient.schema.BlobAndProofV1; import tech.pegasys.teku.ethereum.executionclient.schema.ClientVersionV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV2; @@ -150,4 +151,10 @@ public SafeFuture>> getClientVersionV1( final ClientVersionV1 clientVersion) { return taskQueue.queueTask(() -> delegate.getClientVersionV1(clientVersion)); } + + @Override + public SafeFuture>> getBlobsV1( + final List blobVersionedHashes) { + return taskQueue.queueTask(() -> delegate.getBlobsV1(blobVersionedHashes)); + } } diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineApiMethod.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineApiMethod.java index 44ec33d7a4c..a47654c47fe 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineApiMethod.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineApiMethod.java @@ -16,7 +16,8 @@ public enum EngineApiMethod { ENGINE_NEW_PAYLOAD("engine_newPayload"), ENGINE_GET_PAYLOAD("engine_getPayload"), - ENGINE_FORK_CHOICE_UPDATED("engine_forkchoiceUpdated"); + ENGINE_FORK_CHOICE_UPDATED("engine_forkchoiceUpdated"), + ENGINE_GET_BLOBS("engine_getBlobs"); private final String name; diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java new file mode 100644 index 00000000000..f6183289b58 --- /dev/null +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java @@ -0,0 +1,86 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.ethereum.executionclient.methods; + +import java.util.Collections; +import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import tech.pegasys.teku.ethereum.executionclient.ExecutionEngineClient; +import tech.pegasys.teku.ethereum.executionclient.response.ResponseUnwrapper; +import tech.pegasys.teku.infrastructure.async.SafeFuture; +import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.spec.Spec; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSchema; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; +import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; +import tech.pegasys.teku.spec.schemas.SchemaDefinitions; +import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb; + +public class EngineGetBlobsV1 extends AbstractEngineJsonRpcMethod> { + + private static final Logger LOG = LogManager.getLogger(); + private final Spec spec; + + public EngineGetBlobsV1(final ExecutionEngineClient executionEngineClient, final Spec spec) { + super(executionEngineClient); + this.spec = spec; + } + + @Override + public String getName() { + return EngineApiMethod.ENGINE_GET_BLOBS.getName(); + } + + @Override + public int getVersion() { + return 1; + } + + @Override + public SafeFuture> execute(final JsonRpcRequestParams params) { + + final List blobVersionedHashes = + params.getRequiredListParameter(0, VersionedHash.class); + + final UInt64 slot = params.getRequiredParameter(1, UInt64.class); + + LOG.trace( + "Calling {}(blobVersionedHashes={}, slot={})", + getVersionedName(), + blobVersionedHashes, + slot); + + return executionEngineClient + .getBlobsV1(blobVersionedHashes) + .thenApply(ResponseUnwrapper::unwrapExecutionClientResponseOrThrow) + .thenApply( + response -> { + final SchemaDefinitions schemaDefinitions = spec.atSlot(slot).getSchemaDefinitions(); + final BlobSchema blobSchema = + SchemaDefinitionsDeneb.required(schemaDefinitions).getBlobSchema(); + return response.stream() + .map(blobAndProofV1 -> blobAndProofV1.asInternalBlobsAndProofs(blobSchema)) + .toList(); + }) + .thenPeek( + blobsAndProofs -> + LOG.trace( + "Response {}(blobVersionedHashes={}) -> {}", + getVersionedName(), + blobVersionedHashes, + blobsAndProofs)) + .exceptionally(throwable -> Collections.emptyList()); + } +} diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/metrics/MetricRecordingExecutionEngineClient.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/metrics/MetricRecordingExecutionEngineClient.java index f45056c92a0..574a7141b07 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/metrics/MetricRecordingExecutionEngineClient.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/metrics/MetricRecordingExecutionEngineClient.java @@ -19,6 +19,7 @@ import org.apache.tuweni.bytes.Bytes32; import org.hyperledger.besu.plugin.services.MetricsSystem; import tech.pegasys.teku.ethereum.executionclient.ExecutionEngineClient; +import tech.pegasys.teku.ethereum.executionclient.schema.BlobAndProofV1; import tech.pegasys.teku.ethereum.executionclient.schema.ClientVersionV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV2; @@ -66,6 +67,7 @@ public class MetricRecordingExecutionEngineClient extends MetricRecordingAbstrac public static final String NEW_PAYLOAD_V4_METHOD = "new_payloadV4"; public static final String EXCHANGE_CAPABILITIES_METHOD = "exchange_capabilities"; public static final String GET_CLIENT_VERSION_V1_METHOD = "get_client_versionV1"; + public static final String GET_BLOBS_V1_METHOD = "get_blobs_versionV1"; private final ExecutionEngineClient delegate; @@ -192,4 +194,10 @@ public SafeFuture>> getClientVersionV1( return countRequest( () -> delegate.getClientVersionV1(clientVersion), GET_CLIENT_VERSION_V1_METHOD); } + + @Override + public SafeFuture>> getBlobsV1( + final List blobVersionedHashes) { + return countRequest(() -> delegate.getBlobsV1(blobVersionedHashes), GET_BLOBS_V1_METHOD); + } } diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/schema/BlobAndProofV1.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/schema/BlobAndProofV1.java new file mode 100644 index 00000000000..fb6d978bf68 --- /dev/null +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/schema/BlobAndProofV1.java @@ -0,0 +1,88 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.ethereum.executionclient.schema; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.google.common.base.MoreObjects; +import java.util.Objects; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes48; +import tech.pegasys.teku.ethereum.executionclient.serialization.Bytes48Deserializer; +import tech.pegasys.teku.ethereum.executionclient.serialization.BytesDeserializer; +import tech.pegasys.teku.ethereum.executionclient.serialization.BytesSerializer; +import tech.pegasys.teku.kzg.KZGProof; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.Blob; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSchema; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; + +public class BlobAndProofV1 { + + @JsonSerialize(contentUsing = BytesSerializer.class) + @JsonDeserialize(contentUsing = BytesDeserializer.class) + private final Bytes blob; + + @JsonSerialize(contentUsing = BytesSerializer.class) + @JsonDeserialize(contentUsing = Bytes48Deserializer.class) + private final Bytes48 proof; + + public BlobAndProofV1( + @JsonProperty("blob") final Bytes blob, @JsonProperty("proof") final Bytes48 proof) { + checkNotNull(blob, "blob"); + checkNotNull(proof, "proof"); + this.proof = proof; + this.blob = blob; + } + + public BlobAndProof asInternalBlobsAndProofs(final BlobSchema blobSchema) { + return new BlobAndProof(new Blob(blobSchema, blob), new KZGProof(proof)); + } + + public static BlobAndProofV1 fromInternalBlobsBundle(final BlobAndProof blobAndProof) { + return new BlobAndProofV1( + blobAndProof.blob().getBytes(), blobAndProof.proof().getBytesCompressed()); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final BlobAndProofV1 that = (BlobAndProofV1) o; + return Objects.equals(blob, that.blob) && Objects.equals(proof, that.proof); + } + + @Override + public int hashCode() { + return Objects.hash(blob, proof); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("blob", bytesToBriefString(blob)) + .add("proof", bytesToBriefString(proof)) + .toString(); + } + + private String bytesToBriefString(final Bytes bytes) { + return bytes.slice(0, 7).toUnprefixedHexString(); + } +} diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java index f5550f60b1e..a21e0424f84 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java @@ -27,6 +27,7 @@ import org.web3j.protocol.core.Request; import org.web3j.protocol.core.methods.response.EthBlock; import tech.pegasys.teku.ethereum.executionclient.ExecutionEngineClient; +import tech.pegasys.teku.ethereum.executionclient.schema.BlobAndProofV1; import tech.pegasys.teku.ethereum.executionclient.schema.ClientVersionV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV2; @@ -52,6 +53,7 @@ public class Web3JExecutionEngineClient implements ExecutionEngineClient { private static final Duration EXCHANGE_CAPABILITIES_TIMEOUT = Duration.ofSeconds(1); private static final Duration GET_CLIENT_VERSION_TIMEOUT = Duration.ofSeconds(1); + private static final Duration GET_BLOBS_TIMEOUT = Duration.ofSeconds(1); private final Web3JClient web3JClient; @@ -253,6 +255,18 @@ public SafeFuture>> getClientVersionV1( return web3JClient.doRequest(web3jRequest, GET_CLIENT_VERSION_TIMEOUT); } + @Override + public SafeFuture>> getBlobsV1( + List blobVersionedHashes) { + final Request web3jRequest = + new Request<>( + "engine_getBlobsVersionV1", + Collections.singletonList(blobVersionedHashes), + web3JClient.getWeb3jService(), + GetBlobsVersionV1Web3jResponse.class); + return web3JClient.doRequest(web3jRequest, GET_BLOBS_TIMEOUT); + } + static class ExecutionPayloadV1Web3jResponse extends org.web3j.protocol.core.Response {} @@ -277,6 +291,9 @@ static class ExchangeCapabilitiesWeb3jResponse static class GetClientVersionV1Web3jResponse extends org.web3j.protocol.core.Response> {} + static class GetBlobsVersionV1Web3jResponse + extends org.web3j.protocol.core.Response> {} + /** * Returns a list that supports null items. * diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandler.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandler.java index 9c8290fd1d6..134f9fae8a4 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandler.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandler.java @@ -18,6 +18,7 @@ import org.apache.tuweni.bytes.Bytes32; import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; import tech.pegasys.teku.spec.datastructures.execution.ClientVersion; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayloadContext; import tech.pegasys.teku.spec.datastructures.execution.GetPayloadResponse; @@ -27,6 +28,7 @@ import tech.pegasys.teku.spec.executionlayer.ForkChoiceUpdatedResult; import tech.pegasys.teku.spec.executionlayer.PayloadBuildingAttributes; import tech.pegasys.teku.spec.executionlayer.PayloadStatus; +import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; public interface ExecutionClientHandler { @@ -44,4 +46,7 @@ SafeFuture engineGetPayload( SafeFuture engineNewPayload(NewPayloadRequest newPayloadRequest); SafeFuture> engineGetClientVersion(ClientVersion clientVersion); + + SafeFuture> engineGetBlobs( + List blobVersionedHashes, UInt64 slot); } diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandlerImpl.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandlerImpl.java index 301d4d46ca0..d46ed1f2bb6 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandlerImpl.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandlerImpl.java @@ -24,6 +24,8 @@ import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSchema; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; import tech.pegasys.teku.spec.datastructures.execution.ClientVersion; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayload; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayloadContext; @@ -34,6 +36,9 @@ import tech.pegasys.teku.spec.executionlayer.ForkChoiceUpdatedResult; import tech.pegasys.teku.spec.executionlayer.PayloadBuildingAttributes; import tech.pegasys.teku.spec.executionlayer.PayloadStatus; +import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; +import tech.pegasys.teku.spec.schemas.SchemaDefinitions; +import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb; public class ExecutionClientHandlerImpl implements ExecutionClientHandler { @@ -123,4 +128,21 @@ public SafeFuture> engineGetClientVersion(final ClientVersio clientVersions -> clientVersions.stream().map(ClientVersionV1::asInternalClientVersion).toList()); } + + @Override + public SafeFuture> engineGetBlobs( + final List blobVersionedHashes, final UInt64 slot) { + // TODO Make it versioned + final SchemaDefinitions schemaDefinitions = spec.atSlot(slot).getSchemaDefinitions(); + final BlobSchema blobSchema = + SchemaDefinitionsDeneb.required(schemaDefinitions).getBlobSchema(); + return executionEngineClient + .getBlobsV1(blobVersionedHashes) + .thenApply(ResponseUnwrapper::unwrapExecutionClientResponseOrThrow) + .thenApply( + blobAndProofV1s -> + blobAndProofV1s.stream() + .map(blobAndProofV1 -> blobAndProofV1.asInternalBlobsAndProofs(blobSchema)) + .toList()); + } } diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImpl.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImpl.java index 3dad1c3b220..acf21f010c8 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImpl.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImpl.java @@ -47,6 +47,7 @@ import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.builder.SignedValidatorRegistration; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; import tech.pegasys.teku.spec.datastructures.execution.BuilderBidOrFallbackData; import tech.pegasys.teku.spec.datastructures.execution.BuilderPayloadOrFallbackData; import tech.pegasys.teku.spec.datastructures.execution.ClientVersion; @@ -61,6 +62,7 @@ import tech.pegasys.teku.spec.executionlayer.ForkChoiceUpdatedResult; import tech.pegasys.teku.spec.executionlayer.PayloadBuildingAttributes; import tech.pegasys.teku.spec.executionlayer.PayloadStatus; +import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; public class ExecutionLayerManagerImpl implements ExecutionLayerManager { @@ -218,6 +220,13 @@ public SafeFuture> engineGetClientVersion(final ClientVersio return executionClientHandler.engineGetClientVersion(clientVersion); } + @Override + public SafeFuture> engineGetBlobs( + final List blobVersionedHashes, final UInt64 slot) { + LOG.trace("calling engineGetBlobs(blobVersionedHashes={}, slot={})", blobVersionedHashes, slot); + return executionClientHandler.engineGetBlobs(blobVersionedHashes, slot); + } + @Override public SafeFuture builderRegisterValidators( final SszList signedValidatorRegistrations, final UInt64 slot) { diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java index 992aa0e5095..ab19a273df0 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java @@ -14,6 +14,7 @@ package tech.pegasys.teku.ethereum.executionlayer; import static tech.pegasys.teku.ethereum.executionclient.methods.EngineApiMethod.ENGINE_FORK_CHOICE_UPDATED; +import static tech.pegasys.teku.ethereum.executionclient.methods.EngineApiMethod.ENGINE_GET_BLOBS; import static tech.pegasys.teku.ethereum.executionclient.methods.EngineApiMethod.ENGINE_GET_PAYLOAD; import static tech.pegasys.teku.ethereum.executionclient.methods.EngineApiMethod.ENGINE_NEW_PAYLOAD; @@ -29,6 +30,7 @@ import tech.pegasys.teku.ethereum.executionclient.methods.EngineForkChoiceUpdatedV1; import tech.pegasys.teku.ethereum.executionclient.methods.EngineForkChoiceUpdatedV2; import tech.pegasys.teku.ethereum.executionclient.methods.EngineForkChoiceUpdatedV3; +import tech.pegasys.teku.ethereum.executionclient.methods.EngineGetBlobsV1; import tech.pegasys.teku.ethereum.executionclient.methods.EngineGetPayloadV1; import tech.pegasys.teku.ethereum.executionclient.methods.EngineGetPayloadV2; import tech.pegasys.teku.ethereum.executionclient.methods.EngineGetPayloadV3; @@ -116,6 +118,7 @@ private Map> electraSupportedMethods() { methods.put(ENGINE_NEW_PAYLOAD, new EngineNewPayloadV4(executionEngineClient)); methods.put(ENGINE_GET_PAYLOAD, new EngineGetPayloadV4(executionEngineClient, spec)); methods.put(ENGINE_FORK_CHOICE_UPDATED, new EngineForkChoiceUpdatedV3(executionEngineClient)); + methods.put(ENGINE_GET_BLOBS, new EngineGetBlobsV1(executionEngineClient, spec)); return methods; } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/execution/BlobAndProof.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/execution/BlobAndProof.java new file mode 100644 index 00000000000..8509e4be196 --- /dev/null +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/execution/BlobAndProof.java @@ -0,0 +1,19 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.spec.datastructures.execution; + +import tech.pegasys.teku.kzg.KZGProof; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.Blob; + +public record BlobAndProof(Blob blob, KZGProof proof) {} diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannel.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannel.java index 77ec790fa06..c3723a961db 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannel.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannel.java @@ -25,6 +25,7 @@ import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.builder.SignedValidatorRegistration; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; import tech.pegasys.teku.spec.datastructures.execution.BuilderBidOrFallbackData; import tech.pegasys.teku.spec.datastructures.execution.BuilderPayloadOrFallbackData; import tech.pegasys.teku.spec.datastructures.execution.ClientVersion; @@ -34,6 +35,7 @@ import tech.pegasys.teku.spec.datastructures.execution.NewPayloadRequest; import tech.pegasys.teku.spec.datastructures.execution.PowBlock; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; +import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; public interface ExecutionLayerChannel extends ChannelInterface { String PREVIOUS_STUB_ENDPOINT_PREFIX = "stub"; @@ -76,6 +78,12 @@ public SafeFuture> engineGetClientVersion( return SafeFuture.completedFuture(List.of()); } + @Override + public SafeFuture> engineGetBlobs( + final List blobVersionedHashes, final UInt64 slot) { + return SafeFuture.completedFuture(List.of()); + } + @Override public SafeFuture builderRegisterValidators( final SszList signedValidatorRegistrations, @@ -115,6 +123,9 @@ SafeFuture engineForkChoiceUpdated( SafeFuture> engineGetClientVersion(ClientVersion clientVersion); + SafeFuture> engineGetBlobs( + List blobVersionedHashes, UInt64 slot); + /** * This is low level method, use {@link * ExecutionLayerBlockProductionManager#initiateBlockProduction(ExecutionPayloadContext, diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannelStub.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannelStub.java index f1777120e15..608092c3b7e 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannelStub.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannelStub.java @@ -27,6 +27,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; +import java.util.stream.IntStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.tuweni.bytes.Bytes; @@ -53,6 +54,7 @@ import tech.pegasys.teku.spec.datastructures.builder.BuilderBid; import tech.pegasys.teku.spec.datastructures.builder.BuilderPayload; import tech.pegasys.teku.spec.datastructures.builder.SignedValidatorRegistration; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; import tech.pegasys.teku.spec.datastructures.execution.BlobsBundle; import tech.pegasys.teku.spec.datastructures.execution.BuilderBidOrFallbackData; import tech.pegasys.teku.spec.datastructures.execution.BuilderPayloadOrFallbackData; @@ -69,6 +71,7 @@ import tech.pegasys.teku.spec.datastructures.type.SszKZGCommitment; import tech.pegasys.teku.spec.datastructures.util.BlobsUtil; import tech.pegasys.teku.spec.datastructures.util.DepositRequestsUtil; +import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; import tech.pegasys.teku.spec.schemas.SchemaDefinitions; import tech.pegasys.teku.spec.schemas.SchemaDefinitionsBellatrix; import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb; @@ -348,6 +351,22 @@ public SafeFuture> engineGetClientVersion(final ClientVersio return SafeFuture.completedFuture(List.of(STUB_CLIENT_VERSION)); } + @Override + public SafeFuture> engineGetBlobs( + final List blobVersionedHashes, final UInt64 slot) { + final List blobs = + blobsUtil.generateBlobs( + slot, + blobsToGenerate.orElseGet( + () -> random.nextInt(spec.getMaxBlobsPerBlock().orElseThrow() + 1))); + final List commitments = blobsUtil.blobsToKzgCommitments(blobs); + final List proofs = blobsUtil.computeKzgProofs(blobs, commitments); + return SafeFuture.completedFuture( + IntStream.range(0, blobs.size()) + .mapToObj(index -> new BlobAndProof(blobs.get(index), proofs.get(index))) + .toList()); + } + @Override public SafeFuture builderRegisterValidators( final SszList signedValidatorRegistrations, final UInt64 slot) { From db389fa0fba1581ee07ebf8ece41f71b439455bd Mon Sep 17 00:00:00 2001 From: Mehdi AOUADI Date: Mon, 30 Sep 2024 16:10:34 +0200 Subject: [PATCH 02/14] make engine get blobs versioned --- .../web3j/Web3JExecutionEngineClient.java | 2 +- .../EngineJsonRpcMethodsResolver.java | 4 ++++ .../ExecutionClientHandlerImpl.java | 23 +++++++------------ ...toneBasedEngineJsonRpcMethodsResolver.java | 20 ++++++++++++++++ 4 files changed, 33 insertions(+), 16 deletions(-) diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java index a21e0424f84..a7fae4b6f9e 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java @@ -257,7 +257,7 @@ public SafeFuture>> getClientVersionV1( @Override public SafeFuture>> getBlobsV1( - List blobVersionedHashes) { + final List blobVersionedHashes) { final Request web3jRequest = new Request<>( "engine_getBlobsVersionV1", diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineJsonRpcMethodsResolver.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineJsonRpcMethodsResolver.java index bcf580c3659..97b16c2acac 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineJsonRpcMethodsResolver.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineJsonRpcMethodsResolver.java @@ -13,6 +13,7 @@ package tech.pegasys.teku.ethereum.executionlayer; +import java.util.List; import java.util.Set; import java.util.function.Supplier; import tech.pegasys.teku.ethereum.executionclient.methods.EngineApiMethod; @@ -24,6 +25,9 @@ public interface EngineJsonRpcMethodsResolver { EngineJsonRpcMethod getMethod( EngineApiMethod method, Supplier milestoneSupplier, Class resultType); + EngineJsonRpcMethod> getMethodList( + EngineApiMethod method, Supplier milestoneSupplier, Class resultType); + /** * Get CL capabilities required for the engine_exchangeCapabilities diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandlerImpl.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandlerImpl.java index d46ed1f2bb6..8b12ad2df03 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandlerImpl.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandlerImpl.java @@ -24,7 +24,6 @@ import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSchema; import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; import tech.pegasys.teku.spec.datastructures.execution.ClientVersion; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayload; @@ -37,8 +36,6 @@ import tech.pegasys.teku.spec.executionlayer.PayloadBuildingAttributes; import tech.pegasys.teku.spec.executionlayer.PayloadStatus; import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; -import tech.pegasys.teku.spec.schemas.SchemaDefinitions; -import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb; public class ExecutionClientHandlerImpl implements ExecutionClientHandler { @@ -132,17 +129,13 @@ public SafeFuture> engineGetClientVersion(final ClientVersio @Override public SafeFuture> engineGetBlobs( final List blobVersionedHashes, final UInt64 slot) { - // TODO Make it versioned - final SchemaDefinitions schemaDefinitions = spec.atSlot(slot).getSchemaDefinitions(); - final BlobSchema blobSchema = - SchemaDefinitionsDeneb.required(schemaDefinitions).getBlobSchema(); - return executionEngineClient - .getBlobsV1(blobVersionedHashes) - .thenApply(ResponseUnwrapper::unwrapExecutionClientResponseOrThrow) - .thenApply( - blobAndProofV1s -> - blobAndProofV1s.stream() - .map(blobAndProofV1 -> blobAndProofV1.asInternalBlobsAndProofs(blobSchema)) - .toList()); + final JsonRpcRequestParams params = + new JsonRpcRequestParams.Builder().add(blobVersionedHashes).add(slot).build(); + return engineMethodsResolver + .getMethodList( + EngineApiMethod.ENGINE_GET_BLOBS, + () -> spec.atSlot(slot).getMilestone(), + BlobAndProof.class) + .execute(params); } } diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java index ab19a273df0..e7962629b1d 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java @@ -21,6 +21,7 @@ import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Supplier; @@ -108,6 +109,7 @@ private Map> denebSupportedMethods() { methods.put(ENGINE_NEW_PAYLOAD, new EngineNewPayloadV3(executionEngineClient)); methods.put(ENGINE_GET_PAYLOAD, new EngineGetPayloadV3(executionEngineClient, spec)); methods.put(ENGINE_FORK_CHOICE_UPDATED, new EngineForkChoiceUpdatedV3(executionEngineClient)); + methods.put(ENGINE_GET_BLOBS, new EngineGetBlobsV1(executionEngineClient, spec)); return methods; } @@ -141,6 +143,24 @@ public EngineJsonRpcMethod getMethod( return foundMethod; } + @Override + @SuppressWarnings({"unchecked", "unused"}) + public EngineJsonRpcMethod> getMethodList( + final EngineApiMethod method, + final Supplier milestoneSupplier, + final Class resultType) { + final SpecMilestone milestone = milestoneSupplier.get(); + final Map> milestoneMethods = + methodsByMilestone.getOrDefault(milestone, Collections.emptyMap()); + final EngineJsonRpcMethod> foundMethod = + (EngineJsonRpcMethod>) milestoneMethods.get(method); + if (foundMethod == null) { + throw new IllegalArgumentException( + "Can't find method with name " + method.getName() + " for milestone " + milestone); + } + return foundMethod; + } + @Override public Set getCapabilities() { return methodsByMilestone.values().stream() From d961b79f51063164d33aaa4da34bab1025ee5d76 Mon Sep 17 00:00:00 2001 From: Mehdi AOUADI Date: Mon, 30 Sep 2024 20:59:21 +0200 Subject: [PATCH 03/14] update capabilities unit test --- .../MilestoneBasedEngineJsonRpcMethodsResolverTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolverTest.java b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolverTest.java index a98ad90fbb8..4e2f9ba89a1 100644 --- a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolverTest.java +++ b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolverTest.java @@ -223,6 +223,7 @@ void getsCapabilities() { "engine_getPayloadV3", "engine_forkchoiceUpdatedV3", "engine_newPayloadV4", - "engine_getPayloadV4"); + "engine_getPayloadV4", + "engine_getBlobsV1"); } } From 5c8abfeca7c25e051aefc9a0c5d9d93299b9919f Mon Sep 17 00:00:00 2001 From: Mehdi AOUADI Date: Mon, 30 Sep 2024 21:01:17 +0200 Subject: [PATCH 04/14] rename resolver method --- .../ethereum/executionlayer/EngineJsonRpcMethodsResolver.java | 2 +- .../ethereum/executionlayer/ExecutionClientHandlerImpl.java | 2 +- .../MilestoneBasedEngineJsonRpcMethodsResolver.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineJsonRpcMethodsResolver.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineJsonRpcMethodsResolver.java index 97b16c2acac..1e34bc8c7a8 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineJsonRpcMethodsResolver.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineJsonRpcMethodsResolver.java @@ -25,7 +25,7 @@ public interface EngineJsonRpcMethodsResolver { EngineJsonRpcMethod getMethod( EngineApiMethod method, Supplier milestoneSupplier, Class resultType); - EngineJsonRpcMethod> getMethodList( + EngineJsonRpcMethod> getListMethod( EngineApiMethod method, Supplier milestoneSupplier, Class resultType); /** diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandlerImpl.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandlerImpl.java index 8b12ad2df03..c099f78574d 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandlerImpl.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionClientHandlerImpl.java @@ -132,7 +132,7 @@ public SafeFuture> engineGetBlobs( final JsonRpcRequestParams params = new JsonRpcRequestParams.Builder().add(blobVersionedHashes).add(slot).build(); return engineMethodsResolver - .getMethodList( + .getListMethod( EngineApiMethod.ENGINE_GET_BLOBS, () -> spec.atSlot(slot).getMilestone(), BlobAndProof.class) diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java index e7962629b1d..b1709d19886 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java @@ -145,7 +145,7 @@ public EngineJsonRpcMethod getMethod( @Override @SuppressWarnings({"unchecked", "unused"}) - public EngineJsonRpcMethod> getMethodList( + public EngineJsonRpcMethod> getListMethod( final EngineApiMethod method, final Supplier milestoneSupplier, final Class resultType) { From ccb1d655858f816ec2b75a6bfe366e288b98c681 Mon Sep 17 00:00:00 2001 From: Mehdi AOUADI Date: Tue, 1 Oct 2024 09:35:17 +0200 Subject: [PATCH 05/14] make blobs optional --- .../executionclient/methods/EngineGetBlobsV1.java | 11 ++++++++--- .../executionlayer/ExecutionLayerManagerImpl.java | 6 ++++-- .../spec/executionlayer/ExecutionLayerChannel.java | 7 ++++--- .../executionlayer/ExecutionLayerChannelStub.java | 14 ++------------ 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java index f6183289b58..f010ecd422e 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java @@ -15,6 +15,7 @@ import java.util.Collections; import java.util.List; +import java.util.Optional; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import tech.pegasys.teku.ethereum.executionclient.ExecutionEngineClient; @@ -28,7 +29,7 @@ import tech.pegasys.teku.spec.schemas.SchemaDefinitions; import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb; -public class EngineGetBlobsV1 extends AbstractEngineJsonRpcMethod> { +public class EngineGetBlobsV1 extends AbstractEngineJsonRpcMethod>> { private static final Logger LOG = LogManager.getLogger(); private final Spec spec; @@ -49,7 +50,7 @@ public int getVersion() { } @Override - public SafeFuture> execute(final JsonRpcRequestParams params) { + public SafeFuture>> execute(final JsonRpcRequestParams params) { final List blobVersionedHashes = params.getRequiredListParameter(0, VersionedHash.class); @@ -71,7 +72,11 @@ public SafeFuture> execute(final JsonRpcRequestParams params) final BlobSchema blobSchema = SchemaDefinitionsDeneb.required(schemaDefinitions).getBlobSchema(); return response.stream() - .map(blobAndProofV1 -> blobAndProofV1.asInternalBlobsAndProofs(blobSchema)) + .map( + blobAndProofV1 -> + blobAndProofV1 == null + ? Optional.empty() + : Optional.of(blobAndProofV1.asInternalBlobsAndProofs(blobSchema))) .toList(); }) .thenPeek( diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImpl.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImpl.java index acf21f010c8..2a666b0cdd7 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImpl.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImpl.java @@ -221,10 +221,12 @@ public SafeFuture> engineGetClientVersion(final ClientVersio } @Override - public SafeFuture> engineGetBlobs( + public SafeFuture>> engineGetBlobs( final List blobVersionedHashes, final UInt64 slot) { LOG.trace("calling engineGetBlobs(blobVersionedHashes={}, slot={})", blobVersionedHashes, slot); - return executionClientHandler.engineGetBlobs(blobVersionedHashes, slot); + return executionClientHandler + .engineGetBlobs(blobVersionedHashes, slot) + .thenApply(blobsAndProofs -> blobsAndProofs.stream().map(Optional::ofNullable).toList()); } @Override diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannel.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannel.java index c3723a961db..9c4aaaea35c 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannel.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannel.java @@ -79,9 +79,10 @@ public SafeFuture> engineGetClientVersion( } @Override - public SafeFuture> engineGetBlobs( + public SafeFuture>> engineGetBlobs( final List blobVersionedHashes, final UInt64 slot) { - return SafeFuture.completedFuture(List.of()); + return SafeFuture.completedFuture( + blobVersionedHashes.stream().map(e -> Optional.empty()).toList()); } @Override @@ -123,7 +124,7 @@ SafeFuture engineForkChoiceUpdated( SafeFuture> engineGetClientVersion(ClientVersion clientVersion); - SafeFuture> engineGetBlobs( + SafeFuture>> engineGetBlobs( List blobVersionedHashes, UInt64 slot); /** diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannelStub.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannelStub.java index 608092c3b7e..112c50efd1f 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannelStub.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/executionlayer/ExecutionLayerChannelStub.java @@ -27,7 +27,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; -import java.util.stream.IntStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.tuweni.bytes.Bytes; @@ -352,19 +351,10 @@ public SafeFuture> engineGetClientVersion(final ClientVersio } @Override - public SafeFuture> engineGetBlobs( + public SafeFuture>> engineGetBlobs( final List blobVersionedHashes, final UInt64 slot) { - final List blobs = - blobsUtil.generateBlobs( - slot, - blobsToGenerate.orElseGet( - () -> random.nextInt(spec.getMaxBlobsPerBlock().orElseThrow() + 1))); - final List commitments = blobsUtil.blobsToKzgCommitments(blobs); - final List proofs = blobsUtil.computeKzgProofs(blobs, commitments); return SafeFuture.completedFuture( - IntStream.range(0, blobs.size()) - .mapToObj(index -> new BlobAndProof(blobs.get(index), proofs.get(index))) - .toList()); + blobVersionedHashes.stream().map(e -> Optional.empty()).toList()); } @Override From 77e7a3287faf8d648b1bb794760dcf47c6127179 Mon Sep 17 00:00:00 2001 From: Mehdi AOUADI Date: Tue, 1 Oct 2024 14:39:59 +0200 Subject: [PATCH 06/14] add execution engine test --- .../web3j/Web3JExecutionEngineClientTest.java | 46 +++++++++++++++++++ .../schema/BlobAndProofV1.java | 8 ++-- .../web3j/Web3JExecutionEngineClient.java | 4 +- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/ethereum/executionclient/src/integration-test/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClientTest.java b/ethereum/executionclient/src/integration-test/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClientTest.java index 3426bb968f5..0ee7489190a 100644 --- a/ethereum/executionclient/src/integration-test/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClientTest.java +++ b/ethereum/executionclient/src/integration-test/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClientTest.java @@ -48,6 +48,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestTemplate; import tech.pegasys.teku.ethereum.events.ExecutionClientEventsChannel; +import tech.pegasys.teku.ethereum.executionclient.schema.BlobAndProofV1; import tech.pegasys.teku.ethereum.executionclient.schema.ClientVersionV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV3; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV4; @@ -66,6 +67,7 @@ import tech.pegasys.teku.spec.SpecMilestone; import tech.pegasys.teku.spec.TestSpecContext; import tech.pegasys.teku.spec.TestSpecInvocationContextProvider.SpecContext; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayload; import tech.pegasys.teku.spec.executionlayer.PayloadStatus; import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; @@ -354,6 +356,50 @@ public void newPayloadV4_shouldBuildRequestAndResponseSuccessfully() { .isEqualTo(parentBeaconBlockRoot.toHexString()); } + @TestTemplate + @SuppressWarnings("unchecked") + public void getBlobsV1_shouldBuildRequestAndResponseSuccessfully() { + assumeThat(specMilestone).isGreaterThanOrEqualTo(DENEB); + final List blobSidecars = + dataStructureUtil.randomBlobSidecars(spec.getMaxBlobsPerBlock().orElseThrow()); + final List blobsAndProofsV1 = + blobSidecars.stream() + .map( + blobSidecar -> + new BlobAndProofV1( + blobSidecar.getBlob().getBytes(), + blobSidecar.getKZGProof().getBytesCompressed())) + .toList(); + final String blobsAndProofsJson = + blobSidecars.stream() + .map( + blobSidecar -> + String.format( + "{ \"blob\": \"%s\", \"proof\": \"%s\" }", + blobSidecar.getBlob().getBytes().toHexString(), + blobSidecar.getKZGProof().getBytesCompressed().toHexString())) + .collect(Collectors.joining(", ")); + final String bodyResponse = + "{\"jsonrpc\": \"2.0\", \"id\": 0, \"result\": [" + blobsAndProofsJson + "]}"; + + mockSuccessfulResponse(bodyResponse); + + final List blobVersionedHashes = dataStructureUtil.randomVersionedHashes(3); + + final SafeFuture>> futureResponse = + eeClient.getBlobsV1(blobVersionedHashes); + + assertThat(futureResponse) + .succeedsWithin(1, TimeUnit.SECONDS) + .matches(response -> response.getPayload().equals(blobsAndProofsV1)); + + final Map requestData = takeRequest(); + verifyJsonRpcMethodCall(requestData, "engine_getBlobsVersionV1"); + assertThat(requestData.get("params")) + .asInstanceOf(LIST) + .containsExactly(blobVersionedHashes.stream().map(VersionedHash::toHexString).toList()); + } + private void mockSuccessfulResponse(final String responseBody) { mockWebServer.enqueue( new MockResponse() diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/schema/BlobAndProofV1.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/schema/BlobAndProofV1.java index fb6d978bf68..1a73026a865 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/schema/BlobAndProofV1.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/schema/BlobAndProofV1.java @@ -32,12 +32,12 @@ public class BlobAndProofV1 { - @JsonSerialize(contentUsing = BytesSerializer.class) - @JsonDeserialize(contentUsing = BytesDeserializer.class) + @JsonSerialize(using = BytesSerializer.class) + @JsonDeserialize(using = BytesDeserializer.class) private final Bytes blob; - @JsonSerialize(contentUsing = BytesSerializer.class) - @JsonDeserialize(contentUsing = Bytes48Deserializer.class) + @JsonSerialize(using = BytesSerializer.class) + @JsonDeserialize(using = Bytes48Deserializer.class) private final Bytes48 proof; public BlobAndProofV1( diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java index a7fae4b6f9e..9b6ff4a5921 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java @@ -258,10 +258,12 @@ public SafeFuture>> getClientVersionV1( @Override public SafeFuture>> getBlobsV1( final List blobVersionedHashes) { + final List expectedBlobVersionedHashes = + blobVersionedHashes.stream().map(VersionedHash::toHexString).toList(); final Request web3jRequest = new Request<>( "engine_getBlobsVersionV1", - Collections.singletonList(blobVersionedHashes), + list(expectedBlobVersionedHashes), web3JClient.getWeb3jService(), GetBlobsVersionV1Web3jResponse.class); return web3JClient.doRequest(web3jRequest, GET_BLOBS_TIMEOUT); From ca20c14636e553908ab1c32f91191005c74b58a2 Mon Sep 17 00:00:00 2001 From: Mehdi AOUADI Date: Tue, 1 Oct 2024 15:33:14 +0200 Subject: [PATCH 07/14] add execution client tests --- .../methods/EngineGetBlobsV1.java | 4 +- .../methods/EngineGetBlobsV1Test.java | 151 ++++++++++++++++++ .../DenebExecutionClientHandlerTest.java | 27 ++++ .../ElectraExecutionClientHandlerTest.java | 27 ++++ ...BasedEngineJsonRpcMethodsResolverTest.java | 8 +- 5 files changed, 212 insertions(+), 5 deletions(-) create mode 100644 ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java index f010ecd422e..b21f389d06d 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java @@ -13,7 +13,6 @@ package tech.pegasys.teku.ethereum.executionclient.methods; -import java.util.Collections; import java.util.List; import java.util.Optional; import org.apache.logging.log4j.LogManager; @@ -85,7 +84,6 @@ public SafeFuture>> execute(final JsonRpcRequestPara "Response {}(blobVersionedHashes={}) -> {}", getVersionedName(), blobVersionedHashes, - blobsAndProofs)) - .exceptionally(throwable -> Collections.emptyList()); + blobsAndProofs)); } } diff --git a/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java b/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java new file mode 100644 index 00000000000..48a9547344d --- /dev/null +++ b/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java @@ -0,0 +1,151 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.ethereum.executionclient.methods; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import tech.pegasys.teku.ethereum.executionclient.ExecutionEngineClient; +import tech.pegasys.teku.ethereum.executionclient.response.InvalidRemoteResponseException; +import tech.pegasys.teku.ethereum.executionclient.schema.BlobAndProofV1; +import tech.pegasys.teku.ethereum.executionclient.schema.Response; +import tech.pegasys.teku.infrastructure.async.SafeFuture; +import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.spec.Spec; +import tech.pegasys.teku.spec.TestSpecFactory; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; +import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; +import tech.pegasys.teku.spec.util.DataStructureUtil; + +public class EngineGetBlobsV1Test { + + private final Spec spec = TestSpecFactory.createMinimalElectra(); + private final DataStructureUtil dataStructureUtil = new DataStructureUtil(spec); + private final ExecutionEngineClient executionEngineClient = mock(ExecutionEngineClient.class); + private EngineGetBlobsV1 jsonRpcMethod; + + @BeforeEach + public void setUp() { + jsonRpcMethod = new EngineGetBlobsV1(executionEngineClient, spec); + } + + @Test + public void shouldReturnExpectedNameAndVersion() { + assertThat(jsonRpcMethod.getName()).isEqualTo("engine_getBlobs"); + assertThat(jsonRpcMethod.getVersion()).isEqualTo(1); + assertThat(jsonRpcMethod.getVersionedName()).isEqualTo("engine_getBlobsV1"); + } + + @Test + public void blobVersionedHashesParamIsRequired() { + final JsonRpcRequestParams params = new JsonRpcRequestParams.Builder().build(); + + assertThatThrownBy(() -> jsonRpcMethod.execute(params)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Missing required parameter at index 0"); + + verifyNoInteractions(executionEngineClient); + } + + @Test + public void slotParamIsRequired() { + final List versionedHashes = dataStructureUtil.randomVersionedHashes(4); + + final JsonRpcRequestParams params = + new JsonRpcRequestParams.Builder().add(versionedHashes).build(); + + assertThatThrownBy(() -> jsonRpcMethod.execute(params)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Missing required parameter at index 1"); + + verifyNoInteractions(executionEngineClient); + } + + @Test + public void shouldReturnFailedExecutionWhenEngineClientRequestFails() { + final List versionedHashes = dataStructureUtil.randomVersionedHashes(4); + final String errorResponseFromClient = "error!"; + + when(executionEngineClient.getBlobsV1(any())) + .thenReturn(dummyFailedResponse(errorResponseFromClient)); + + final JsonRpcRequestParams params = + new JsonRpcRequestParams.Builder().add(versionedHashes).add(UInt64.ZERO).build(); + + assertThat(jsonRpcMethod.execute(params)) + .failsWithin(1, TimeUnit.SECONDS) + .withThrowableOfType(ExecutionException.class) + .withRootCauseInstanceOf(InvalidRemoteResponseException.class) + .withMessageContaining( + "Invalid remote response from the execution client: %s", errorResponseFromClient); + } + + @Test + public void shouldCallGetBlobsV1AndParseResponseSuccessfully() { + final List versionedHashes = dataStructureUtil.randomVersionedHashes(4); + final List blobSidecars = + dataStructureUtil.randomBlobSidecars(spec.getMaxBlobsPerBlock().orElseThrow()); + + when(executionEngineClient.getBlobsV1(eq(versionedHashes))) + .thenReturn(dummySuccessfulResponse(blobSidecars)); + + final JsonRpcRequestParams params = + new JsonRpcRequestParams.Builder().add(versionedHashes).add(UInt64.ZERO).build(); + + jsonRpcMethod = new EngineGetBlobsV1(executionEngineClient, spec); + + final List> expectedResponse = + blobSidecars.stream() + .map( + blobSidecar -> + Optional.of(new BlobAndProof(blobSidecar.getBlob(), blobSidecar.getKZGProof()))) + .toList(); + assertThat(jsonRpcMethod.execute(params)).isCompletedWithValue(expectedResponse); + + verify(executionEngineClient).getBlobsV1(eq(versionedHashes)); + verifyNoMoreInteractions(executionEngineClient); + } + + private SafeFuture>> dummySuccessfulResponse( + final List blobSidecars) { + return SafeFuture.completedFuture( + new Response<>( + blobSidecars.stream() + .map( + blobSidecar -> + new BlobAndProofV1( + blobSidecar.getBlob().getBytes(), + blobSidecar.getKZGProof().getBytesCompressed())) + .toList())); + } + + private SafeFuture>> dummyFailedResponse( + final String errorMessage) { + return SafeFuture.completedFuture(Response.withErrorMessage(errorMessage)); + } +} diff --git a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/DenebExecutionClientHandlerTest.java b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/DenebExecutionClientHandlerTest.java index af50e6cecfe..4854b75fe09 100644 --- a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/DenebExecutionClientHandlerTest.java +++ b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/DenebExecutionClientHandlerTest.java @@ -24,6 +24,7 @@ import org.apache.tuweni.units.bigints.UInt256; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import tech.pegasys.teku.ethereum.executionclient.schema.BlobAndProofV1; import tech.pegasys.teku.ethereum.executionclient.schema.BlobsBundleV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV3; import tech.pegasys.teku.ethereum.executionclient.schema.ForkChoiceStateV1; @@ -35,6 +36,8 @@ import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.TestSpecFactory; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayload; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayloadContext; import tech.pegasys.teku.spec.datastructures.execution.GetPayloadResponse; @@ -134,6 +137,30 @@ void engineForkChoiceUpdated_shouldCallEngineForkChoiceUpdatedV3() { assertThat(future).isCompleted(); } + @Test + void engineGetBlobs_shouldCallGetBlobsV1() { + final ExecutionClientHandler handler = getHandler(); + final int maxBlobsPerBlock = spec.getMaxBlobsPerBlock().orElseThrow(); + final List versionedHashes = + dataStructureUtil.randomVersionedHashes(maxBlobsPerBlock); + final List blobSidecars = dataStructureUtil.randomBlobSidecars(maxBlobsPerBlock); + final UInt64 slot = dataStructureUtil.randomUInt64(1_000_000); + final SafeFuture>> dummyResponse = + SafeFuture.completedFuture( + new Response<>( + blobSidecars.stream() + .map( + blobSidecar -> + new BlobAndProofV1( + blobSidecar.getBlob().getBytes(), + blobSidecar.getKZGProof().getBytesCompressed())) + .toList())); + when(executionEngineClient.getBlobsV1(versionedHashes)).thenReturn(dummyResponse); + final SafeFuture> future = handler.engineGetBlobs(versionedHashes, slot); + verify(executionEngineClient).getBlobsV1(versionedHashes); + assertThat(future).isCompleted(); + } + private ExecutionPayloadContext randomContext() { return new ExecutionPayloadContext( dataStructureUtil.randomBytes8(), diff --git a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ElectraExecutionClientHandlerTest.java b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ElectraExecutionClientHandlerTest.java index 0f67f059f8a..6e21f68b88d 100644 --- a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ElectraExecutionClientHandlerTest.java +++ b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ElectraExecutionClientHandlerTest.java @@ -25,6 +25,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import tech.pegasys.teku.ethereum.executionclient.schema.BlobAndProofV1; import tech.pegasys.teku.ethereum.executionclient.schema.BlobsBundleV1; import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV4; import tech.pegasys.teku.ethereum.executionclient.schema.ForkChoiceStateV1; @@ -36,6 +37,8 @@ import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.TestSpecFactory; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayload; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayloadContext; import tech.pegasys.teku.spec.datastructures.execution.GetPayloadResponse; @@ -136,6 +139,30 @@ void engineForkChoiceUpdated_shouldCallEngineForkChoiceUpdatedV3() { assertThat(future).isCompleted(); } + @Test + void engineGetBlobs_shouldCallGetBlobsV1() { + final ExecutionClientHandler handler = getHandler(); + final int maxBlobsPerBlock = spec.getMaxBlobsPerBlock().orElseThrow(); + final List versionedHashes = + dataStructureUtil.randomVersionedHashes(maxBlobsPerBlock); + final List blobSidecars = dataStructureUtil.randomBlobSidecars(maxBlobsPerBlock); + final UInt64 slot = dataStructureUtil.randomUInt64(1_000_000); + final SafeFuture>> dummyResponse = + SafeFuture.completedFuture( + new Response<>( + blobSidecars.stream() + .map( + blobSidecar -> + new BlobAndProofV1( + blobSidecar.getBlob().getBytes(), + blobSidecar.getKZGProof().getBytesCompressed())) + .toList())); + when(executionEngineClient.getBlobsV1(versionedHashes)).thenReturn(dummyResponse); + final SafeFuture> future = handler.engineGetBlobs(versionedHashes, slot); + verify(executionEngineClient).getBlobsV1(versionedHashes); + assertThat(future).isCompleted(); + } + private ExecutionPayloadContext randomContext() { return new ExecutionPayloadContext( dataStructureUtil.randomBytes8(), diff --git a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolverTest.java b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolverTest.java index 4e2f9ba89a1..3bd17159088 100644 --- a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolverTest.java +++ b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolverTest.java @@ -18,6 +18,7 @@ import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.Mockito.mock; import static tech.pegasys.teku.ethereum.executionclient.methods.EngineApiMethod.ENGINE_FORK_CHOICE_UPDATED; +import static tech.pegasys.teku.ethereum.executionclient.methods.EngineApiMethod.ENGINE_GET_BLOBS; import static tech.pegasys.teku.ethereum.executionclient.methods.EngineApiMethod.ENGINE_GET_PAYLOAD; import static tech.pegasys.teku.ethereum.executionclient.methods.EngineApiMethod.ENGINE_NEW_PAYLOAD; @@ -33,6 +34,7 @@ import tech.pegasys.teku.ethereum.executionclient.methods.EngineForkChoiceUpdatedV1; import tech.pegasys.teku.ethereum.executionclient.methods.EngineForkChoiceUpdatedV2; import tech.pegasys.teku.ethereum.executionclient.methods.EngineForkChoiceUpdatedV3; +import tech.pegasys.teku.ethereum.executionclient.methods.EngineGetBlobsV1; import tech.pegasys.teku.ethereum.executionclient.methods.EngineGetPayloadV1; import tech.pegasys.teku.ethereum.executionclient.methods.EngineGetPayloadV2; import tech.pegasys.teku.ethereum.executionclient.methods.EngineGetPayloadV3; @@ -161,7 +163,8 @@ private static Stream denebMethods() { return Stream.of( arguments(ENGINE_NEW_PAYLOAD, EngineNewPayloadV3.class), arguments(ENGINE_GET_PAYLOAD, EngineGetPayloadV3.class), - arguments(ENGINE_FORK_CHOICE_UPDATED, EngineForkChoiceUpdatedV3.class)); + arguments(ENGINE_FORK_CHOICE_UPDATED, EngineForkChoiceUpdatedV3.class), + arguments(ENGINE_GET_BLOBS, EngineGetBlobsV1.class)); } @Test @@ -197,7 +200,8 @@ private static Stream electraMethods() { return Stream.of( arguments(ENGINE_NEW_PAYLOAD, EngineNewPayloadV4.class), arguments(ENGINE_GET_PAYLOAD, EngineGetPayloadV4.class), - arguments(ENGINE_FORK_CHOICE_UPDATED, EngineForkChoiceUpdatedV3.class)); + arguments(ENGINE_FORK_CHOICE_UPDATED, EngineForkChoiceUpdatedV3.class), + arguments(ENGINE_GET_BLOBS, EngineGetBlobsV1.class)); } @Test From 1bc04e76df0817ee6d3a926258d21a2c321c4fac Mon Sep 17 00:00:00 2001 From: Mehdi AOUADI Date: Tue, 1 Oct 2024 15:48:06 +0200 Subject: [PATCH 08/14] add execution layer manager tests --- .../ExecutionLayerManagerImplTest.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImplTest.java b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImplTest.java index 6104c88d512..e6fd00930a9 100644 --- a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImplTest.java +++ b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImplTest.java @@ -44,9 +44,11 @@ import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.TestSpecFactory; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.builder.BuilderBid; import tech.pegasys.teku.spec.datastructures.builder.SignedBuilderBid; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; import tech.pegasys.teku.spec.datastructures.execution.BlobsBundle; import tech.pegasys.teku.spec.datastructures.execution.BuilderBidOrFallbackData; import tech.pegasys.teku.spec.datastructures.execution.BuilderPayloadOrFallbackData; @@ -58,6 +60,7 @@ import tech.pegasys.teku.spec.datastructures.execution.FallbackReason; import tech.pegasys.teku.spec.datastructures.execution.GetPayloadResponse; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; +import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; import tech.pegasys.teku.spec.util.DataStructureUtil; class ExecutionLayerManagerImplTest { @@ -656,6 +659,18 @@ void onSlot_shouldCleanUpFallbackCache() { verifyNoMoreInteractions(executionClientHandler); } + @Test + public void engineGetBlobs_shouldReturnGetBlobsResponseViaEngine() { + setupDeneb(); + final List versionedHashes = + dataStructureUtil.randomVersionedHashes(spec.getMaxBlobsPerBlock().orElseThrow()); + final UInt64 slot = dataStructureUtil.randomSlot(); + final List getBlobsResponse = + prepareEngineGetBlobsResponse(versionedHashes, slot); + assertThat(executionLayerManager.engineGetBlobs(versionedHashes, slot)) + .isCompletedWithValue(getBlobsResponse.stream().map(Optional::ofNullable).toList()); + } + private void setupDeneb() { spec = TestSpecFactory.createMinimalDeneb(); dataStructureUtil = new DataStructureUtil(spec); @@ -786,6 +801,19 @@ private GetPayloadResponse prepareEngineGetPayloadResponseWithBlobs( return getPayloadResponse; } + private List prepareEngineGetBlobsResponse( + final List blobVersionedHashes, final UInt64 slot) { + final List blobSidecars = + dataStructureUtil.randomBlobSidecars(spec.getMaxBlobsPerBlock().orElseThrow()); + final List getBlobsResponse = + blobSidecars.stream() + .map(blobSidecar -> new BlobAndProof(blobSidecar.getBlob(), blobSidecar.getKZGProof())) + .toList(); + when(executionClientHandler.engineGetBlobs(blobVersionedHashes, slot)) + .thenReturn(SafeFuture.completedFuture(getBlobsResponse)); + return getBlobsResponse; + } + private ExecutionLayerManagerImpl createExecutionLayerChannelImpl( final boolean builderEnabled, final boolean builderValidatorEnabled) { return createExecutionLayerChannelImpl( From 898dba1d8fa1e13a0250b48fdfce1eeb467f0606 Mon Sep 17 00:00:00 2001 From: Mehdi AOUADI Date: Tue, 1 Oct 2024 17:06:44 +0200 Subject: [PATCH 09/14] fix method name --- .../executionclient/web3j/Web3JExecutionEngineClientTest.java | 2 +- .../executionclient/web3j/Web3JExecutionEngineClient.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ethereum/executionclient/src/integration-test/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClientTest.java b/ethereum/executionclient/src/integration-test/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClientTest.java index 0ee7489190a..6113d62e89f 100644 --- a/ethereum/executionclient/src/integration-test/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClientTest.java +++ b/ethereum/executionclient/src/integration-test/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClientTest.java @@ -394,7 +394,7 @@ public void getBlobsV1_shouldBuildRequestAndResponseSuccessfully() { .matches(response -> response.getPayload().equals(blobsAndProofsV1)); final Map requestData = takeRequest(); - verifyJsonRpcMethodCall(requestData, "engine_getBlobsVersionV1"); + verifyJsonRpcMethodCall(requestData, "engine_getBlobsV1"); assertThat(requestData.get("params")) .asInstanceOf(LIST) .containsExactly(blobVersionedHashes.stream().map(VersionedHash::toHexString).toList()); diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java index 9b6ff4a5921..0eb8c215fda 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClient.java @@ -262,7 +262,7 @@ public SafeFuture>> getBlobsV1( blobVersionedHashes.stream().map(VersionedHash::toHexString).toList(); final Request web3jRequest = new Request<>( - "engine_getBlobsVersionV1", + "engine_getBlobsV1", list(expectedBlobVersionedHashes), web3JClient.getWeb3jService(), GetBlobsVersionV1Web3jResponse.class); From e5529f4303d65b1919c4f66a624b54d1031ea80c Mon Sep 17 00:00:00 2001 From: Mehdi AOUADI Date: Tue, 1 Oct 2024 18:01:23 +0200 Subject: [PATCH 10/14] remove dupplicate optional --- .../executionclient/methods/EngineGetBlobsV1.java | 9 ++++----- .../executionclient/methods/EngineGetBlobsV1Test.java | 7 ++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java index b21f389d06d..798ab132c89 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java @@ -14,7 +14,6 @@ package tech.pegasys.teku.ethereum.executionclient.methods; import java.util.List; -import java.util.Optional; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import tech.pegasys.teku.ethereum.executionclient.ExecutionEngineClient; @@ -28,7 +27,7 @@ import tech.pegasys.teku.spec.schemas.SchemaDefinitions; import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb; -public class EngineGetBlobsV1 extends AbstractEngineJsonRpcMethod>> { +public class EngineGetBlobsV1 extends AbstractEngineJsonRpcMethod> { private static final Logger LOG = LogManager.getLogger(); private final Spec spec; @@ -49,7 +48,7 @@ public int getVersion() { } @Override - public SafeFuture>> execute(final JsonRpcRequestParams params) { + public SafeFuture> execute(final JsonRpcRequestParams params) { final List blobVersionedHashes = params.getRequiredListParameter(0, VersionedHash.class); @@ -74,8 +73,8 @@ public SafeFuture>> execute(final JsonRpcRequestPara .map( blobAndProofV1 -> blobAndProofV1 == null - ? Optional.empty() - : Optional.of(blobAndProofV1.asInternalBlobsAndProofs(blobSchema))) + ? null + : blobAndProofV1.asInternalBlobsAndProofs(blobSchema)) .toList(); }) .thenPeek( diff --git a/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java b/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java index 48a9547344d..96267cb13f6 100644 --- a/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java +++ b/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java @@ -24,7 +24,6 @@ import static org.mockito.Mockito.when; import java.util.List; -import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.BeforeEach; @@ -119,11 +118,9 @@ public void shouldCallGetBlobsV1AndParseResponseSuccessfully() { jsonRpcMethod = new EngineGetBlobsV1(executionEngineClient, spec); - final List> expectedResponse = + final List expectedResponse = blobSidecars.stream() - .map( - blobSidecar -> - Optional.of(new BlobAndProof(blobSidecar.getBlob(), blobSidecar.getKZGProof()))) + .map(blobSidecar -> new BlobAndProof(blobSidecar.getBlob(), blobSidecar.getKZGProof())) .toList(); assertThat(jsonRpcMethod.execute(params)).isCompletedWithValue(expectedResponse); From caade1d4cd43a061a095411664116159b1e7e7af Mon Sep 17 00:00:00 2001 From: Mehdi AOUADI Date: Wed, 2 Oct 2024 16:35:07 +0200 Subject: [PATCH 11/14] add engine RPC method optionality notion --- .../methods/EngineGetBlobsV1.java | 8 +++++- .../methods/EngineJsonRpcMethod.java | 6 +++++ .../DefaultExecutionWeb3jClientProvider.java | 3 ++- .../methods/EngineGetBlobsV1Test.java | 6 ++--- .../EngineCapabilitiesMonitor.java | 27 ++++++++++++++++--- .../EngineJsonRpcMethodsResolver.java | 6 +++++ ...toneBasedEngineJsonRpcMethodsResolver.java | 11 ++++++++ .../EngineCapabilitiesMonitorTest.java | 23 +++++++++++++--- .../infrastructure/logging/EventLogger.java | 8 +++--- 9 files changed, 84 insertions(+), 14 deletions(-) diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java index 798ab132c89..4e4d43a2d0a 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java @@ -83,6 +83,12 @@ public SafeFuture> execute(final JsonRpcRequestParams params) "Response {}(blobVersionedHashes={}) -> {}", getVersionedName(), blobVersionedHashes, - blobsAndProofs)); + blobsAndProofs)) + .exceptionally( + throwable -> { + throw new UnsupportedOperationException( + String.format( + "Call to engineGetBlobsV1 failed. %s", throwable.getCause().getMessage())); + }); } } diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineJsonRpcMethod.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineJsonRpcMethod.java index ddc4b4f196e..8c41f711697 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineJsonRpcMethod.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineJsonRpcMethod.java @@ -28,6 +28,12 @@ default boolean isDeprecated() { return false; } + // TODO should be remove once all ELs implement engine_getBlobsV1. It has been added only to + // better handle the use case when the method is missing in the EL side + default boolean isOptional() { + return false; + } + default String getVersionedName() { return getVersion() == 0 ? getName() : getName() + "V" + getVersion(); } diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/DefaultExecutionWeb3jClientProvider.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/DefaultExecutionWeb3jClientProvider.java index b414b3756a9..dd3465735b6 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/DefaultExecutionWeb3jClientProvider.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/web3j/DefaultExecutionWeb3jClientProvider.java @@ -58,7 +58,8 @@ private synchronized void buildClient() { .jwtConfigOpt(jwtConfig) .timeProvider(timeProvider) .executionClientEventsPublisher(executionClientEventsPublisher) - .nonCriticalMethods("engine_exchangeCapabilities", "engine_getClientVersionV1") + .nonCriticalMethods( + "engine_exchangeCapabilities", "engine_getClientVersionV1", "engine_getBlobsV1") .build(); this.alreadyBuilt = true; } diff --git a/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java b/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java index 96267cb13f6..8a7ddb84de4 100644 --- a/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java +++ b/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java @@ -29,7 +29,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import tech.pegasys.teku.ethereum.executionclient.ExecutionEngineClient; -import tech.pegasys.teku.ethereum.executionclient.response.InvalidRemoteResponseException; import tech.pegasys.teku.ethereum.executionclient.schema.BlobAndProofV1; import tech.pegasys.teku.ethereum.executionclient.schema.Response; import tech.pegasys.teku.infrastructure.async.SafeFuture; @@ -99,9 +98,10 @@ public void shouldReturnFailedExecutionWhenEngineClientRequestFails() { assertThat(jsonRpcMethod.execute(params)) .failsWithin(1, TimeUnit.SECONDS) .withThrowableOfType(ExecutionException.class) - .withRootCauseInstanceOf(InvalidRemoteResponseException.class) + .withRootCauseInstanceOf(UnsupportedOperationException.class) .withMessageContaining( - "Invalid remote response from the execution client: %s", errorResponseFromClient); + "Call to engineGetBlobsV1 failed. Invalid remote response from the execution client: %s", + errorResponseFromClient); } @Test diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineCapabilitiesMonitor.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineCapabilitiesMonitor.java index 702835fe05a..8ecc3afbcce 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineCapabilitiesMonitor.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineCapabilitiesMonitor.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; +import java.util.stream.Stream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import tech.pegasys.teku.ethereum.events.SlotEventsChannel; @@ -40,6 +41,7 @@ public class EngineCapabilitiesMonitor implements SlotEventsChannel { private final Spec spec; private final EventLogger eventLogger; private final Supplier> capabilitiesSupplier; + private final Supplier> optionalCapabilitiesSupplier; private final ExecutionEngineClient executionEngineClient; public EngineCapabilitiesMonitor( @@ -51,6 +53,8 @@ public EngineCapabilitiesMonitor( this.eventLogger = eventLogger; this.capabilitiesSupplier = Suppliers.memoize(() -> new ArrayList<>(engineMethodsResolver.getCapabilities())); + this.optionalCapabilitiesSupplier = + Suppliers.memoize(() -> new ArrayList<>(engineMethodsResolver.getOptionalCapabilities())); this.executionEngineClient = executionEngineClient; } @@ -79,18 +83,35 @@ private boolean slotIsApplicable(final UInt64 slot) { private SafeFuture monitor() { final List capabilities = capabilitiesSupplier.get(); + final List optionalCapabilities = optionalCapabilitiesSupplier.get(); return executionEngineClient - .exchangeCapabilities(capabilities) + .exchangeCapabilities( + Stream.concat(capabilities.stream(), optionalCapabilities.stream()).toList()) .thenApply(ResponseUnwrapper::unwrapExecutionClientResponseOrThrow) .thenAccept( engineCapabilities -> { LOG.debug("Engine API capabilities response: " + engineCapabilities); + final List missingEngineCapabilities = capabilities.stream() - .filter(capability -> !engineCapabilities.contains(capability)) + .filter( + capability -> + !engineCapabilities.contains(capability) + && !optionalCapabilities.contains(capability)) + .toList(); + + final List missingOptionalCapabilities = + optionalCapabilities.stream() + .filter( + optionalCapability -> !engineCapabilities.contains(optionalCapability)) .toList(); + if (!missingEngineCapabilities.isEmpty()) { - eventLogger.missingEngineApiCapabilities(missingEngineCapabilities); + eventLogger.missingEngineApiCapabilities(missingEngineCapabilities, false); + } + + if (!missingOptionalCapabilities.isEmpty()) { + eventLogger.missingEngineApiCapabilities(missingOptionalCapabilities, true); } }); } diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineJsonRpcMethodsResolver.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineJsonRpcMethodsResolver.java index 1e34bc8c7a8..ce182addfea 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineJsonRpcMethodsResolver.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/EngineJsonRpcMethodsResolver.java @@ -34,4 +34,10 @@ EngineJsonRpcMethod> getListMethod( * request */ Set getCapabilities(); + + /** + * TODO this optionality notion should be removed once all ELs implement the engine_getBlobsV1 RPC + * method. It has been added to ensure a softer and better logging when the method is missing only + */ + Set getOptionalCapabilities(); } diff --git a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java index b1709d19886..0651980f143 100644 --- a/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java +++ b/ethereum/executionlayer/src/main/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolver.java @@ -165,6 +165,17 @@ public EngineJsonRpcMethod> getListMethod( public Set getCapabilities() { return methodsByMilestone.values().stream() .flatMap(methods -> methods.values().stream()) + .filter(method -> !method.isOptional()) + .filter(method -> !method.isDeprecated()) + .map(EngineJsonRpcMethod::getVersionedName) + .collect(Collectors.toSet()); + } + + @Override + public Set getOptionalCapabilities() { + return methodsByMilestone.values().stream() + .flatMap(methods -> methods.values().stream()) + .filter(EngineJsonRpcMethod::isOptional) .filter(method -> !method.isDeprecated()) .map(EngineJsonRpcMethod::getVersionedName) .collect(Collectors.toSet()); diff --git a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/EngineCapabilitiesMonitorTest.java b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/EngineCapabilitiesMonitorTest.java index ff4d6c5343c..9849430c6f1 100644 --- a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/EngineCapabilitiesMonitorTest.java +++ b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/EngineCapabilitiesMonitorTest.java @@ -25,6 +25,7 @@ import java.util.HashSet; import java.util.List; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import tech.pegasys.teku.ethereum.executionclient.ExecutionEngineClient; @@ -43,14 +44,18 @@ public class EngineCapabilitiesMonitorTest { mock(EngineJsonRpcMethodsResolver.class); private final ExecutionEngineClient executionEngineClient = mock(ExecutionEngineClient.class); + private final List engineCapabilities = List.of("method1", "method2", "method3"); private final List capabilities = List.of("method1", "method2"); + private final List optionalCapabilities = List.of("method3"); private EngineCapabilitiesMonitor engineCapabilitiesMonitor; @BeforeEach public void setUp() { when(engineMethodsResolver.getCapabilities()).thenReturn(new HashSet<>(capabilities)); - mockEngineCapabilitiesResponse(capabilities); + when(engineMethodsResolver.getOptionalCapabilities()) + .thenReturn(new HashSet<>(optionalCapabilities)); + mockEngineCapabilitiesResponse(engineCapabilities); engineCapabilitiesMonitor = new EngineCapabilitiesMonitor( spec, eventLogger, engineMethodsResolver, executionEngineClient); @@ -64,7 +69,18 @@ public void logsWarningIfEngineDoesNotSupportCapabilities() { // 3rd slot in epoch engineCapabilitiesMonitor.onSlot(UInt64.valueOf(2)); - verify(eventLogger).missingEngineApiCapabilities(List.of("method2")); + verify(eventLogger).missingEngineApiCapabilities(List.of("method2"), false); + } + + @Test + public void logsWarningIfEngineDoesNotSupportOptionalCapabilities() { + // engine only supports one of the methods + mockEngineCapabilitiesResponse(List.of("method1", "method2")); + + // 3rd slot in epoch + engineCapabilitiesMonitor.onSlot(UInt64.valueOf(2)); + + verify(eventLogger).missingEngineApiCapabilities(List.of("method3"), true); } @Test @@ -129,7 +145,8 @@ public void doesNotRunMonitoringIfNotAtRequiredSlot() { } private void mockEngineCapabilitiesResponse(final List engineCapabilities) { - when(executionEngineClient.exchangeCapabilities(capabilities)) + when(executionEngineClient.exchangeCapabilities( + Stream.concat(capabilities.stream(), optionalCapabilities.stream()).toList())) .thenReturn(SafeFuture.completedFuture(new Response<>(engineCapabilities))); } diff --git a/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/EventLogger.java b/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/EventLogger.java index 4893ec3e908..651eff10f2a 100644 --- a/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/EventLogger.java +++ b/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/EventLogger.java @@ -162,11 +162,13 @@ public void executionClientRecovered() { info("Execution Client is responding to requests again after a previous failure", Color.GREEN); } - public void missingEngineApiCapabilities(final List missingCapabilities) { + // TODO remove the isOptional param when all ELs implement the engine_getBlob + public void missingEngineApiCapabilities( + final List missingCapabilities, final boolean isOptional) { warn( String.format( - "Execution Client does not support required Engine API methods: %s. Make sure it is upgraded to a compatible version.", - missingCapabilities), + "Execution Client does not support %s Engine API methods: %s. Make sure it is upgraded to a compatible version.", + isOptional ? "optional" : "required", missingCapabilities), Color.YELLOW); } From 58bbaa81df7b61b1fc889cda714a97bd7c7f6157 Mon Sep 17 00:00:00 2001 From: Mehdi AOUADI Date: Wed, 2 Oct 2024 16:53:48 +0200 Subject: [PATCH 12/14] make engineGetBlobs optional --- .../ethereum/executionclient/methods/EngineGetBlobsV1.java | 5 +++++ .../executionclient/methods/EngineGetBlobsV1Test.java | 1 + 2 files changed, 6 insertions(+) diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java index 4e4d43a2d0a..fc87c96cc85 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java @@ -47,6 +47,11 @@ public int getVersion() { return 1; } + @Override + public boolean isOptional() { + return true; + } + @Override public SafeFuture> execute(final JsonRpcRequestParams params) { diff --git a/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java b/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java index 8a7ddb84de4..9a26233251b 100644 --- a/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java +++ b/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java @@ -55,6 +55,7 @@ public void setUp() { @Test public void shouldReturnExpectedNameAndVersion() { assertThat(jsonRpcMethod.getName()).isEqualTo("engine_getBlobs"); + assertThat(jsonRpcMethod.isOptional()).isTrue(); assertThat(jsonRpcMethod.getVersion()).isEqualTo(1); assertThat(jsonRpcMethod.getVersionedName()).isEqualTo("engine_getBlobsV1"); } From 868eebe7ddd8e659eb6139c2cacf91c2bcdf4a80 Mon Sep 17 00:00:00 2001 From: Mehdi AOUADI Date: Wed, 2 Oct 2024 17:39:45 +0200 Subject: [PATCH 13/14] throw web3client error directly --- .../executionclient/methods/EngineGetBlobsV1.java | 8 +------- .../executionclient/methods/EngineGetBlobsV1Test.java | 6 +++--- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java index fc87c96cc85..40426616b87 100644 --- a/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java +++ b/ethereum/executionclient/src/main/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1.java @@ -88,12 +88,6 @@ public SafeFuture> execute(final JsonRpcRequestParams params) "Response {}(blobVersionedHashes={}) -> {}", getVersionedName(), blobVersionedHashes, - blobsAndProofs)) - .exceptionally( - throwable -> { - throw new UnsupportedOperationException( - String.format( - "Call to engineGetBlobsV1 failed. %s", throwable.getCause().getMessage())); - }); + blobsAndProofs)); } } diff --git a/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java b/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java index 9a26233251b..85243691452 100644 --- a/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java +++ b/ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java @@ -29,6 +29,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import tech.pegasys.teku.ethereum.executionclient.ExecutionEngineClient; +import tech.pegasys.teku.ethereum.executionclient.response.InvalidRemoteResponseException; import tech.pegasys.teku.ethereum.executionclient.schema.BlobAndProofV1; import tech.pegasys.teku.ethereum.executionclient.schema.Response; import tech.pegasys.teku.infrastructure.async.SafeFuture; @@ -99,10 +100,9 @@ public void shouldReturnFailedExecutionWhenEngineClientRequestFails() { assertThat(jsonRpcMethod.execute(params)) .failsWithin(1, TimeUnit.SECONDS) .withThrowableOfType(ExecutionException.class) - .withRootCauseInstanceOf(UnsupportedOperationException.class) + .withRootCauseInstanceOf(InvalidRemoteResponseException.class) .withMessageContaining( - "Call to engineGetBlobsV1 failed. Invalid remote response from the execution client: %s", - errorResponseFromClient); + "Invalid remote response from the execution client: %s", errorResponseFromClient); } @Test From d878f31d0b958c0e53400f05d496ad29381d3d72 Mon Sep 17 00:00:00 2001 From: Mehdi AOUADI Date: Wed, 2 Oct 2024 17:41:56 +0200 Subject: [PATCH 14/14] add optional capabilities test --- ...neBasedEngineJsonRpcMethodsResolverTest.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolverTest.java b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolverTest.java index 3bd17159088..79d6577e40a 100644 --- a/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolverTest.java +++ b/ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/MilestoneBasedEngineJsonRpcMethodsResolverTest.java @@ -227,7 +227,20 @@ void getsCapabilities() { "engine_getPayloadV3", "engine_forkchoiceUpdatedV3", "engine_newPayloadV4", - "engine_getPayloadV4", - "engine_getBlobsV1"); + "engine_getPayloadV4"); + } + + @Test + void getsOptionalCapabilities() { + final Spec spec = + TestSpecFactory.createMinimalWithCapellaDenebAndElectraForkEpoch( + UInt64.ONE, UInt64.valueOf(2), UInt64.valueOf(3)); + + final MilestoneBasedEngineJsonRpcMethodsResolver engineMethodsResolver = + new MilestoneBasedEngineJsonRpcMethodsResolver(spec, executionEngineClient); + + final Set capabilities = engineMethodsResolver.getOptionalCapabilities(); + + assertThat(capabilities).containsExactlyInAnyOrder("engine_getBlobsV1"); } }