From daf7f899c21b045dec8af0bd7e36ce78e3efa6f0 Mon Sep 17 00:00:00 2001 From: Pedro Miranda <32689555+pedromiguelmiranda@users.noreply.github.com> Date: Wed, 25 Sep 2024 13:33:58 +0100 Subject: [PATCH] Attestation API updates for Electra (#6557) * new V2 endpoint for beacon getBlockAttestations * nnew GET endpoint version (V2) for getPoolAttestations * new POST endpoint version (V2) for submitPoolAttestations * remove premature ncli tests * review improvements * review comments and increased test coverage * small improvements * documentation typos --------- Co-authored-by: Pedro Miranda --- .../attestation_pool.nim | 42 +++++ beacon_chain/rpc/rest_beacon_api.nim | 121 +++++++++++++- .../eth2_apis/eth2_rest_serialization.nim | 47 ++++-- .../spec/eth2_apis/rest_beacon_calls.nim | 21 +++ beacon_chain/validators/message_router.nim | 6 +- ncli/resttest-rules.json | 156 ++++++++++++++++++ 6 files changed, 374 insertions(+), 19 deletions(-) diff --git a/beacon_chain/consensus_object_pools/attestation_pool.nim b/beacon_chain/consensus_object_pools/attestation_pool.nim index 18d648249a..be4b3eac95 100644 --- a/beacon_chain/consensus_object_pools/attestation_pool.nim +++ b/beacon_chain/consensus_object_pools/attestation_pool.nim @@ -570,6 +570,48 @@ iterator attestations*( for v in entry.aggregates: yield entry.toAttestation(v) +iterator electraAttestations*( + pool: AttestationPool, slot: Opt[Slot], + committee_index: Opt[CommitteeIndex]): electra.Attestation = + let candidateIndices = + if slot.isSome(): + let candidateIdx = pool.candidateIdx(slot.get(), true) + if candidateIdx.isSome(): + candidateIdx.get() .. candidateIdx.get() + else: + 1 .. 0 + else: + 0 ..< pool.electraCandidates.len() + + for candidateIndex in candidateIndices: + for _, entry in pool.electraCandidates[candidateIndex]: + ## data.index field from phase0 is still being used while we have + ## 2 attestation pools (pre and post electra). Refer to template addAttToPool + ## at addAttestation proc. + if committee_index.isNone() or entry.data.index == committee_index.get(): + var committee_bits: AttestationCommitteeBits + committee_bits[int(entry.data.index)] = true + + var singleAttestation = electra.Attestation( + aggregation_bits: ElectraCommitteeValidatorsBits.init(entry.committee_len), + committee_bits: committee_bits, + data: AttestationData( + slot: entry.data.slot, + index: 0, + beacon_block_root: entry.data.beacon_block_root, + source: entry.data.source, + target: entry.data.target) + ) + + for index, signature in entry.singles: + singleAttestation.aggregation_bits.setBit(index) + singleAttestation.signature = signature.toValidatorSig() + yield singleAttestation + singleAttestation.aggregation_bits.clearBit(index) + + for v in entry.aggregates: + yield entry.toElectraAttestation(v) + type AttestationCacheKey = (Slot, uint64) AttestationCache[CVBType] = Table[AttestationCacheKey, CVBType] ##\ diff --git a/beacon_chain/rpc/rest_beacon_api.nim b/beacon_chain/rpc/rest_beacon_api.nim index a9f03eafcb..66c22bcef5 100644 --- a/beacon_chain/rpc/rest_beacon_api.nim +++ b/beacon_chain/rpc/rest_beacon_api.nim @@ -1297,6 +1297,26 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) = node.dag.isFinalized(bid) ) + # https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getBlockAttestationsV2 + router.api2(MethodGet, + "/eth/v2/beacon/blocks/{block_id}/attestations") do ( + block_id: BlockIdent) -> RestApiResponse: + let + blockIdent = block_id.valueOr: + return RestApiResponse.jsonError(Http400, InvalidBlockIdValueError, + $error) + bdata = node.getForkedBlock(blockIdent).valueOr: + return RestApiResponse.jsonError(Http404, BlockNotFoundError) + + withBlck(bdata): + let bid = BlockId(root: forkyBlck.root, slot: forkyBlck.message.slot) + RestApiResponse.jsonResponseFinalizedWVersion( + forkyBlck.message.body.attestations.asSeq(), + node.getBlockOptimistic(bdata), + node.dag.isFinalized(bid), + consensusFork + ) + # https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolAttestations router.api2(MethodGet, "/eth/v1/beacon/pool/attestations") do ( slot: Option[Slot], @@ -1325,6 +1345,45 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) = res.add(item) RestApiResponse.jsonResponse(res) + # https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getPoolAttestationsV2 + router.api2(MethodGet, "/eth/v2/beacon/pool/attestations") do ( + slot: Option[Slot], + committee_index: Option[CommitteeIndex]) -> RestApiResponse: + let vindex = + if committee_index.isSome(): + let rindex = committee_index.get() + if rindex.isErr(): + return RestApiResponse.jsonError(Http400, + InvalidCommitteeIndexValueError, + $rindex.error) + Opt.some(rindex.get()) + else: + Opt.none(CommitteeIndex) + let vslot = + if slot.isSome(): + let rslot = slot.get() + if rslot.isErr(): + return RestApiResponse.jsonError(Http400, InvalidSlotValueError, + $rslot.error) + Opt.some(rslot.get()) + else: + Opt.none(Slot) + + let consensusFork = + if vslot.isNone(): + node.dag.cfg.consensusForkAtEpoch(node.currentSlot().epoch()) + else: + node.dag.cfg.consensusForkAtEpoch(vslot.get().epoch) + + if consensusFork < ConsensusFork.Electra: + return RestApiResponse.jsonResponseWVersion( + toSeq(node.attestationPool[].attestations(vslot, vindex)), + consensusFork) + else: + return RestApiResponse.jsonResponseWVersion( + toSeq(node.attestationPool[].electraAttestations(vslot, vindex)), + consensusFork) + # https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolAttestations router.api2(MethodPost, "/eth/v1/beacon/pool/attestations") do ( contentBody: Option[ContentBody]) -> RestApiResponse: @@ -1342,11 +1401,7 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) = # Since our validation logic supports batch processing, we will submit all # attestations for validation. let pending = - block: - var res: seq[Future[SendResult]] - for attestation in attestations: - res.add(node.router.routeAttestation(attestation)) - res + mapIt(attestations, node.router.routeAttestation(it)) let failures = block: var res: seq[RestIndexedErrorMessageItem] @@ -1372,6 +1427,62 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) = else: RestApiResponse.jsonMsgResponse(AttestationValidationSuccess) + # https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/submitPoolAttestationsV2 + router.api2(MethodPost, "/eth/v2/beacon/pool/attestations") do ( + contentBody: Option[ContentBody]) -> RestApiResponse: + + let + headerVersion = request.headers.getString("Eth-Consensus-Version") + consensusVersion = ConsensusFork.init(headerVersion) + if consensusVersion.isNone(): + return RestApiResponse.jsonError(Http400, FailedToObtainConsensusForkError) + + if contentBody.isNone(): + return RestApiResponse.jsonError(Http400, EmptyRequestBodyError) + + var pendingAttestations: seq[Future[SendResult]] + template decodeAttestations(AttestationType: untyped) = + let dres = decodeBody(seq[AttestationType], contentBody.get()) + if dres.isErr(): + return RestApiResponse.jsonError(Http400, + InvalidAttestationObjectError, + $dres.error) + # Since our validation logic supports batch processing, we will submit all + # attestations for validation. + for attestation in dres.get(): + pendingAttestations.add(node.router.routeAttestation(attestation)) + + case consensusVersion.get(): + of ConsensusFork.Phase0 .. ConsensusFork.Deneb: + decodeAttestations(phase0.Attestation) + of ConsensusFork.Electra: + decodeAttestations(electra.Attestation) + + let failures = + block: + var res: seq[RestIndexedErrorMessageItem] + await allFutures(pendingAttestations) + for index, future in pendingAttestations: + if future.completed(): + let fres = future.value() + if fres.isErr(): + let failure = RestIndexedErrorMessageItem(index: index, + message: $fres.error) + res.add(failure) + elif future.failed() or future.cancelled(): + # This is unexpected failure, so we log the error message. + let exc = future.error() + let failure = RestIndexedErrorMessageItem(index: index, + message: $exc.msg) + res.add(failure) + res + + if len(failures) > 0: + RestApiResponse.jsonErrorList(Http400, AttestationValidationError, + failures) + else: + RestApiResponse.jsonMsgResponse(AttestationValidationSuccess) + # https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolAttesterSlashings router.api2(MethodGet, "/eth/v1/beacon/pool/attester_slashings") do ( ) -> RestApiResponse: diff --git a/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim b/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim index 20f38fe729..b93fdf3aeb 100644 --- a/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim +++ b/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim @@ -699,6 +699,31 @@ proc jsonResponseFinalized*(t: typedesc[RestApiResponse], data: auto, let res = RestApiResponse.prepareJsonResponseFinalized(data, exec, finalized) RestApiResponse.response(res, Http200, "application/json") +proc jsonResponseFinalizedWVersion*(t: typedesc[RestApiResponse], + data: auto, + exec: Opt[bool], + finalized: bool, + version: ConsensusFork): RestApiResponse = + let + headers = [("eth-consensus-version", version.toString())] + res = + block: + var default: seq[byte] + try: + var stream = memoryOutput() + var writer = JsonWriter[RestJson].init(stream) + writer.beginRecord() + writer.writeField("version", version.toString()) + if exec.isSome(): + writer.writeField("execution_optimistic", exec.get()) + writer.writeField("finalized", finalized) + writer.writeField("data", data) + writer.endRecord() + stream.getOutput(seq[byte]) + except IOError: + default + RestApiResponse.response(res, Http200, "application/json", headers = headers) + proc jsonResponseWVersion*(t: typedesc[RestApiResponse], data: auto, version: ConsensusFork): RestApiResponse = let @@ -787,18 +812,16 @@ proc jsonResponseWMeta*(t: typedesc[RestApiResponse], proc jsonMsgResponse*(t: typedesc[RestApiResponse], msg: string = ""): RestApiResponse = let data = - block: - var default: seq[byte] - try: - var stream = memoryOutput() - var writer = JsonWriter[RestJson].init(stream) - writer.beginRecord() - writer.writeField("code", 200) - writer.writeField("message", msg) - writer.endRecord() - stream.getOutput(seq[byte]) - except IOError: - default + try: + var stream = memoryOutput() + var writer = JsonWriter[RestJson].init(stream) + writer.beginRecord() + writer.writeField("code", 200) + writer.writeField("message", msg) + writer.endRecord() + stream.getOutput(seq[byte]) + except IOError: + default(seq[byte]) RestApiResponse.response(data, Http200, "application/json") proc jsonError*(t: typedesc[RestApiResponse], status: HttpCode = Http200, diff --git a/beacon_chain/spec/eth2_apis/rest_beacon_calls.nim b/beacon_chain/spec/eth2_apis/rest_beacon_calls.nim index 370b3dff52..4620ccaba2 100644 --- a/beacon_chain/spec/eth2_apis/rest_beacon_calls.nim +++ b/beacon_chain/spec/eth2_apis/rest_beacon_calls.nim @@ -313,6 +313,12 @@ proc getBlockAttestations*(block_id: BlockIdent meth: MethodGet.} ## https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockAttestations +proc getBlockAttestationsV2Plain*(block_id: BlockIdent + ): RestPlainResponse {. + rest, endpoint: "/eth/v2/beacon/blocks/{block_id}/attestations", + meth: MethodGet.} + ## https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getBlockAttestationsV2 + proc getPoolAttestations*( slot: Option[Slot], committee_index: Option[CommitteeIndex] @@ -321,12 +327,27 @@ proc getPoolAttestations*( meth: MethodGet.} ## https://ethereum.github.io/beacon-APIs/#/Beacon/getPoolAttestations +proc getPoolAttestationsV2Plain*( + slot: Option[Slot], + committee_index: Option[CommitteeIndex] + ): RestPlainResponse {. + rest, endpoint: "/eth/v2/beacon/pool/attestations", + meth: MethodGet.} + ## https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/getPoolAttestationsV2 + proc submitPoolAttestations*(body: seq[phase0.Attestation]): RestPlainResponse {. rest, endpoint: "/eth/v1/beacon/pool/attestations", meth: MethodPost.} ## https://ethereum.github.io/beacon-APIs/#/Beacon/submitPoolAttestations +proc submitPoolAttestationsV2*( + body: seq[phase0.Attestation] | seq[electra.Attestation]): + RestPlainResponse {. + rest, endpoint: "/eth/v2/beacon/pool/attestations", + meth: MethodPost.} + ## https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Beacon/submitPoolAttestationsV2 + proc getPoolAttesterSlashings*(): RestResponse[GetPoolAttesterSlashingsResponse] {. rest, endpoint: "/eth/v1/beacon/pool/attester_slashings", meth: MethodGet.} diff --git a/beacon_chain/validators/message_router.nim b/beacon_chain/validators/message_router.nim index a96d865e13..7a65774fa1 100644 --- a/beacon_chain/validators/message_router.nim +++ b/beacon_chain/validators/message_router.nim @@ -235,7 +235,9 @@ proc routeAttestation*( return ok() proc routeAttestation*( - router: ref MessageRouter, attestation: phase0.Attestation | electra.Attestation): + router: ref MessageRouter, + attestation: phase0.Attestation | electra.Attestation, + on_chain: static bool = false): Future[SendResult] {.async: (raises: [CancelledError]).} = # Compute subnet, then route attestation let @@ -252,7 +254,7 @@ proc routeAttestation*( attestation = shortLog(attestation) return committee_index = - shufflingRef.get_committee_index(attestation.committee_index()).valueOr: + shufflingRef.get_committee_index(attestation.committee_index(on_chain)).valueOr: notice "Invalid committee index in attestation", attestation = shortLog(attestation) return err("Invalid committee index in attestation") diff --git a/ncli/resttest-rules.json b/ncli/resttest-rules.json index aec181d077..b65a702341 100644 --- a/ncli/resttest-rules.json +++ b/ncli/resttest-rules.json @@ -3807,6 +3807,162 @@ "body": [{"operator": "jstructcmpns", "value": {"code": 400, "message": ""}}] } }, + { + "topics": ["beacon", "pool_attestations_electra"], + "request": { + "url": "/eth/v2/beacon/pool/attestations", + "headers": {"Accept": "application/json"} + }, + "response": { + "status": {"operator": "equals", "value": "200"}, + "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals", "eth-consensus-version": "electra"}], + "body": [{"operator": "jstructcmps", "start": ["data"],"value": [{"aggregation_bits": "", "signature": "", "data": {"slot": "", "index": "", "beacon_block_root": "", "source": {"epoch": "", "root": ""}, "target": {"epoch": "", "root": ""}}}]}] + } + }, + { + "topics": ["beacon", "pool_attestations_electra"], + "request": { + "url": "/eth/v2/beacon/pool/attestations?slot=0", + "headers": {"Eth-Consensus-Version": "electra", "Accept": "application/json"} + }, + "response": { + "status": {"operator": "equals", "value": "200"}, + "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals", "eth-consensus-version": "electra"}], + "body": [{"operator": "jstructcmps", "start": ["data"],"value": [{"aggregation_bits": "", "signature": "", "data": {"slot": "", "index": "", "beacon_block_root": "", "source": {"epoch": "", "root": ""}, "target": {"epoch": "", "root": ""}}}]}] + } + }, + { + "topics": ["beacon", "pool_attestations_electra"], + "request": { + "url": "/eth/v2/beacon/pool/attestations?slot=18446744073709551615", + "headers": {"Eth-Consensus-Version": "electra", "Accept": "application/json"} + }, + "response": { + "status": {"operator": "equals", "value": "200"}, + "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals", "eth-consensus-version": "electra"}], + "body": [{"operator": "jstructcmps", "start": ["data"],"value": [{"aggregation_bits": "", "signature": "", "data": {"slot": "", "index": "", "beacon_block_root": "", "source": {"epoch": "", "root": ""}, "target": {"epoch": "", "root": ""}}}]}] + } + }, + { + "topics": ["beacon", "pool_attestations_electra"], + "request": { + "url": "/eth/v2/beacon/pool/attestations?slot=18446744073709551616", + "headers": {"Eth-Consensus-Version": "electra", "Accept": "application/json"} + }, + "response": { + "status": {"operator": "equals", "value": "400"}, + "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals", "eth-consensus-version": "electra"}], + "body": [{"operator": "jstructcmpns", "value": {"code": 400, "message": ""}}] + } + }, + { + "topics": ["beacon", "pool_attestations_electra"], + "request": { + "url": "/eth/v2/beacon/pool/attestations?slot=word", + "headers": {"Eth-Consensus-Version": "electra", "Accept": "application/json"} + }, + "response": { + "status": {"operator": "equals", "value": "400"}, + "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals", "eth-consensus-version": "electra"}], + "body": [{"operator": "jstructcmpns", "value": {"code": 400, "message": ""}}] + } + }, + { + "topics": ["beacon", "pool_attestations_electra"], + "request": { + "url": "/eth/v2/beacon/pool/attestations?committee_index=0", + "headers": {"Eth-Consensus-Version": "electra", "Accept": "application/json"} + }, + "response": { + "status": {"operator": "equals", "value": "200"}, + "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals", "eth-consensus-version": "electra"}], + "body": [{"operator": "jstructcmps", "start": ["data"],"value": [{"aggregation_bits": "", "signature": "", "data": {"slot": "", "index": "", "beacon_block_root": "", "source": {"epoch": "", "root": ""}, "target": {"epoch": "", "root": ""}}}]}] + } + }, + { + "topics": ["beacon", "pool_attestations_electra"], + "request": { + "url": "/eth/v2/beacon/pool/attestations?committee_index=18446744073709551615", + "headers": {"Eth-Consensus-Version": "electra", "Accept": "application/json"} + }, + "response": { + "status": {"operator": "equals", "value": "400"}, + "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals", "eth-consensus-version": "electra"}], + "body": [{"operator": "jstructcmpns", "value": {"code": 400, "message": ""}}] + } + }, + { + "topics": ["beacon", "pool_attestations_electra"], + "request": { + "url": "/eth/v2/beacon/pool/attestations?committee_index=18446744073709551616", + "headers": {"Eth-Consensus-Version": "electra", "Accept": "application/json"} + }, + "response": { + "status": {"operator": "equals", "value": "400"}, + "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals", "eth-consensus-version": "electra"}], + "body": [{"operator": "jstructcmpns", "value": {"code": 400, "message": ""}}] + } + }, + { + "topics": ["beacon", "pool_attestations_electra"], + "request": { + "url": "/eth/v2/beacon/pool/attestations?committee_index=word", + "headers": {"Eth-Consensus-Version": "electra", "Accept": "application/json"} + }, + "response": { + "status": {"operator": "equals", "value": "400"}, + "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals", "eth-consensus-version": "electra"}], + "body": [{"operator": "jstructcmpns", "value": {"code": 400, "message": ""}}] + } + }, + { + "topics": ["beacon", "pool_attestations_electra"], + "request": { + "url": "/eth/v2/beacon/pool/attestations?slot=0&committee_index=0", + "headers": {"Eth-Consensus-Version": "electra", "Accept": "application/json"} + }, + "response": { + "status": {"operator": "equals", "value": "200"}, + "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals", "eth-consensus-version": "electra"}], + "body": [{"operator": "jstructcmps", "start": ["data"],"value": [{"aggregation_bits": "", "signature": "", "data": {"slot": "", "index": "", "beacon_block_root": "", "source": {"epoch": "", "root": ""}, "target": {"epoch": "", "root": ""}}}]}] + } + }, + { + "topics": ["beacon", "pool_attestations_electra"], + "request": { + "url": "/eth/v2/beacon/pool/attestations?slot=word&committee_index=word", + "headers": {"Eth-Consensus-Version": "electra", "Accept": "application/json"} + }, + "response": { + "status": {"operator": "equals", "value": "400"}, + "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals", "eth-consensus-version": "electra"}], + "body": [{"operator": "jstructcmpns", "value": {"code": 400, "message": ""}}] + } + }, + { + "topics": ["beacon", "pool_attestations_electra"], + "request": { + "url": "/eth/v2/beacon/pool/attestations?slot=18446744073709551615&committee_index=18446744073709551615", + "headers": {"Eth-Consensus-Version": "electra", "Accept": "application/json"} + }, + "response": { + "status": {"operator": "equals", "value": "400"}, + "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals", "eth-consensus-version": "electra"}], + "body": [{"operator": "jstructcmpns", "value": {"code": 400, "message": ""}}] + } + }, + { + "topics": ["beacon", "pool_attestations_electra"], + "request": { + "url": "/eth/v2/beacon/pool/attestations?slot=18446744073709551616&committee_index=18446744073709551616", + "headers": {"Eth-Consensus-Version": "electra", "Accept": "application/json"} + }, + "response": { + "status": {"operator": "equals", "value": "400"}, + "headers": [{"key": "Content-Type", "value": "application/json", "operator": "equals", "eth-consensus-version": "electra"}], + "body": [{"operator": "jstructcmpns", "value": {"code": 400, "message": ""}}] + } + }, { "topics": ["beacon", "pool_attester_slashings"], "request": {