Mev/Builder api integration (#3969)

* Mev builder api integration

 * extract builder in its own folder space

 * fix engine path

 * rebase fixes

 * refactor the builder with latest beacon state transtion processing changes

 * fix the registration signature

 * make typescript happy

 * remove the early return

 * rebase fixes

 * gas limit option

 * builder enabling on validator, gaslimit and other refac

 * fixes

 * Rebase fixes with the api refac

 * some cleanup fixes

 * comment explaining the immediate registration call

 * use already defined batchItems

 * serialize the validator registration chunks

 * validate transactions root on payload response

 * removing the BlockType from global types and sandboxing it just inside block factory

 * make builder optional in dev validator setup

 * option builder with dev val init with keystores

 * add builder api test routes

 * fix validator tests

 * fix block assembly scripts

 * handle the mergemock customization

 * fix body extraction

 * remove linter superession for boolean expressions

* Implement type-guards

* fix breaking build

* execution is enabled if stateroot non zero

* Fix spec tests

Co-authored-by: dapplion <35266934+dapplion@users.noreply.github.com>

packages/api/src/beacon/routes/beacon/block.ts

@@ -1,7 +1,8 @@
 import {ContainerType} from "@chainsafe/ssz";
 import {ForkName} from "@chainsafe/lodestar-params";
 import {IChainForkConfig} from "@chainsafe/lodestar-config";
-import {phase0, allForks, Slot, Root, ssz} from "@chainsafe/lodestar-types";
+import {phase0, allForks, Slot, Root, ssz, bellatrix} from "@chainsafe/lodestar-types";
+
 import {
   RoutesData,
   ReturnTypes,
@@ -89,6 +90,11 @@ export type Api = {
    * @returns any The block was validated successfully and has been broadcast. It has also been integrated into the beacon node's database.
    */
   publishBlock(block: allForks.SignedBeaconBlock): Promise<void>;
+  /**
+   * Publish a signed blinded block by submitting it to the mev relay and patching in the block
+   * transactions beacon node gets in response.
+   */
+  publishBlindedBlock(block: bellatrix.SignedBlindedBeaconBlock): Promise<void>;
 };
 
 /**
@@ -102,6 +108,7 @@ export const routesData: RoutesData<Api> = {
   getBlockHeaders: {url: "/eth/v1/beacon/headers", method: "GET"},
   getBlockRoot: {url: "/eth/v1/beacon/blocks/:blockId/root", method: "GET"},
   publishBlock: {url: "/eth/v1/beacon/blocks", method: "POST"},
+  publishBlindedBlock: {url: "/eth/v1/beacon/blinded_blocks", method: "POST"},
 };
 
 type BlockIdOnlyReq = {params: {blockId: string | number}};
@@ -115,6 +122,7 @@ export type ReqTypes = {
   getBlockHeaders: {query: {slot?: number; parent_root?: string}};
   getBlockRoot: BlockIdOnlyReq;
   publishBlock: {body: unknown};
+  publishBlindedBlock: {body: unknown};
 };
 
 export function getReqSerializers(config: IChainForkConfig): ReqSerializers<Api, ReqTypes> {
@@ -145,6 +153,7 @@ export function getReqSerializers(config: IChainForkConfig): ReqSerializers<Api,
     },
     getBlockRoot: blockIdOnlyReq,
     publishBlock: reqOnlyBody(AllForksSignedBeaconBlock, Schema.Object),
+    publishBlindedBlock: reqOnlyBody(ssz.bellatrix.SignedBlindedBeaconBlock, Schema.Object),
   };
 }
 

packages/api/src/beacon/routes/validator.ts

@@ -8,6 +8,7 @@ import {
   CommitteeIndex,
   Epoch,
   phase0,
+  bellatrix,
   Root,
   Slot,
   ssz,
@@ -157,6 +158,12 @@ export type Api = {
     graffiti: string
   ): Promise<{data: allForks.BeaconBlock; version: ForkName}>;
 
+  produceBlindedBlock(
+    slot: Slot,
+    randaoReveal: BLSSignature,
+    graffiti: string
+  ): Promise<{data: bellatrix.BlindedBeaconBlock; version: ForkName}>;
+
   /**
    * Produce an attestation data
    * Requests that the beacon node produce an AttestationData.
@@ -218,6 +225,8 @@ export type Api = {
 
   /** Returns validator indices that have been observed to be active on the network */
   getLiveness(indices: ValidatorIndex[], epoch: Epoch): Promise<{data: LivenessResponseData[]}>;
+
+  registerValidator(registrations: bellatrix.SignedValidatorRegistrationV1[]): Promise<void>;
 };
 
 /**
@@ -229,6 +238,7 @@ export const routesData: RoutesData<Api> = {
   getSyncCommitteeDuties: {url: "/eth/v1/validator/duties/sync/:epoch", method: "POST"},
   produceBlock: {url: "/eth/v1/validator/blocks/:slot", method: "GET"},
   produceBlockV2: {url: "/eth/v2/validator/blocks/:slot", method: "GET"},
+  produceBlindedBlock: {url: "/eth/v2/validator/blinded_blocks/:slot", method: "GET"},
   produceAttestationData: {url: "/eth/v1/validator/attestation_data", method: "GET"},
   produceSyncCommitteeContribution: {url: "/eth/v1/validator/sync_committee_contribution", method: "GET"},
   getAggregatedAttestation: {url: "/eth/v1/validator/aggregate_attestation", method: "GET"},
@@ -238,6 +248,7 @@ export const routesData: RoutesData<Api> = {
   prepareSyncCommitteeSubnets: {url: "/eth/v1/validator/sync_committee_subscriptions", method: "POST"},
   prepareBeaconProposer: {url: "/eth/v1/validator/prepare_beacon_proposer", method: "POST"},
   getLiveness: {url: "/eth/v1/validator/liveness", method: "GET"},
+  registerValidator: {url: "/eth/v1/validator/register_validator", method: "POST"},
 };
 
 /* eslint-disable @typescript-eslint/naming-convention */
@@ -247,6 +258,7 @@ export type ReqTypes = {
   getSyncCommitteeDuties: {params: {epoch: Epoch}; body: ValidatorIndex[]};
   produceBlock: {params: {slot: number}; query: {randao_reveal: string; grafitti: string}};
   produceBlockV2: {params: {slot: number}; query: {randao_reveal: string; grafitti: string}};
+  produceBlindedBlock: {params: {slot: number}; query: {randao_reveal: string; grafitti: string}};
   produceAttestationData: {query: {slot: number; committee_index: number}};
   produceSyncCommitteeContribution: {query: {slot: number; subcommittee_index: number; beacon_block_root: string}};
   getAggregatedAttestation: {query: {attestation_data_root: string; slot: number}};
@@ -256,6 +268,7 @@ export type ReqTypes = {
   prepareSyncCommitteeSubnets: {body: unknown};
   prepareBeaconProposer: {body: unknown};
   getLiveness: {query: {indices: ValidatorIndex[]; epoch: Epoch}};
+  registerValidator: {body: unknown};
 };
 
 export function getReqSerializers(): ReqSerializers<Api, ReqTypes> {
@@ -320,6 +333,7 @@ export function getReqSerializers(): ReqSerializers<Api, ReqTypes> {
 
     produceBlock: produceBlock,
     produceBlockV2: produceBlock,
+    produceBlindedBlock: produceBlock,
 
     produceAttestationData: {
       writeReq: (index, slot) => ({query: {slot, committee_index: index}}),
@@ -367,6 +381,7 @@ export function getReqSerializers(): ReqSerializers<Api, ReqTypes> {
       parseReq: ({query}) => [query.indices, query.epoch],
       schema: {query: {indices: Schema.UintArray, epoch: Schema.Uint}},
     },
+    registerValidator: reqOnlyBody(ArrayOf(ssz.bellatrix.SignedValidatorRegistrationV1), Schema.ObjectArray),
   };
 }
 
@@ -412,6 +427,7 @@ export function getReturnTypes(): ReturnTypes<Api> {
     getSyncCommitteeDuties: WithDependentRoot(ArrayOf(SyncDuty)),
     produceBlock: ContainerData(ssz.phase0.BeaconBlock),
     produceBlockV2: WithVersion((fork: ForkName) => ssz[fork].BeaconBlock),
+    produceBlindedBlock: WithVersion((_fork: ForkName) => ssz.bellatrix.BlindedBeaconBlock),
     produceAttestationData: ContainerData(ssz.phase0.AttestationData),
     produceSyncCommitteeContribution: ContainerData(ssz.altair.SyncCommitteeContribution),
     getAggregatedAttestation: ContainerData(ssz.phase0.Attestation),

packages/api/src/builder/routes.ts

@@ -1,28 +1,69 @@
-import {ReturnTypes, RoutesData, ReqSerializers, reqEmpty, ReqEmpty} from "../utils/index.js";
+import {ssz, bellatrix, Slot, Root, BLSPubkey} from "@chainsafe/lodestar-types";
+import {fromHexString, toHexString} from "@chainsafe/ssz";
+import {
+  ReturnTypes,
+  RoutesData,
+  Schema,
+  ReqSerializers,
+  reqOnlyBody,
+  ContainerData,
+  reqEmpty,
+  ReqEmpty,
+  ArrayOf,
+} from "../utils/index.js";
 // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes
 
 export type Api = {
   checkStatus(): Promise<void>;
+  registerValidator(registrations: bellatrix.SignedValidatorRegistrationV1[]): Promise<void>;
+  getPayloadHeader(
+    slot: Slot,
+    parentHash: Root,
+    proposerPubKey: BLSPubkey
+  ): Promise<{data: bellatrix.SignedBuilderBid}>;
+  submitSignedBlindedBlock(
+    signedBlock: bellatrix.SignedBlindedBeaconBlock
+  ): Promise<{data: bellatrix.ExecutionPayload}>;
 };
 
 /**
  * Define javascript values for each route
  */
 export const routesData: RoutesData<Api> = {
   checkStatus: {url: "/eth/v1/builder/status", method: "GET"},
+  registerValidator: {url: "/eth/v1/builder/validators", method: "POST"},
+  getPayloadHeader: {url: "/eth/v1/builder/header/:slot/:parent_hash/:pubkey", method: "GET"},
+  submitSignedBlindedBlock: {url: "/eth/v1/builder/blinded_blocks", method: "POST"},
 };
 
 /* eslint-disable @typescript-eslint/naming-convention */
 export type ReqTypes = {
   checkStatus: ReqEmpty;
+  registerValidator: {body: unknown};
+  getPayloadHeader: {params: {slot: Slot; parent_hash: string; pubkey: string}};
+  submitSignedBlindedBlock: {body: unknown};
 };
 
 export function getReqSerializers(): ReqSerializers<Api, ReqTypes> {
   return {
     checkStatus: reqEmpty,
+    registerValidator: reqOnlyBody(ArrayOf(ssz.bellatrix.SignedValidatorRegistrationV1), Schema.ObjectArray),
+    getPayloadHeader: {
+      writeReq: (slot, parentHash, proposerPubKey) => ({
+        params: {slot, parent_hash: toHexString(parentHash), pubkey: toHexString(proposerPubKey)},
+      }),
+      parseReq: ({params}) => [params.slot, fromHexString(params.parent_hash), fromHexString(params.pubkey)],
+      schema: {
+        params: {slot: Schema.UintRequired, parent_hash: Schema.StringRequired, pubkey: Schema.StringRequired},
+      },
+    },
+    submitSignedBlindedBlock: reqOnlyBody(ssz.bellatrix.SignedBlindedBeaconBlock, Schema.Object),
   };
 }
 
 export function getReturnTypes(): ReturnTypes<Api> {
-  return {};
+  return {
+    getPayloadHeader: ContainerData(ssz.bellatrix.SignedBuilderBid),
+    submitSignedBlindedBlock: ContainerData(ssz.bellatrix.ExecutionPayload),
+  };
 }

packages/api/test/unit/beacon/beacon.test.ts

@@ -56,6 +56,10 @@ describe("beacon / beacon", () => {
       args: [ssz.phase0.SignedBeaconBlock.defaultValue()],
       res: undefined,
     },
+    publishBlindedBlock: {
+      args: [ssz.bellatrix.SignedBlindedBeaconBlock.defaultValue()],
+      res: undefined,
+    },
 
     // pool
 

packages/api/test/unit/beacon/validator.test.ts

@@ -46,6 +46,10 @@ describe("beacon / validator", () => {
       args: [32000, Buffer.alloc(96, 1), "graffiti"],
       res: {data: ssz.altair.BeaconBlock.defaultValue(), version: ForkName.altair},
     },
+    produceBlindedBlock: {
+      args: [32000, Buffer.alloc(96, 1), "graffiti"],
+      res: {data: ssz.bellatrix.BlindedBeaconBlock.defaultValue(), version: ForkName.bellatrix},
+    },
     produceAttestationData: {
       args: [2, 32000],
       res: {data: ssz.phase0.AttestationData.defaultValue()},
@@ -82,6 +86,10 @@ describe("beacon / validator", () => {
       args: [[0], 0],
       res: {data: []},
     },
+    registerValidator: {
+      args: [[ssz.bellatrix.SignedValidatorRegistrationV1.defaultValue()]],
+      res: undefined,
+    },
   });
 
   // TODO: Extra tests to implement maybe

packages/api/test/unit/builder/builder.test.ts

@@ -1,4 +1,7 @@
 import {config} from "@chainsafe/lodestar-config/default";
+import {ssz} from "@chainsafe/lodestar-types";
+import {fromHexString} from "@chainsafe/ssz";
+
 import {Api, ReqTypes} from "../../../src/builder/routes.js";
 import {getClient} from "../../../src/builder/client.js";
 import {getRoutes} from "../../../src/builder/server/index.js";
@@ -10,5 +13,17 @@ describe("builder", () => {
       args: [],
       res: undefined,
     },
+    registerValidator: {
+      args: [[ssz.bellatrix.SignedValidatorRegistrationV1.defaultValue()]],
+      res: undefined,
+    },
+    getPayloadHeader: {
+      args: [1, fromHexString("0x00000000000000000000000000000000"), fromHexString("0x1234")],
+      res: {data: ssz.bellatrix.SignedBuilderBid.defaultValue()},
+    },
+    submitSignedBlindedBlock: {
+      args: [ssz.bellatrix.SignedBlindedBeaconBlock.defaultValue()],
+      res: {data: ssz.bellatrix.ExecutionPayload.defaultValue()},
+    },
   });
 });

packages/beacon-state-transition/src/block/index.ts

@@ -1,7 +1,7 @@
 import {ForkSeq} from "@chainsafe/lodestar-params";
-import {allForks, altair, bellatrix} from "@chainsafe/lodestar-types";
+import {allForks, altair} from "@chainsafe/lodestar-types";
 import {ExecutionEngine} from "../util/executionEngine.js";
-import {isExecutionEnabled} from "../util/bellatrix.js";
+import {getFullOrBlindedPayload, isExecutionEnabled} from "../util/bellatrix.js";
 import {CachedBeaconStateAllForks, CachedBeaconStateBellatrix} from "../types.js";
 import {processExecutionPayload} from "./processExecutionPayload.js";
 import {processSyncAggregate} from "./processSyncCommittee.js";
@@ -20,7 +20,7 @@ export * from "./isValidIndexedAttestation.js";
 export function processBlock(
   fork: ForkSeq,
   state: CachedBeaconStateAllForks,
-  block: allForks.BeaconBlock,
+  block: allForks.FullOrBlindedBeaconBlock,
   verifySignatures = true,
   executionEngine: ExecutionEngine | null
 ): void {
@@ -29,12 +29,10 @@ export function processBlock(
   // The call to the process_execution_payload must happen before the call to the process_randao as the former depends
   // on the randao_mix computed with the reveal of the previous block.
   if (fork >= ForkSeq.bellatrix) {
-    if (isExecutionEnabled(state as CachedBeaconStateBellatrix, (block as bellatrix.BeaconBlock).body)) {
-      processExecutionPayload(
-        state as CachedBeaconStateBellatrix,
-        (block as bellatrix.BeaconBlock).body.executionPayload,
-        executionEngine
-      );
+    const fullOrBlindedPayload = getFullOrBlindedPayload(block);
+
+    if (isExecutionEnabled(state as CachedBeaconStateBellatrix, block)) {
+      processExecutionPayload(state as CachedBeaconStateBellatrix, fullOrBlindedPayload, executionEngine);
     }
   }
 

packages/beacon-state-transition/src/block/processBlockHeader.ts

@@ -1,5 +1,5 @@
 import {toHexString, byteArrayEquals} from "@chainsafe/ssz";
-import {allForks, ssz} from "@chainsafe/lodestar-types";
+import {allForks, ssz, bellatrix} from "@chainsafe/lodestar-types";
 import {CachedBeaconStateAllForks} from "../types.js";
 import {ZERO_HASH} from "../constants/index.js";
 
@@ -9,7 +9,7 @@ import {ZERO_HASH} from "../constants/index.js";
  * PERF: Fixed work independent of block contents.
  * NOTE: `block` body root MUST be pre-cached.
  */
-export function processBlockHeader(state: CachedBeaconStateAllForks, block: allForks.BeaconBlock): void {
+export function processBlockHeader(state: CachedBeaconStateAllForks, block: allForks.FullOrBlindedBeaconBlock): void {
   const slot = state.slot;
   // verify that the slots match
   if (block.slot !== slot) {
@@ -37,13 +37,18 @@ export function processBlockHeader(state: CachedBeaconStateAllForks, block: allF
     );
   }
 
+  const bodyRoot =
+    (block.body as bellatrix.BlindedBeaconBlockBody).executionPayloadHeader !== undefined
+      ? ssz.bellatrix.BlindedBeaconBlockBody.hashTreeRoot(block.body as bellatrix.BlindedBeaconBlockBody)
+      : types.BeaconBlockBody.hashTreeRoot(block.body as bellatrix.BeaconBlockBody);
+
   // cache current block as the new latest block
   state.latestBlockHeader = ssz.phase0.BeaconBlockHeader.toViewDU({
     slot: slot,
     proposerIndex: block.proposerIndex,
     parentRoot: block.parentRoot,
     stateRoot: ZERO_HASH,
-    bodyRoot: types.BeaconBlockBody.hashTreeRoot(block.body),
+    bodyRoot,
   });
 
   // verify proposer is not slashed. Only once per block, may use the slower read from tree

packages/beacon-state-transition/src/block/processExecutionPayload.ts

@@ -1,20 +1,20 @@
-import {bellatrix, ssz} from "@chainsafe/lodestar-types";
+import {ssz, allForks} from "@chainsafe/lodestar-types";
 import {toHexString, byteArrayEquals} from "@chainsafe/ssz";
 import {CachedBeaconStateBellatrix} from "../types.js";
 import {getRandaoMix} from "../util/index.js";
 import {ExecutionEngine} from "../util/executionEngine.js";
-import {isMergeTransitionComplete} from "../util/bellatrix.js";
+import {isExecutionPayload, isMergeTransitionComplete} from "../util/bellatrix.js";
 
 export function processExecutionPayload(
   state: CachedBeaconStateBellatrix,
-  payload: bellatrix.ExecutionPayload,
+  payload: allForks.FullOrBlindedExecutionPayload,
   executionEngine: ExecutionEngine | null
 ): void {
   // Verify consistency of the parent hash, block number, base fee per gas and gas limit
   // with respect to the previous execution payload header
   if (isMergeTransitionComplete(state)) {
     const {latestExecutionPayloadHeader} = state;
-    if (!byteArrayEquals(payload.parentHash as Uint8Array, latestExecutionPayloadHeader.blockHash as Uint8Array)) {
+    if (!byteArrayEquals(payload.parentHash, latestExecutionPayloadHeader.blockHash)) {
       throw Error(
         `Invalid execution payload parentHash ${toHexString(payload.parentHash)} latest blockHash ${toHexString(
           latestExecutionPayloadHeader.blockHash
@@ -25,7 +25,7 @@ export function processExecutionPayload(
 
   // Verify random
   const expectedRandom = getRandaoMix(state, state.epochCtx.epoch);
-  if (!byteArrayEquals(payload.prevRandao as Uint8Array, expectedRandom as Uint8Array)) {
+  if (!byteArrayEquals(payload.prevRandao, expectedRandom)) {
     throw Error(
       `Invalid execution payload random ${toHexString(payload.prevRandao)} expected=${toHexString(expectedRandom)}`
     );
@@ -46,10 +46,14 @@ export function processExecutionPayload(
   // if executionEngine is null, executionEngine.onPayload MUST be called after running processBlock to get the
   // correct randao mix. Since executionEngine will be an async call in most cases it is called afterwards to keep
   // the state transition sync
-  if (executionEngine && !executionEngine.notifyNewPayload(payload)) {
+  if (isExecutionPayload(payload) && executionEngine && !executionEngine.notifyNewPayload(payload)) {
     throw Error("Invalid execution payload");
   }
 
+  const transactionsRoot = isExecutionPayload(payload)
+    ? ssz.bellatrix.Transactions.hashTreeRoot(payload.transactions)
+    : payload.transactionsRoot;
+
   // Cache execution payload header
   state.latestExecutionPayloadHeader = ssz.bellatrix.ExecutionPayloadHeader.toViewDU({
     parentHash: payload.parentHash,
@@ -65,6 +69,6 @@ export function processExecutionPayload(
     extraData: payload.extraData,
     baseFeePerGas: payload.baseFeePerGas,
     blockHash: payload.blockHash,
-    transactionsRoot: ssz.bellatrix.Transactions.hashTreeRoot(payload.transactions),
+    transactionsRoot,
   });
 }