common: configure and fix kaustinen4 verkle testnet sync

packages/block/src/from-beacon-payload.ts

@@ -1,6 +1,7 @@
 import { bigIntToHex } from '@ethereumjs/util'
 
 import type { ExecutionPayload, VerkleExecutionWitness } from './types.js'
+import type { PrefixedHexString } from '@ethereumjs/util'
 
 type BeaconWithdrawal = {
   index: string
@@ -34,6 +35,59 @@ export type BeaconPayloadJson = {
   execution_witness?: VerkleExecutionWitness
 }
 
+type VerkleProofSnakeJson = {
+  commitments_by_path: PrefixedHexString[]
+  d: PrefixedHexString
+  depth_extension_present: PrefixedHexString
+  ipa_proof: {
+    cl: PrefixedHexString[]
+    cr: PrefixedHexString[]
+    final_evaluation: PrefixedHexString
+  }
+  other_stems: PrefixedHexString[]
+}
+
+type VerkleStateDiffSnakeJson = {
+  stem: PrefixedHexString
+  suffix_diffs: {
+    current_value: PrefixedHexString | null
+    new_value: PrefixedHexString | null
+    suffix: number | string
+  }[]
+}
+
+type VerkleExecutionWitnessSnakeJson = {
+  state_diff: VerkleStateDiffSnakeJson[]
+  verkle_proof: VerkleProofSnakeJson
+}
+
+function parseExecutionWitnessFromSnakeJson({
+  state_diff,
+  verkle_proof,
+}: VerkleExecutionWitnessSnakeJson): VerkleExecutionWitness {
+  return {
+    stateDiff: state_diff.map(({ stem, suffix_diffs }) => ({
+      stem,
+      suffixDiffs: suffix_diffs.map(({ current_value, new_value, suffix }) => ({
+        currentValue: current_value,
+        newValue: new_value,
+        suffix,
+      })),
+    })),
+    verkleProof: {
+      commitmentsByPath: verkle_proof.commitments_by_path,
+      d: verkle_proof.d,
+      depthExtensionPresent: verkle_proof.depth_extension_present,
+      ipaProof: {
+        cl: verkle_proof.ipa_proof.cl,
+        cr: verkle_proof.ipa_proof.cr,
+        finalEvaluation: verkle_proof.ipa_proof.final_evaluation,
+      },
+      otherStems: verkle_proof.other_stems,
+    },
+  }
+}
+
 /**
  * Converts a beacon block execution payload JSON object {@link BeaconPayloadJson} to the {@link ExecutionPayload} data needed to construct a {@link Block}.
  * The JSON data can be retrieved from a consensus layer (CL) client on this Beacon API `/eth/v2/beacon/blocks/[block number]`
@@ -75,9 +129,13 @@ export function executionPayloadFromBeaconPayload(payload: BeaconPayloadJson): E
     executionPayload.parentBeaconBlockRoot = payload.parent_beacon_block_root
   }
   if (payload.execution_witness !== undefined && payload.execution_witness !== null) {
-    // the casing structure in payload is already camel case, might be updated in
-    // kaustinen relaunch
-    executionPayload.executionWitness = payload.execution_witness
+    // the casing structure in payload could be camel case or snake depending upon the CL
+    executionPayload.executionWitness =
+      payload.execution_witness.verkleProof !== undefined
+        ? payload.execution_witness
+        : parseExecutionWitnessFromSnakeJson(
+            payload.execution_witness as unknown as VerkleExecutionWitnessSnakeJson
+          )
   }
 
   return executionPayload

packages/client/src/execution/vmexecution.ts

@@ -218,6 +218,26 @@ export class VMExecution extends Execution {
         return
       }
 
+      const blockchain = this.chain.blockchain
+      if (typeof blockchain.getIteratorHead !== 'function') {
+        throw new Error('cannot get iterator head: blockchain has no getIteratorHead function')
+      }
+      const headBlock = await blockchain.getIteratorHead()
+      const { number, timestamp, stateRoot } = headBlock.header
+      this.chainStatus = {
+        height: number,
+        status: ExecStatus.VALID,
+        root: stateRoot,
+        hash: headBlock.hash(),
+      }
+
+      if (typeof blockchain.getTotalDifficulty !== 'function') {
+        throw new Error('cannot get iterator head: blockchain has no getTotalDifficulty function')
+      }
+      const td = await blockchain.getTotalDifficulty(headBlock.header.hash())
+      this.config.execCommon.setHardforkBy({ blockNumber: number, td, timestamp })
+      this.hardfork = this.config.execCommon.hardfork()
+
       if (this.config.execCommon.gteHardfork(Hardfork.Prague)) {
         if (!this.config.statelessVerkle) {
           throw Error(`Currently stateful verkle execution not supported`)
@@ -233,33 +253,19 @@ export class VMExecution extends Execution {
         this.vm = this.merkleVM!
       }
 
-      if (typeof this.vm.blockchain.getIteratorHead !== 'function') {
-        throw new Error('cannot get iterator head: blockchain has no getIteratorHead function')
-      }
-      const headBlock = await this.vm.blockchain.getIteratorHead()
-      const { number, timestamp, stateRoot } = headBlock.header
-      this.chainStatus = {
-        height: number,
-        status: ExecStatus.VALID,
-        root: stateRoot,
-        hash: headBlock.hash(),
-      }
       if (number === BIGINT_0) {
         const genesisState =
-          this.chain['_customGenesisState'] ?? getGenesis(Number(this.vm.common.chainId()))
-        if (!genesisState) {
+          this.chain['_customGenesisState'] ?? getGenesis(Number(blockchain.common.chainId()))
+        if (
+          !genesisState &&
+          (this.vm instanceof DefaultStateManager || !this.config.statelessVerkle)
+        ) {
           throw new Error('genesisState not available')
+        } else {
+          await this.vm.stateManager.generateCanonicalGenesis(genesisState)
         }
-        await this.vm.stateManager.generateCanonicalGenesis(genesisState)
       }
 
-      if (typeof this.vm.blockchain.getTotalDifficulty !== 'function') {
-        throw new Error('cannot get iterator head: blockchain has no getTotalDifficulty function')
-      }
-      const td = await this.vm.blockchain.getTotalDifficulty(headBlock.header.hash())
-      this.config.execCommon.setHardforkBy({ blockNumber: number, td, timestamp })
-      this.hardfork = this.config.execCommon.hardfork()
-
       await super.open()
       // TODO: Should a run be started to execute any left over blocks?
       // void this.run()

packages/client/test/rpc/engine/kaustinen4.spec.ts

@@ -0,0 +1,107 @@
+import { Block, BlockHeader, executionPayloadFromBeaconPayload } from '@ethereumjs/block'
+import * as td from 'testdouble'
+import { assert, describe, it } from 'vitest'
+
+import blocks from '../../testdata/blocks/kaustinen4.json'
+import genesisJSON from '../../testdata/geth-genesis/kaustinen4.json'
+import { getRpcClient, setupChain } from '../helpers.js'
+
+import type { Chain } from '../../../src/blockchain'
+import type { BeaconPayloadJson } from '@ethereumjs/block'
+import type { Common } from '@ethereumjs/common'
+import type { HttpClient } from 'jayson/promise'
+const genesisVerkleStateRoot = '0x382960711d9ccf58b9db20122e2253eb9bfa99d513f8c9d4e85b55971721f4de'
+const genesisVerkleBlockHash = '0x8493ed97fd4314acb6ed519867b086dc698e25df37ebe8f2bc77313537710744'
+
+/**
+ * One can run this test in two formats:
+ *   1. On the saved blocks, comma separated which are limited (345,375,368,467)
+ *      `TEST_SAVED_NUMBERS=345,375,368,467 npx vitest run test/rpc/engine/kaustinen4.spec.ts`
+ *   2. Directly pull slots from a kaustinen beacon url
+ *     `TEST_ONLINE_SLOTS=345,353..360 PEER_BEACON_URL=https://beacon.verkle-gen-devnet-4.ethpandaops.io npx vitest run test/rpc/engine/kaustinen4.spec.ts`
+ */
+
+const originalValidate = (BlockHeader as any).prototype._consensusFormatValidation
+
+async function fetchExecutionPayload(
+  peerBeaconUrl: string,
+  slot: number | string
+): Promise<BeaconPayloadJson> {
+  const beaconBlock = await (await fetch(`${peerBeaconUrl}/eth/v2/beacon/blocks/${slot}`)).json()
+  return beaconBlock.data.message.body.execution_payload
+}
+
+async function runBlock(
+  { chain, rpc, common }: { chain: Chain; rpc: HttpClient; common: Common },
+  { execute, parent }: { execute: any; parent: any }
+) {
+  const blockCache = chain.blockCache
+
+  const parentPayload = executionPayloadFromBeaconPayload(parent as any)
+  const parentBlock = await Block.fromExecutionPayload(parentPayload, { common })
+  blockCache.remoteBlocks.set(parentPayload.blockHash.slice(2), parentBlock)
+  blockCache.executedBlocks.set(parentPayload.blockHash.slice(2), parentBlock)
+
+  const executePayload = executionPayloadFromBeaconPayload(execute as any)
+  const res = await rpc.request('engine_newPayloadV2', [executePayload])
+  assert.equal(res.result.status, 'VALID', 'valid status should be received')
+}
+
+describe(`valid verkle network setup`, async () => {
+  const { server, chain, common } = await setupChain(genesisJSON, 'post-merge', {
+    engine: true,
+    genesisStateRoot: genesisVerkleStateRoot,
+  })
+  const rpc = getRpcClient(server)
+  it('genesis should be correctly setup', async () => {
+    const res = await rpc.request('eth_getBlockByNumber', ['0x0', false])
+
+    const block0 = res.result
+    assert.equal(block0.hash, genesisVerkleBlockHash)
+    assert.equal(block0.stateRoot, genesisVerkleStateRoot)
+  })
+
+  // currently it seems the the blocks can't be played one after another as it seems
+  // to not do clean init of the statemanager. this isn't a problem in sequential
+  // execution, but need to be fixed up in the stateless random execution
+  // TEST_SAVED_NUMBERS=353,368,374,479
+  const savedTestCases = process.env.TEST_SAVED_NUMBERS?.split(',') ?? []
+
+  for (const testCase of savedTestCases) {
+    it(`run saved block ${testCase}`, async () => {
+      const testData = blocks[testCase]
+      if (testData === undefined) {
+        throw Error('unavailable data')
+      }
+      await runBlock({ common, chain, rpc }, blocks[testCase])
+    })
+  }
+
+  // we can also test online slots (not numbers because ELs don't provide witnesses
+  // directly, so we need to pull payload from beacon)
+  // TEST_ONLINE_SLOTS=345,375,368..467,
+  for (const numberOrRange of process.env.TEST_ONLINE_SLOTS?.split(',') ?? []) {
+    if (process.env.PEER_BEACON_URL === undefined) {
+      throw Error(`PEER_BEACON_URL env is not defined`)
+    }
+
+    const rangeSplit = numberOrRange.split('..')
+    const startSlot = parseInt(rangeSplit[0])
+    const endSlot = parseInt(rangeSplit[1] ?? rangeSplit[0])
+    let parent = await fetchExecutionPayload(process.env.PEER_BEACON_URL, startSlot - 1)
+    for (let i = startSlot; i <= endSlot; i++) {
+      const execute = await fetchExecutionPayload(process.env.PEER_BEACON_URL, startSlot)
+      console.log(execute.execution_witness)
+      it(`run fetched block slot: ${i} number: ${execute.block_number}`, async () => {
+        await runBlock({ common, chain, rpc }, { parent, execute })
+        parent = execute
+      })
+    }
+  }
+
+  it(`reset TD`, () => {
+    server.close()
+    BlockHeader.prototype['_consensusFormatValidation'] = originalValidate
+    td.reset()
+  })
+})