common: configure and fix kaustinen4 verkle testnet sync

packages/block/src/block.ts

@@ -228,12 +228,16 @@
     // executionWitness are not part of the EL fetched blocks via eth_ bodies method
     // they are currently only available via the engine api constructed blocks
     let executionWitness
-    if (header.common.isActivatedEIP(6800) && executionWitnessBytes !== undefined) {
-      executionWitness = JSON.parse(bytesToUtf8(RLP.decode(executionWitnessBytes) as Uint8Array))
-    } else {
-      // don't assign default witness if eip 6800 is implemented as it leads to incorrect
-      // assumptions while executing the block. if not present in input implies its unavailable
-      executionWitness = null
+    if (header.common.isActivatedEIP(6800)) {
+      if (executionWitnessBytes !== undefined) {
+        executionWitness = JSON.parse(bytesToUtf8(RLP.decode(executionWitnessBytes) as Uint8Array))
+      } else if (opts?.executionWitness !== undefined) {
+        executionWitness = opts.executionWitness
+      } else {
+        // don't assign default witness if eip 6800 is implemented as it leads to incorrect
+        // assumptions while executing the block. if not present in input implies its unavailable
+        executionWitness = null
+      }
     }
 
     return new Block(header, transactions, uncleHeaders, withdrawals, opts, executionWitness)
@@ -800,6 +804,37 @@
     }
   }
 
+  toExecutionPayload(): ExecutionPayload {
+    const blockJson = this.toJSON()
+    const header = blockJson.header!
+    const transactions = this.transactions.map((tx) => bytesToHex(tx.serialize())) ?? []
+    const withdrawalsArr = blockJson.withdrawals ? { withdrawals: blockJson.withdrawals } : {}
+
+    const executionPayload: ExecutionPayload = {
+      blockNumber: header.number!,
+      parentHash: header.parentHash!,
+      feeRecipient: header.coinbase!,
+      stateRoot: header.stateRoot!,
+      receiptsRoot: header.receiptTrie!,
+      logsBloom: header.logsBloom!,
+      gasLimit: header.gasLimit!,
+      gasUsed: header.gasUsed!,
+      timestamp: header.timestamp!,
+      extraData: header.extraData!,
+      baseFeePerGas: header.baseFeePerGas!,
+      blobGasUsed: header.blobGasUsed,
+      excessBlobGas: header.excessBlobGas,
+      blockHash: bytesToHex(this.hash()),
+      prevRandao: header.mixHash!,
+      transactions,
+      ...withdrawalsArr,
+      parentBeaconBlockRoot: header.parentBeaconBlockRoot,
+      executionWitness: this.executionWitness,
+    }
+
+    return executionPayload
+  }
+
   /**
    * Return a compact error string representation of the object
    */

packages/block/src/from-beacon-payload.ts

@@ -1,6 +1,7 @@
 import { bigIntToHex } from '@ethereumjs/util'
 
 import type { ExecutionPayload, VerkleExecutionWitness } from './types.js'
+import type { PrefixedHexString } from '@ethereumjs/util'
 
 type BeaconWithdrawal = {
   index: string
@@ -34,6 +35,59 @@
   execution_witness?: VerkleExecutionWitness
 }
 
+type VerkleProofSnakeJson = {
+  commitments_by_path: PrefixedHexString[]
+  d: PrefixedHexString
+  depth_extension_present: PrefixedHexString
+  ipa_proof: {
+    cl: PrefixedHexString[]
+    cr: PrefixedHexString[]
+    final_evaluation: PrefixedHexString
+  }
+  other_stems: PrefixedHexString[]
+}
+
+type VerkleStateDiffSnakeJson = {
+  stem: PrefixedHexString
+  suffix_diffs: {
+    current_value: PrefixedHexString | null
+    new_value: PrefixedHexString | null
+    suffix: number | string
+  }[]
+}
+
+type VerkleExecutionWitnessSnakeJson = {
+  state_diff: VerkleStateDiffSnakeJson[]
+  verkle_proof: VerkleProofSnakeJson
+}
+
+function parseExecutionWitnessFromSnakeJson({
+  state_diff,
+  verkle_proof,
+}: VerkleExecutionWitnessSnakeJson): VerkleExecutionWitness {
+  return {
+    stateDiff: state_diff.map(({ stem, suffix_diffs }) => ({
+      stem,
+      suffixDiffs: suffix_diffs.map(({ current_value, new_value, suffix }) => ({
+        currentValue: current_value,
+        newValue: new_value,
+        suffix,
+      })),
+    })),
+    verkleProof: {
+      commitmentsByPath: verkle_proof.commitments_by_path,
+      d: verkle_proof.d,
+      depthExtensionPresent: verkle_proof.depth_extension_present,
+      ipaProof: {
+        cl: verkle_proof.ipa_proof.cl,
+        cr: verkle_proof.ipa_proof.cr,
+        finalEvaluation: verkle_proof.ipa_proof.final_evaluation,
+      },
+      otherStems: verkle_proof.other_stems,
+    },
+  }
+}
+
 /**
  * Converts a beacon block execution payload JSON object {@link BeaconPayloadJson} to the {@link ExecutionPayload} data needed to construct a {@link Block}.
  * The JSON data can be retrieved from a consensus layer (CL) client on this Beacon API `/eth/v2/beacon/blocks/[block number]`
@@ -75,9 +129,13 @@
     executionPayload.parentBeaconBlockRoot = payload.parent_beacon_block_root
   }
   if (payload.execution_witness !== undefined && payload.execution_witness !== null) {
-    // the casing structure in payload is already camel case, might be updated in
-    // kaustinen relaunch
-    executionPayload.executionWitness = payload.execution_witness
+    // the casing structure in payload could be camel case or snake depending upon the CL
+    executionPayload.executionWitness =
+      payload.execution_witness.verkleProof !== undefined
+        ? payload.execution_witness
+        : parseExecutionWitnessFromSnakeJson(
+            payload.execution_witness as unknown as VerkleExecutionWitnessSnakeJson
+          )
   }
 
   return executionPayload

packages/block/src/types.ts

@@ -69,6 +69,8 @@ export interface BlockOptions {
    *  Skip consensus format validation checks on header if set. Defaults to false.
    */
   skipConsensusFormatValidation?: boolean
+
+  executionWitness?: VerkleExecutionWitness
 }
 
 export interface VerkleProof {

packages/client/bin/cli.ts

@@ -381,6 +381,16 @@ const args: ClientOpts = yargs
       'Block number to start syncing from. Must be lower than the local chain tip. Note: this is destructive and removes blocks from the blockchain, please back up your datadir before using.',
     number: true,
   })
+  .option('startExecutionFrom', {
+    describe:
+      'Block number to start/restart execution from. For merkle based state, parent state should be present in the the db while in verkle stateless mode the chain should be synced till the block and witnesses available this block onwards',
+    number: true,
+  })
+  .option('startExecution', {
+    describe:
+      'Start execution of unexecuted blocks without waiting for the CL fcU, set to `true` if `startExecutionFrom` provided',
+    boolean: true,
+  })
   .option('isSingleNode', {
     describe:
       'To run client in single node configuration without need to discover the sync height from peer. Particularly useful in test configurations. This flag is automically activated in the "dev" mode',
@@ -420,6 +430,15 @@ const args: ClientOpts = yargs
       'Skip executing blocks in new payload calls in engine, alias for --engineNewpayloadMaxExecute=0 and overrides any engineNewpayloadMaxExecute if also provided',
     boolean: true,
   })
+  .option('ignoreStatelessInvalidExecs', {
+    describe:
+      'Ignore stateless execution failures and keep moving the vm execution along using execution witnesses available in block (verkle). Sets/overrides --statelessVerkle=true and --engineNewpayloadMaxExecute=0 to prevent engine newPayload direct block execution where block execution faliures may stall the CL client. Useful for debugging the verkle. If provided a valid filename as arg, the invalid blocks will be stored there which one may use later for debugging',
+    coerce: (arg: string | boolean) =>
+      typeof arg === 'string' && arg !== 'true' && arg !== 'false'
+        ? path.resolve(arg)
+        : arg === true || arg === 'true',
+    hidden: true,
+  })
   .option('useJsCrypto', {
     describe: 'Use pure Javascript cryptography functions',
     boolean: true,
@@ -525,6 +544,52 @@ async function startBlock(client: EthereumClient) {
   }
 }
 
+async function startExecutionFrom(client: EthereumClient) {
+  if (args.startExecutionFrom === undefined) return
+  const startExecutionFrom = BigInt(args.startExecutionFrom)
+
+  const height = client.chain.headers.height
+  if (height < startExecutionFrom) {
+    logger.error(`Cannot start merkle chain higher than current height ${height}`)
+    process.exit()
+  }
+
+  const startExecutionBlock = await client.chain.getBlock(startExecutionFrom)
+  const startExecutionParent = await client.chain.getBlock(startExecutionBlock.header.parentHash)
+  const startExecutionParentTd = await client.chain.getTd(
+    startExecutionParent.hash(),
+    startExecutionParent.header.number
+  )
+
+  const startExecutionHardfork = client.config.execCommon.getHardforkBy({
+    blockNumber: startExecutionBlock.header.number,
+    td: startExecutionParentTd,
+    timestamp: startExecutionBlock.header.timestamp,
+  })
+
+  if (
+    client.config.execCommon.hardforkGteHardfork(startExecutionHardfork, Hardfork.Prague) &&
+    client.config.statelessVerkle
+  ) {
+    // for stateless verkle sync execution witnesses are available and hence we can blindly set the vmHead
+    // to startExecutionParent's hash
+    try {
+      await client.chain.blockchain.setIteratorHead('vm', startExecutionParent.hash())
+      await client.chain.update(false)
+      logger.info(
+        `vmHead set to ${client.chain.headers.height} for starting stateless execution at hardfork=${startExecutionHardfork}`
+      )
+    } catch (err: any) {
+      logger.error(`Error setting vmHead for starting stateless execution: ${err}`)
+      process.exit()
+    }
+  } else {
+    // we need parent state availability to set the vmHead to the parent
+    logger.error(`Stateful execution reset not implemented at hardfork=${startExecutionHardfork}`)
+    process.exit()
+  }
+}
+
 /**
  * Starts and returns the {@link EthereumClient}
  */
@@ -601,6 +666,9 @@ async function startClient(
   if (typeof args.startBlock === 'number') {
     await startBlock(client)
   }
+  if (typeof args.startExecutionFrom === 'number') {
+    await startExecutionFrom(client)
+  }
 
   // update client's sync status and start txpool if synchronized
   client.config.updateSynchronizedState(client.chain.headers.latest)
@@ -959,6 +1027,7 @@ async function run() {
   const multiaddrs = args.multiaddrs !== undefined ? parseMultiaddrs(args.multiaddrs) : undefined
   const mine = args.mine !== undefined ? args.mine : args.dev !== undefined
   const isSingleNode = args.isSingleNode !== undefined ? args.isSingleNode : args.dev !== undefined
+
   const config = new Config({
     accounts,
     bootnodes,
@@ -997,8 +1066,13 @@ async function run() {
     useStringValueTrieDB: args.useStringValueTrieDB,
     txLookupLimit: args.txLookupLimit,
     pruneEngineCache: args.pruneEngineCache,
-    statelessVerkle: args.statelessVerkle,
-    engineNewpayloadMaxExecute: args.skipEngineExec === true ? 0 : args.engineNewpayloadMaxExecute,
+    statelessVerkle: args.ignoreStatelessInvalidExecs !== false ? true : args.statelessVerkle,
+    startExecution: args.startExecutionFrom !== undefined ? true : args.startExecution,
+    engineNewpayloadMaxExecute:
+      args.ignoreStatelessInvalidExecs !== false || args.skipEngineExec === true
+        ? 0
+        : args.engineNewpayloadMaxExecute,
+    ignoreStatelessInvalidExecs: args.ignoreStatelessInvalidExecs,
   })
   config.events.setMaxListeners(50)
   config.events.on(Event.SERVER_LISTENING, (details) => {

packages/client/src/config.ts

@@ -336,6 +336,8 @@ export interface ConfigOptions {
    * Enables stateless verkle block execution (default: false)
    */
   statelessVerkle?: boolean
+  startExecution?: boolean
+  ignoreStatelessInvalidExecs?: boolean | string
 }
 
 export class Config {
@@ -442,6 +444,8 @@ export class Config {
   public readonly savePreimages: boolean
 
   public readonly statelessVerkle: boolean
+  public readonly startExecution: boolean
+  public readonly ignoreStatelessInvalidExecs: boolean | string
 
   public synchronized: boolean
   public lastsyncronized?: boolean
@@ -529,6 +533,8 @@ export class Config {
     this.useStringValueTrieDB = options.useStringValueTrieDB ?? false
 
     this.statelessVerkle = options.statelessVerkle ?? true
+    this.startExecution = options.startExecution ?? false
+    this.ignoreStatelessInvalidExecs = options.ignoreStatelessInvalidExecs ?? false
 
     // Start it off as synchronized if this is configured to mine or as single node
     this.synchronized = this.isSingleNode ?? this.mine