Optimize storage of default values in VerkleNode

packages/verkle/src/node/internalNode.ts

@@ -1,7 +1,7 @@
 import { type VerkleCrypto } from '@ethereumjs/util'
 
 import { BaseVerkleNode } from './baseVerkleNode.js'
-import { NODE_WIDTH, VerkleNodeType } from './types.js'
+import { EMPTY_CHILD, NODE_WIDTH, VerkleNodeType } from './types.js'
 
 import type { ChildNode, VerkleNodeOptions } from './types.js'
 
@@ -12,18 +12,13 @@ export class InternalNode extends BaseVerkleNode<VerkleNodeType.Internal> {
 
   constructor(options: VerkleNodeOptions[VerkleNodeType.Internal]) {
     super(options)
-    this.children =
-      options.children ??
-      new Array(256).fill({
-        commitment: options.verkleCrypto.zeroCommitment,
-        path: new Uint8Array(),
-      })
+    this.children = options.children ?? new Array(256).fill(null)
   }
 
   // Updates the commitment value for a child node at the corresponding index
   setChild(childIndex: number, child: ChildNode) {
     // Get previous child commitment at `index`
-    const oldChildReference = this.children[childIndex]
+    const oldChildReference = this.children[childIndex] ?? EMPTY_CHILD
     // Updates the commitment to the child node at `index`
     this.children[childIndex] = { ...child }
     // Updates the overall node commitment based on the update to this child
@@ -52,7 +47,8 @@ export class InternalNode extends BaseVerkleNode<VerkleNodeType.Internal> {
     const childrenPaths = rawNode.slice(NODE_WIDTH + 1, NODE_WIDTH * 2 + 1)
 
     const children = childrenCommitments.map((commitment, idx) => {
-      return { commitment, path: childrenPaths[idx] }
+      if (commitment.length > 0) return { commitment, path: childrenPaths[idx] }
+      return null
     })
     return new InternalNode({ commitment, verkleCrypto, children })
   }
@@ -81,8 +77,8 @@ export class InternalNode extends BaseVerkleNode<VerkleNodeType.Internal> {
   raw(): Uint8Array[] {
     return [
       new Uint8Array([VerkleNodeType.Internal]),
-      ...this.children.map((child) => child.commitment),
-      ...this.children.map((child) => child.path),
+      ...this.children.map((child) => (child !== null ? child.commitment : new Uint8Array())),
+      ...this.children.map((child) => (child !== null ? child.path : new Uint8Array())),
       this.commitment,
     ]
   }

packages/verkle/src/node/leafNode.ts

@@ -1,15 +1,15 @@
 import { equalsBytes, intToBytes, setLengthLeft, setLengthRight } from '@ethereumjs/util'
 
 import { BaseVerkleNode } from './baseVerkleNode.js'
-import { NODE_WIDTH, VerkleNodeType } from './types.js'
+import { DEFAULT_LEAF_VALUES, DELETED_LEAF_VALUE, NODE_WIDTH, VerkleNodeType } from './types.js'
 import { createCValues } from './util.js'
 
 import type { VerkleNodeOptions } from './types.js'
 import type { VerkleCrypto } from '@ethereumjs/util'
 
 export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
   public stem: Uint8Array
-  public values: Uint8Array[] // Array of 256 possible values represented as 32 byte Uint8Arrays
+  public values: (Uint8Array | 0 | 1)[] // Array of 256 possible values represented as 32 byte Uint8Arrays or 0 if untouched or 1 if deleted
   public c1?: Uint8Array
   public c2?: Uint8Array
   public type = VerkleNodeType.Leaf
@@ -18,7 +18,7 @@
     super(options)
 
     this.stem = options.stem
-    this.values = options.values
+    this.values = options.values ?? DEFAULT_LEAF_VALUES
     this.c1 = options.c1
     this.c2 = options.c2
   }
@@ -32,11 +32,12 @@
    */
   static async create(
     stem: Uint8Array,
-    values: Uint8Array[],
-    verkleCrypto: VerkleCrypto
+    verkleCrypto: VerkleCrypto,
+    values?: (Uint8Array | 0 | 1)[]
   ): Promise<LeafNode> {
     // Generate the value arrays for c1 and c2
+    values = values !== undefined ? values : DEFAULT_LEAF_VALUES
     const c1Values = createCValues(values.slice(0, 128))
     const c2Values = createCValues(values.slice(128))
     let c1 = verkleCrypto.zeroCommitment
     let c2 = verkleCrypto.zeroCommitment
@@ -107,14 +108,23 @@
     const commitment = rawNode[2]
     const c1 = rawNode[3]
     const c2 = rawNode[4]
-    const values = rawNode.slice(5, rawNode.length)
 
+    const values = rawNode
+      .slice(5, rawNode.length)
+      .map((el) => (el.length === 0 ? 0 : equalsBytes(el, DELETED_LEAF_VALUE) ? 1 : el))
     return new LeafNode({ stem, values, c1, c2, commitment, verkleCrypto })
   }
 
   // Retrieve the value at the provided index from the values array
   getValue(index: number): Uint8Array | undefined {
-    return this.values?.[index] ?? null
+    const value = this.values[index]
+    switch (value) {
+      case 0:
+      case 1:
+        return undefined
+      default:
+        return value as Uint8Array
+    }
   }
 
   // Set the value at the provided index from the values array and update the node commitments
@@ -175,7 +185,16 @@
       this.commitment,
       this.c1 ?? new Uint8Array(),
       this.c2 ?? new Uint8Array(),
-      ...this.values,
+      ...this.values.map((val) => {
+        switch (val) {
+          case 0:
+            return new Uint8Array()
+          case 1:
+            return DELETED_LEAF_VALUE
+          default:
+            return val
+        }
+      }),
     ]
   }
 }

packages/verkle/src/node/types.ts

@@ -1,6 +1,7 @@
+import { type VerkleCrypto, bigIntToBytes, bytesToBigInt, setLengthLeft } from '@ethereumjs/util'
+
 import type { InternalNode } from './internalNode.js'
 import type { LeafNode } from './leafNode.js'
-import type { VerkleCrypto } from '@ethereumjs/util'
 
 export enum VerkleNodeType {
   Internal,
@@ -30,11 +31,11 @@ interface BaseVerkleNodeOptions {
 
 interface VerkleInternalNodeOptions extends BaseVerkleNodeOptions {
   // Children nodes of this internal node.
-  children?: ChildNode[]
+  children?: (ChildNode | null)[]
 }
 interface VerkleLeafNodeOptions extends BaseVerkleNodeOptions {
   stem: Uint8Array
-  values: Uint8Array[]
+  values?: (Uint8Array | 0 | 1)[]
   c1?: Uint8Array
   c2?: Uint8Array
 }
@@ -45,3 +46,17 @@ export interface VerkleNodeOptions {
 }
 
 export const NODE_WIDTH = 256
+
+export const EMPTY_CHILD: ChildNode = {
+  commitment: new Uint8Array(64),
+  path: new Uint8Array(),
+}
+
+export const ZERO_LEAF_VALUE = new Uint8Array(32)
+
+export const DELETED_LEAF_VALUE = setLengthLeft(
+  bigIntToBytes(bytesToBigInt(new Uint8Array(32)) + BigInt(2 ** 128)),
+  32
+)
+
+export const DEFAULT_LEAF_VALUES = new Array(256).fill(0)

packages/verkle/src/node/util.ts

@@ -39,21 +39,36 @@ export function isRawNode(node: Uint8Array | Uint8Array[]): node is Uint8Array[]
  * that is being deleted - should always be false if generating C2 values
  * Returns an array of 256 16byte UintArrays with the leaf marker set for each value that is deleted
  */
-export const createCValues = (values: Uint8Array[], deletedValues = new Array(128).fill(false)) => {
+export const createCValues = (
+  values: (Uint8Array | 0 | 1)[],
+  deletedValues = new Array(128).fill(false)
+) => {
   if (values.length !== 128 || deletedValues.length !== 128)
     throw new Error(`got wrong number of values, expected 128, got ${values.length}`)
   const expandedValues: Uint8Array[] = new Array(256)
   for (let x = 0; x < 128; x++) {
+    let val: Uint8Array
+    switch (values[x]) {
+      case 0: // Leaf value that has never been written before
+        val = new Uint8Array(32)
+        break
+      case 1: // Leaf value that has been overwritten with zeros (i.e. a deleted value)
+        // TODO: Improve performance by only flipping the 129th bit of `expandedValues[x]` (instead of bigint addition)
+        val = bigIntToBytes(bytesToBigInt(new Uint8Array(16)) + BigInt(2 ** 128))
+        break
+      default:
+        val = values[x] as Uint8Array
+        break
+    }
     // We add 16 trailing zeros to each value since all commitments are padded to an array of 32 byte values
     expandedValues[x * 2] = setLengthRight(
       deletedValues[x] === true
-        ? // TODO: Improve performance by only flipping the 129th bit of `expandedValues[x]` (instead of bigint addition)
-          bigIntToBytes(bytesToBigInt(values[x].subarray(0, 16)) + BigInt(2 ** 128))
-        : values[x].slice(0, 16),
+        ? bigIntToBytes(bytesToBigInt(val.subarray(0, 16)) + BigInt(2 ** 128))
+        : val.slice(0, 16),
       32
     )
     // TODO: Decide if we should use slice or subarray here (i.e. do we need to copy these slices or not)
-    expandedValues[x * 2 + 1] = setLengthRight(values[x].slice(16), 32)
+    expandedValues[x * 2 + 1] = setLengthRight(val.slice(16), 32)
   }
   return expandedValues
 }

packages/verkle/src/verkleTree.ts

@@ -251,11 +251,7 @@
       }
     } else {
       // Leaf node doesn't exist, create a new one
-      leafNode = await LeafNode.create(
-        stem,
-        new Array(256).fill(new Uint8Array(32)),
-        this.verkleCrypto
-      )
+      leafNode = await LeafNode.create(stem, this.verkleCrypto)
       this.DEBUG && this.debug(`Creating new leaf node at stem: ${bytesToHex(stem)}`, ['PUT'])
     }
     // Update value in leaf node and push to putStack
@@ -406,7 +402,7 @@
     let child = rootNode.children[key[0]]
 
     // Root node doesn't contain a child node's commitment on the first byte of the path so we're done
     if (equalsBytes(child.commitment, this.verkleCrypto.zeroCommitment)) {
       this.DEBUG && this.debug(`Partial Path ${intToHex(key[0])} - found no child.`, ['FIND_PATH'])
       return result
     }

packages/verkle/test/internalNode.spec.ts

@@ -21,7 +21,7 @@ describe('verkle node - internal', () => {
     // Children nodes should all default to null.
     assert.equal(node.children.length, NODE_WIDTH, 'number of children should equal verkle width')
     assert.ok(
-      node.children.every((child) => equalsBytes(child.commitment, verkleCrypto.zeroCommitment)),
+      node.children.every((child) => child === null),
       'every children should be null'
     )
   })
@@ -39,7 +39,7 @@ describe('verkle node - internal', () => {
     // Children nodes should all default to null.
     assert.equal(node.children.length, NODE_WIDTH, 'number of children should equal verkle width')
     assert.ok(
-      node.children.every((child) => equalsBytes(child.commitment, verkleCrypto.zeroCommitment)),
+      node.children.every((child) => child === null),
       'every children should be null'
     )
   })
@@ -59,4 +59,14 @@ describe('verkle node - internal', () => {
     const decoded = decodeNode(serialized, verkleCrypto)
     assert.deepEqual((decoded as InternalNode).children[0].commitment, child.commitment)
   })
+
+  it('should serialize and deserialize a node with no children', async () => {
+    const node = new InternalNode({
+      verkleCrypto,
+      commitment: verkleCrypto.zeroCommitment,
+    })
+    const serialized = node.serialize()
+    const decoded = decodeNode(serialized, verkleCrypto)
+    assert.equal((decoded as InternalNode).children[0], null)
+  })
 })

packages/verkle/test/leafNode.spec.ts

@@ -2,7 +2,7 @@ import { type VerkleCrypto, equalsBytes, randomBytes } from '@ethereumjs/util'
 import { loadVerkleCrypto } from 'verkle-cryptography-wasm'
 import { assert, beforeAll, describe, it } from 'vitest'
 
-import { VerkleNodeType, isLeafNode } from '../src/node/index.js'
+import { VerkleNodeType, decodeNode, isLeafNode } from '../src/node/index.js'
 import { LeafNode } from '../src/node/leafNode.js'
 
 describe('verkle node - leaf', () => {
@@ -35,7 +35,7 @@ describe('verkle node - leaf', () => {
     assert.ok(equalsBytes(node.c2 as unknown as Uint8Array, c2), 'c2 should be set')
     assert.ok(equalsBytes(node.stem, stem), 'stem should be set')
     assert.ok(
-      values.every((value, index) => equalsBytes(value, node.values[index])),
+      values.every((value, index) => equalsBytes(value, node.values[index] as Uint8Array)),
       'values should be set'
     )
   })
@@ -46,17 +46,37 @@ describe('verkle node - leaf', () => {
     const values = new Array<Uint8Array>(256).fill(new Uint8Array(32))
     values[2] = value
     const stem = key.slice(0, 31)
-    const node = await LeafNode.create(stem, values, verkleCrypto)
+    const node = await LeafNode.create(stem, verkleCrypto, values)
     assert.ok(node instanceof LeafNode)
   })
 
+  it('should create a leafnode with default values', async () => {
+    const key = randomBytes(32)
+    const node = await LeafNode.create(key.slice(0, 31), verkleCrypto)
+    assert.ok(node instanceof LeafNode)
+    assert.equal(node.getValue(0), undefined)
+  })
+
   it('should update a commitment when setting a value', async () => {
     const key = randomBytes(32)
     const stem = key.slice(0, 31)
     const values = new Array<Uint8Array>(256).fill(new Uint8Array(32))
-    const node = await LeafNode.create(stem, values, verkleCrypto)
+    const node = await LeafNode.create(stem, verkleCrypto, values)
     assert.deepEqual(node.c1, verkleCrypto.zeroCommitment)
     node.setValue(0, randomBytes(32))
     assert.notDeepEqual(node.c1, verkleCrypto.zeroCommitment)
   })
+
+  it('should serialize and deserialize a node from raw values', async () => {
+    const key = randomBytes(32)
+    const stem = key.slice(0, 31)
+    const values = new Array<Uint8Array>(256).fill(new Uint8Array(32))
+    const node = await LeafNode.create(stem, verkleCrypto, values)
+    const serialized = node.serialize()
+    const decodedNode = decodeNode(serialized, verkleCrypto)
+    assert.deepEqual(node, decodedNode)
+
+    const defaultNode = await LeafNode.create(randomBytes(31), verkleCrypto)
+    assert.deepEqual(defaultNode, decodeNode(defaultNode.serialize(), verkleCrypto))
+  })
 })