Optimize storage of default values in VerkleNode

packages/verkle/src/node/internalNode.ts

@@ -12,18 +12,19 @@ export class InternalNode extends BaseVerkleNode<VerkleNodeType.Internal> {
 
   constructor(options: VerkleNodeOptions[VerkleNodeType.Internal]) {
     super(options)
-    this.children =
-      options.children ??
-      new Array(256).fill({
-        commitment: options.verkleCrypto.zeroCommitment,
-        path: new Uint8Array(),
-      })
+    this.children = options.children ?? new Array(256).fill(null)
   }
 
   // Updates the commitment value for a child node at the corresponding index
   setChild(childIndex: number, child: ChildNode) {
     // Get previous child commitment at `index`
-    const oldChildReference = this.children[childIndex]
+    const oldChildReference =
+      this.children[childIndex] !== null
+        ? this.children[childIndex]
+        : {
+            commitment: this.verkleCrypto.zeroCommitment,
+            path: new Uint8Array(),
+          }
     // Updates the commitment to the child node at `index`
     this.children[childIndex] = { ...child }
     // Updates the overall node commitment based on the update to this child
@@ -52,7 +53,8 @@ export class InternalNode extends BaseVerkleNode<VerkleNodeType.Internal> {
     const childrenPaths = rawNode.slice(NODE_WIDTH + 1, NODE_WIDTH * 2 + 1)
 
     const children = childrenCommitments.map((commitment, idx) => {
-      return { commitment, path: childrenPaths[idx] }
+      if (commitment.length > 0) return { commitment, path: childrenPaths[idx] }
+      return null
     })
     return new InternalNode({ commitment, verkleCrypto, children })
   }
@@ -81,8 +83,8 @@ export class InternalNode extends BaseVerkleNode<VerkleNodeType.Internal> {
   raw(): Uint8Array[] {
     return [
       new Uint8Array([VerkleNodeType.Internal]),
-      ...this.children.map((child) => child.commitment),
-      ...this.children.map((child) => child.path),
+      ...this.children.map((child) => (child !== null ? child.commitment : new Uint8Array())),
+      ...this.children.map((child) => (child !== null ? child.path : new Uint8Array())),
       this.commitment,
     ]
   }

packages/verkle/src/node/leafNode.ts

@@ -1,15 +1,21 @@
 import { equalsBytes, intToBytes, setLengthLeft, setLengthRight } from '@ethereumjs/util'
 
 import { BaseVerkleNode } from './baseVerkleNode.js'
-import { NODE_WIDTH, VerkleNodeType } from './types.js'
+import {
+  NODE_WIDTH,
+  VerkleLeafNodeValue,
+  VerkleNodeType,
+  createDefaultLeafValues,
+  createDeletedLeafValue,
+} from './types.js'
 import { createCValues } from './util.js'
 
 import type { VerkleNodeOptions } from './types.js'
 import type { VerkleCrypto } from '@ethereumjs/util'
 
 export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
   public stem: Uint8Array
-  public values: Uint8Array[] // Array of 256 possible values represented as 32 byte Uint8Arrays
+  public values: (Uint8Array | VerkleLeafNodeValue)[] // Array of 256 possible values represented as 32 byte Uint8Arrays or 0 if untouched or 1 if deleted
   public c1?: Uint8Array
   public c2?: Uint8Array
   public type = VerkleNodeType.Leaf
@@ -18,7 +24,7 @@ export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
     super(options)
 
     this.stem = options.stem
-    this.values = options.values
+    this.values = options.values ?? createDefaultLeafValues()
     this.c1 = options.c1
     this.c2 = options.c2
   }
@@ -32,10 +38,11 @@ export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
    */
   static async create(
     stem: Uint8Array,
-    values: Uint8Array[],
-    verkleCrypto: VerkleCrypto
+    verkleCrypto: VerkleCrypto,
+    values?: (Uint8Array | VerkleLeafNodeValue)[]
   ): Promise<LeafNode> {
     // Generate the value arrays for c1 and c2
+    values = values !== undefined ? values : createDefaultLeafValues()
     const c1Values = createCValues(values.slice(0, 128))
     const c2Values = createCValues(values.slice(128))
     let c1 = verkleCrypto.zeroCommitment
@@ -107,26 +114,42 @@ export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
     const commitment = rawNode[2]
     const c1 = rawNode[3]
     const c2 = rawNode[4]
-    const values = rawNode.slice(5, rawNode.length)
 
+    const values = rawNode
+      .slice(5, rawNode.length)
+      .map((el) => (el.length === 0 ? 0 : equalsBytes(el, createDeletedLeafValue()) ? 1 : el))
     return new LeafNode({ stem, values, c1, c2, commitment, verkleCrypto })
   }
 
   // Retrieve the value at the provided index from the values array
   getValue(index: number): Uint8Array | undefined {
-    return this.values?.[index] ?? null
+    const value = this.values[index]
+    switch (value) {
+      case VerkleLeafNodeValue.Untouched:
+      case VerkleLeafNodeValue.Deleted:
+        return undefined
+      default:
+        return value
+    }
   }
 
-  // Set the value at the provided index from the values array and update the node commitments
-  // TODO: Decide whether we need a separate "deleteValue" function since it has special handling
-  // since we never actually delete a node in a verkle trie but overwrite instead
-  setValue(index: number, value: Uint8Array): void {
+  /**
+   * Set the value at the provided index from the values array and update the node commitments
+   * @param index the index of the specific leaf value to be updated
+   * @param value the value to insert into the leaf value at `index`
+   */
+  setValue(index: number, value: Uint8Array | VerkleLeafNodeValue): void {
+    let val
+    // `val` is a bytes representation of `value` used to update the cCommitment
+    if (value instanceof Uint8Array) val = value
+    else
+      val = value === VerkleLeafNodeValue.Untouched ? new Uint8Array(32) : createDeletedLeafValue()
     // First we update c1 or c2 (depending on whether the index is < 128 or not)
     // Generate the 16 byte values representing the 32 byte values in the half of the values array that
     // contain the old value for the leaf node
     const cValues =
       index < 128 ? createCValues(this.values.slice(0, 128)) : createCValues(this.values.slice(128))
-    // The commitment index is the 2 * the suffix (i.e. the position of the value in the values array)
+    // The commitment index is 2 * the suffix (i.e. the position of the value in the values array)
     // here because each 32 byte value in the leaf node is represented as two 16 byte values in the
     // cValues array.
     const commitmentIndex = index < 128 ? index * 2 : (index - 128) * 2
@@ -137,15 +160,15 @@ export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
       commitmentIndex,
       cValues[commitmentIndex],
       // Right pad the value with zeroes since commitments require 32 byte scalars
-      setLengthRight(value.slice(0, 16), 32)
+      setLengthRight(val.slice(0, 16), 32)
     )
     // Update the commitment for the second 16 bytes of the value
     cCommitment = this.verkleCrypto.updateCommitment(
       cCommitment!,
       commitmentIndex + 1,
       cValues[commitmentIndex + 1],
       // Right pad the value with zeroes since commitments require 32 byte scalars
-      setLengthRight(value.slice(16), 32)
+      setLengthRight(val.slice(16), 32)
     )
     // Update the cCommitment corresponding to the index
     let oldCCommitment: Uint8Array | undefined
@@ -158,7 +181,7 @@ export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
     }
     // Set the new values in the values array
     this.values[index] = value
-    // Update leaf node commitment
+    // Update leaf node commitment -- c1 if index is < 128 or c2 otherwise
     const cIndex = index < 128 ? 2 : 3
     this.commitment = this.verkleCrypto.updateCommitment(
       this.commitment,
@@ -175,7 +198,16 @@ export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
       this.commitment,
       this.c1 ?? new Uint8Array(),
       this.c2 ?? new Uint8Array(),
-      ...this.values,
+      ...this.values.map((val) => {
+        switch (val) {
+          case VerkleLeafNodeValue.Untouched:
+            return new Uint8Array()
+          case VerkleLeafNodeValue.Deleted:
+            return createDeletedLeafValue()
+          default:
+            return val
+        }
+      }),
     ]
   }
 }

packages/verkle/src/node/types.ts

@@ -1,6 +1,7 @@
+import { type VerkleCrypto } from '@ethereumjs/util'
+
 import type { InternalNode } from './internalNode.js'
 import type { LeafNode } from './leafNode.js'
-import type { VerkleCrypto } from '@ethereumjs/util'
 
 export enum VerkleNodeType {
   Internal,
@@ -30,11 +31,16 @@ interface BaseVerkleNodeOptions {
 
 interface VerkleInternalNodeOptions extends BaseVerkleNodeOptions {
   // Children nodes of this internal node.
-  children?: ChildNode[]
+  children?: (ChildNode | null)[]
+}
+
+export enum VerkleLeafNodeValue {
+  Untouched = 0,
+  Deleted = 1,
 }
 interface VerkleLeafNodeOptions extends BaseVerkleNodeOptions {
   stem: Uint8Array
-  values: Uint8Array[]
+  values?: (Uint8Array | VerkleLeafNodeValue)[]
   c1?: Uint8Array
   c2?: Uint8Array
 }
@@ -45,3 +51,15 @@ export interface VerkleNodeOptions {
 }
 
 export const NODE_WIDTH = 256
+
+export const createUntouchedLeafValue = () => new Uint8Array(32)
+
+export const createDeletedLeafValue = () => {
+  const bytes = new Uint8Array(32)
+  // Set the 129th bit to 1 directly by setting the 17th byte (index 16) to 0x80
+  bytes[16] = 0x80
+
+  return bytes
+}
+
+export const createDefaultLeafValues = () => new Array(256).fill(0)

packages/verkle/src/node/util.ts

@@ -3,7 +3,13 @@ import { bigIntToBytes, bytesToBigInt, setLengthRight } from '@ethereumjs/util'
 
 import { InternalNode } from './internalNode.js'
 import { LeafNode } from './leafNode.js'
-import { type VerkleNode, VerkleNodeType } from './types.js'
+import {
+  VerkleLeafNodeValue,
+  type VerkleNode,
+  VerkleNodeType,
+  createDeletedLeafValue,
+  createUntouchedLeafValue,
+} from './types.js'
 
 import type { VerkleCrypto } from '@ethereumjs/util'
 
@@ -39,21 +45,37 @@ export function isRawNode(node: Uint8Array | Uint8Array[]): node is Uint8Array[]
  * that is being deleted - should always be false if generating C2 values
  * Returns an array of 256 16byte UintArrays with the leaf marker set for each value that is deleted
  */
-export const createCValues = (values: Uint8Array[], deletedValues = new Array(128).fill(false)) => {
+export const createCValues = (
+  values: (Uint8Array | VerkleLeafNodeValue)[],
+  deletedValues = new Array(128).fill(false)
+) => {
   if (values.length !== 128 || deletedValues.length !== 128)
     throw new Error(`got wrong number of values, expected 128, got ${values.length}`)
   const expandedValues: Uint8Array[] = new Array(256)
   for (let x = 0; x < 128; x++) {
+    const retrievedValue = values[x]
+    let val: Uint8Array
+    switch (retrievedValue) {
+      case VerkleLeafNodeValue.Untouched: // Leaf value that has never been written before
+        val = createUntouchedLeafValue()
+        break
+      case VerkleLeafNodeValue.Deleted: // Leaf value that has been overwritten with zeros (i.e. a deleted value)
+        val = createDeletedLeafValue()
+
+        break
+      default:
+        val = retrievedValue
+        break
+    }
     // We add 16 trailing zeros to each value since all commitments are padded to an array of 32 byte values
     expandedValues[x * 2] = setLengthRight(
       deletedValues[x] === true
-        ? // TODO: Improve performance by only flipping the 129th bit of `expandedValues[x]` (instead of bigint addition)
-          bigIntToBytes(bytesToBigInt(values[x].subarray(0, 16)) + BigInt(2 ** 128))
-        : values[x].slice(0, 16),
+        ? bigIntToBytes(bytesToBigInt(val.subarray(0, 16)) + BigInt(2 ** 128))
+        : val.slice(0, 16),
       32
     )
     // TODO: Decide if we should use slice or subarray here (i.e. do we need to copy these slices or not)
-    expandedValues[x * 2 + 1] = setLengthRight(values[x].slice(16), 32)
+    expandedValues[x * 2 + 1] = setLengthRight(val.slice(16), 32)
   }
   return expandedValues
 }

packages/verkle/src/verkleTree.ts

@@ -251,11 +251,7 @@ export class VerkleTree {
       }
     } else {
       // Leaf node doesn't exist, create a new one
-      leafNode = await LeafNode.create(
-        stem,
-        new Array(256).fill(new Uint8Array(32)),
-        this.verkleCrypto
-      )
+      leafNode = await LeafNode.create(stem, this.verkleCrypto)
       this.DEBUG && this.debug(`Creating new leaf node at stem: ${bytesToHex(stem)}`, ['PUT'])
     }
     // Update value in leaf node and push to putStack
@@ -406,7 +402,7 @@ export class VerkleTree {
     let child = rootNode.children[key[0]]
 
     // Root node doesn't contain a child node's commitment on the first byte of the path so we're done
-    if (equalsBytes(child.commitment, this.verkleCrypto.zeroCommitment)) {
+    if (child === null) {
       this.DEBUG && this.debug(`Partial Path ${intToHex(key[0])} - found no child.`, ['FIND_PATH'])
       return result
     }
@@ -488,7 +484,6 @@ export class VerkleTree {
       verkleCrypto: this.verkleCrypto,
     })
 
-    // Update the child node's commitment and path
     this.DEBUG && this.debug(`No root node. Creating new root node`, ['INITIALIZE'])
     // Set trie root to serialized (aka compressed) commitment for later use in verkle proof
     this.root(this.verkleCrypto.serializeCommitment(rootNode.commitment))
@@ -582,7 +577,6 @@ export class VerkleTree {
   /**
    * Persists the root hash in the underlying database
    */
-  // TODO: Fix how we reference the root node in `findPath` so this method will work correctly
   async persistRoot() {
     if (this._opts.useRootPersistence) {
       await this._db.put(ROOT_DB_KEY, this.root())

packages/verkle/test/internalNode.spec.ts

@@ -21,7 +21,7 @@ describe('verkle node - internal', () => {
     // Children nodes should all default to null.
     assert.equal(node.children.length, NODE_WIDTH, 'number of children should equal verkle width')
     assert.ok(
-      node.children.every((child) => equalsBytes(child.commitment, verkleCrypto.zeroCommitment)),
+      node.children.every((child) => child === null),
       'every children should be null'
     )
   })
@@ -39,7 +39,7 @@ describe('verkle node - internal', () => {
     // Children nodes should all default to null.
     assert.equal(node.children.length, NODE_WIDTH, 'number of children should equal verkle width')
     assert.ok(
-      node.children.every((child) => equalsBytes(child.commitment, verkleCrypto.zeroCommitment)),
+      node.children.every((child) => child === null),
       'every children should be null'
     )
   })
@@ -59,4 +59,14 @@ describe('verkle node - internal', () => {
     const decoded = decodeNode(serialized, verkleCrypto)
     assert.deepEqual((decoded as InternalNode).children[0].commitment, child.commitment)
   })
+
+  it('should serialize and deserialize a node with no children', async () => {
+    const node = new InternalNode({
+      verkleCrypto,
+      commitment: verkleCrypto.zeroCommitment,
+    })
+    const serialized = node.serialize()
+    const decoded = decodeNode(serialized, verkleCrypto)
+    assert.equal((decoded as InternalNode).children[0], null)
+  })
 })