Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize storage of default values in VerkleNode #3476

Merged
merged 24 commits into from
Jul 4, 2024
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
55d425b
Add optimized default value storage
acolytec3 Jun 28, 2024
03534b1
lint
acolytec3 Jun 28, 2024
c3a4ffb
lint
acolytec3 Jun 28, 2024
132c9e4
Check for null instead of equalsBytes
acolytec3 Jul 1, 2024
b0d1523
Fix check for empty child references [no ci]
acolytec3 Jul 1, 2024
0b679a3
Remove unused constant [no ci]
acolytec3 Jul 1, 2024
ce24136
Reorganize tests
acolytec3 Jul 1, 2024
369477e
Fix test
acolytec3 Jul 2, 2024
cc40c72
Merge branch 'master' into optimize-node-value-storage
gabrocheleau Jul 3, 2024
c175633
verkle: enum instead of numbers
gabrocheleau Jul 3, 2024
78e300f
verkle: use enum
gabrocheleau Jul 3, 2024
9c19dfd
verkle: more enum cases
gabrocheleau Jul 3, 2024
320a733
verkle: address todo, use enum and remove now unnecessary typecast
gabrocheleau Jul 3, 2024
ecf4222
Remove obsolete comments
acolytec3 Jul 3, 2024
702b732
verkle: replace bigint conversion with direct bytes manipulation
gabrocheleau Jul 3, 2024
df19cbc
Merge branch 'optimize-node-value-storage' of https://github.com/ethe…
gabrocheleau Jul 3, 2024
5be4cb1
Merge branch 'master' into optimize-node-value-storage
gabrocheleau Jul 3, 2024
c25a93a
verkle: update constant helpers
gabrocheleau Jul 3, 2024
c05e941
Merge branch 'optimize-node-value-storage' of https://github.com/ethe…
gabrocheleau Jul 3, 2024
9c5e3ac
Accept VerkleLeafNodeValue in setValue
acolytec3 Jul 3, 2024
bcfb711
Update test
acolytec3 Jul 3, 2024
275909f
address feedback
acolytec3 Jul 4, 2024
f967f99
Merge branch 'master' into optimize-node-value-storage
acolytec3 Jul 4, 2024
bb7d30c
Merge branch 'master' into optimize-node-value-storage
acolytec3 Jul 4, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 12 additions & 10 deletions packages/verkle/src/node/internalNode.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,19 @@ export class InternalNode extends BaseVerkleNode<VerkleNodeType.Internal> {

constructor(options: VerkleNodeOptions[VerkleNodeType.Internal]) {
super(options)
this.children =
options.children ??
new Array(256).fill({
commitment: options.verkleCrypto.zeroCommitment,
path: new Uint8Array(),
})
this.children = options.children ?? new Array(256).fill(null)
}

// Updates the commitment value for a child node at the corresponding index
setChild(childIndex: number, child: ChildNode) {
// Get previous child commitment at `index`
const oldChildReference = this.children[childIndex]
const oldChildReference =
this.children[childIndex] !== null
? this.children[childIndex]
: {
commitment: this.verkleCrypto.zeroCommitment,
path: new Uint8Array(),
}
// Updates the commitment to the child node at `index`
this.children[childIndex] = { ...child }
// Updates the overall node commitment based on the update to this child
Expand Down Expand Up @@ -52,7 +53,8 @@ export class InternalNode extends BaseVerkleNode<VerkleNodeType.Internal> {
const childrenPaths = rawNode.slice(NODE_WIDTH + 1, NODE_WIDTH * 2 + 1)

const children = childrenCommitments.map((commitment, idx) => {
return { commitment, path: childrenPaths[idx] }
if (commitment.length > 0) return { commitment, path: childrenPaths[idx] }
return null
})
return new InternalNode({ commitment, verkleCrypto, children })
}
Expand Down Expand Up @@ -81,8 +83,8 @@ export class InternalNode extends BaseVerkleNode<VerkleNodeType.Internal> {
raw(): Uint8Array[] {
return [
new Uint8Array([VerkleNodeType.Internal]),
...this.children.map((child) => child.commitment),
...this.children.map((child) => child.path),
...this.children.map((child) => (child !== null ? child.commitment : new Uint8Array())),
...this.children.map((child) => (child !== null ? child.path : new Uint8Array())),
this.commitment,
]
}
Expand Down
41 changes: 33 additions & 8 deletions packages/verkle/src/node/leafNode.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,21 @@
import { equalsBytes, intToBytes, setLengthLeft, setLengthRight } from '@ethereumjs/util'

import { BaseVerkleNode } from './baseVerkleNode.js'
import { NODE_WIDTH, VerkleNodeType } from './types.js'
import {
DEFAULT_LEAF_VALUES,
DELETED_LEAF_VALUE,
NODE_WIDTH,
VerkleNodeType,
createDeletedLeafValue,
} from './types.js'
import { createCValues } from './util.js'

import type { VerkleNodeOptions } from './types.js'
import type { VerkleCrypto } from '@ethereumjs/util'

export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
public stem: Uint8Array
public values: Uint8Array[] // Array of 256 possible values represented as 32 byte Uint8Arrays
public values: (Uint8Array | 0 | 1)[] // Array of 256 possible values represented as 32 byte Uint8Arrays or 0 if untouched or 1 if deleted
acolytec3 marked this conversation as resolved.
Show resolved Hide resolved
public c1?: Uint8Array
public c2?: Uint8Array
public type = VerkleNodeType.Leaf
Expand All @@ -18,7 +24,7 @@ export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
super(options)

this.stem = options.stem
this.values = options.values
this.values = options.values ?? DEFAULT_LEAF_VALUES
this.c1 = options.c1
this.c2 = options.c2
}
Expand All @@ -32,10 +38,11 @@ export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
*/
static async create(
stem: Uint8Array,
values: Uint8Array[],
verkleCrypto: VerkleCrypto
verkleCrypto: VerkleCrypto,
values?: (Uint8Array | 0 | 1)[]
): Promise<LeafNode> {
// Generate the value arrays for c1 and c2
values = values !== undefined ? values : DEFAULT_LEAF_VALUES
const c1Values = createCValues(values.slice(0, 128))
const c2Values = createCValues(values.slice(128))
let c1 = verkleCrypto.zeroCommitment
Expand Down Expand Up @@ -107,14 +114,23 @@ export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
const commitment = rawNode[2]
const c1 = rawNode[3]
const c2 = rawNode[4]
const values = rawNode.slice(5, rawNode.length)

const values = rawNode
.slice(5, rawNode.length)
.map((el) => (el.length === 0 ? 0 : equalsBytes(el, DELETED_LEAF_VALUE) ? 1 : el))
return new LeafNode({ stem, values, c1, c2, commitment, verkleCrypto })
}

// Retrieve the value at the provided index from the values array
getValue(index: number): Uint8Array | undefined {
return this.values?.[index] ?? null
const value = this.values[index]
switch (value) {
case 0:
case 1:
return undefined
default:
return value as Uint8Array
}
}

// Set the value at the provided index from the values array and update the node commitments
Expand Down Expand Up @@ -175,7 +191,16 @@ export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
this.commitment,
this.c1 ?? new Uint8Array(),
this.c2 ?? new Uint8Array(),
...this.values,
...this.values.map((val) => {
switch (val) {
case 0:
return new Uint8Array()
case 1:
return createDeletedLeafValue()
default:
return val
}
}),
]
}
}
19 changes: 16 additions & 3 deletions packages/verkle/src/node/types.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { type VerkleCrypto, bigIntToBytes, bytesToBigInt, setLengthLeft } from '@ethereumjs/util'

import type { InternalNode } from './internalNode.js'
import type { LeafNode } from './leafNode.js'
import type { VerkleCrypto } from '@ethereumjs/util'

export enum VerkleNodeType {
Internal,
Expand Down Expand Up @@ -30,11 +31,11 @@ interface BaseVerkleNodeOptions {

interface VerkleInternalNodeOptions extends BaseVerkleNodeOptions {
// Children nodes of this internal node.
children?: ChildNode[]
children?: (ChildNode | null)[]
}
interface VerkleLeafNodeOptions extends BaseVerkleNodeOptions {
stem: Uint8Array
values: Uint8Array[]
values?: (Uint8Array | 0 | 1)[]
c1?: Uint8Array
c2?: Uint8Array
}
Expand All @@ -45,3 +46,15 @@ export interface VerkleNodeOptions {
}

export const NODE_WIDTH = 256

export const createZeroLeaf = () => new Uint8Array(32)

export const createDeletedLeafValue = () =>
setLengthLeft(bigIntToBytes(bytesToBigInt(new Uint8Array(32)) + BigInt(2 ** 128)), 32)

export const DELETED_LEAF_VALUE = setLengthLeft(
bigIntToBytes(bytesToBigInt(new Uint8Array(32)) + BigInt(2 ** 128)),
32
)
acolytec3 marked this conversation as resolved.
Show resolved Hide resolved

export const DEFAULT_LEAF_VALUES = new Array(256).fill(0)
25 changes: 20 additions & 5 deletions packages/verkle/src/node/util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,21 +39,36 @@ export function isRawNode(node: Uint8Array | Uint8Array[]): node is Uint8Array[]
* that is being deleted - should always be false if generating C2 values
* Returns an array of 256 16byte UintArrays with the leaf marker set for each value that is deleted
*/
export const createCValues = (values: Uint8Array[], deletedValues = new Array(128).fill(false)) => {
export const createCValues = (
values: (Uint8Array | 0 | 1)[],
deletedValues = new Array(128).fill(false)
) => {
if (values.length !== 128 || deletedValues.length !== 128)
throw new Error(`got wrong number of values, expected 128, got ${values.length}`)
const expandedValues: Uint8Array[] = new Array(256)
for (let x = 0; x < 128; x++) {
let val: Uint8Array
switch (values[x]) {
case 0: // Leaf value that has never been written before
val = new Uint8Array(32)
break
case 1: // Leaf value that has been overwritten with zeros (i.e. a deleted value)
// TODO: Improve performance by only flipping the 129th bit of `expandedValues[x]` (instead of bigint addition)
val = bigIntToBytes(bytesToBigInt(new Uint8Array(16)) + BigInt(2 ** 128))
break
default:
val = values[x] as Uint8Array
break
}
// We add 16 trailing zeros to each value since all commitments are padded to an array of 32 byte values
expandedValues[x * 2] = setLengthRight(
deletedValues[x] === true
? // TODO: Improve performance by only flipping the 129th bit of `expandedValues[x]` (instead of bigint addition)
bigIntToBytes(bytesToBigInt(values[x].subarray(0, 16)) + BigInt(2 ** 128))
: values[x].slice(0, 16),
? bigIntToBytes(bytesToBigInt(val.subarray(0, 16)) + BigInt(2 ** 128))
: val.slice(0, 16),
32
)
// TODO: Decide if we should use slice or subarray here (i.e. do we need to copy these slices or not)
expandedValues[x * 2 + 1] = setLengthRight(values[x].slice(16), 32)
expandedValues[x * 2 + 1] = setLengthRight(val.slice(16), 32)
}
return expandedValues
}
Expand Down
8 changes: 2 additions & 6 deletions packages/verkle/src/verkleTree.ts
Original file line number Diff line number Diff line change
Expand Up @@ -251,11 +251,7 @@ export class VerkleTree {
}
} else {
// Leaf node doesn't exist, create a new one
leafNode = await LeafNode.create(
stem,
new Array(256).fill(new Uint8Array(32)),
this.verkleCrypto
)
leafNode = await LeafNode.create(stem, this.verkleCrypto)
this.DEBUG && this.debug(`Creating new leaf node at stem: ${bytesToHex(stem)}`, ['PUT'])
}
// Update value in leaf node and push to putStack
Expand Down Expand Up @@ -406,7 +402,7 @@ export class VerkleTree {
let child = rootNode.children[key[0]]

// Root node doesn't contain a child node's commitment on the first byte of the path so we're done
if (equalsBytes(child.commitment, this.verkleCrypto.zeroCommitment)) {
if (child === null) {
this.DEBUG && this.debug(`Partial Path ${intToHex(key[0])} - found no child.`, ['FIND_PATH'])
return result
}
Expand Down
14 changes: 12 additions & 2 deletions packages/verkle/test/internalNode.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ describe('verkle node - internal', () => {
// Children nodes should all default to null.
assert.equal(node.children.length, NODE_WIDTH, 'number of children should equal verkle width')
assert.ok(
node.children.every((child) => equalsBytes(child.commitment, verkleCrypto.zeroCommitment)),
node.children.every((child) => child === null),
'every children should be null'
)
})
Expand All @@ -39,7 +39,7 @@ describe('verkle node - internal', () => {
// Children nodes should all default to null.
assert.equal(node.children.length, NODE_WIDTH, 'number of children should equal verkle width')
assert.ok(
node.children.every((child) => equalsBytes(child.commitment, verkleCrypto.zeroCommitment)),
node.children.every((child) => child === null),
'every children should be null'
)
})
Expand All @@ -59,4 +59,14 @@ describe('verkle node - internal', () => {
const decoded = decodeNode(serialized, verkleCrypto)
assert.deepEqual((decoded as InternalNode).children[0].commitment, child.commitment)
})

it('should serialize and deserialize a node with no children', async () => {
const node = new InternalNode({
verkleCrypto,
commitment: verkleCrypto.zeroCommitment,
})
const serialized = node.serialize()
const decoded = decodeNode(serialized, verkleCrypto)
assert.equal((decoded as InternalNode).children[0], null)
})
})
28 changes: 24 additions & 4 deletions packages/verkle/test/leafNode.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { type VerkleCrypto, equalsBytes, randomBytes } from '@ethereumjs/util'
import { loadVerkleCrypto } from 'verkle-cryptography-wasm'
import { assert, beforeAll, describe, it } from 'vitest'

import { VerkleNodeType, isLeafNode } from '../src/node/index.js'
import { VerkleNodeType, decodeNode, isLeafNode } from '../src/node/index.js'
import { LeafNode } from '../src/node/leafNode.js'

describe('verkle node - leaf', () => {
Expand Down Expand Up @@ -35,7 +35,7 @@ describe('verkle node - leaf', () => {
assert.ok(equalsBytes(node.c2 as unknown as Uint8Array, c2), 'c2 should be set')
assert.ok(equalsBytes(node.stem, stem), 'stem should be set')
assert.ok(
values.every((value, index) => equalsBytes(value, node.values[index])),
values.every((value, index) => equalsBytes(value, node.values[index] as Uint8Array)),
'values should be set'
)
})
Expand All @@ -46,17 +46,37 @@ describe('verkle node - leaf', () => {
const values = new Array<Uint8Array>(256).fill(new Uint8Array(32))
values[2] = value
const stem = key.slice(0, 31)
const node = await LeafNode.create(stem, values, verkleCrypto)
const node = await LeafNode.create(stem, verkleCrypto, values)
assert.ok(node instanceof LeafNode)
})

it('should create a leafnode with default values', async () => {
const key = randomBytes(32)
const node = await LeafNode.create(key.slice(0, 31), verkleCrypto)
assert.ok(node instanceof LeafNode)
assert.equal(node.getValue(0), undefined)
})

it('should update a commitment when setting a value', async () => {
const key = randomBytes(32)
const stem = key.slice(0, 31)
const values = new Array<Uint8Array>(256).fill(new Uint8Array(32))
const node = await LeafNode.create(stem, values, verkleCrypto)
const node = await LeafNode.create(stem, verkleCrypto, values)
assert.deepEqual(node.c1, verkleCrypto.zeroCommitment)
node.setValue(0, randomBytes(32))
assert.notDeepEqual(node.c1, verkleCrypto.zeroCommitment)
})

it('should serialize and deserialize a node from raw values', async () => {
const key = randomBytes(32)
const stem = key.slice(0, 31)
const values = new Array<Uint8Array>(256).fill(new Uint8Array(32))
const node = await LeafNode.create(stem, verkleCrypto, values)
const serialized = node.serialize()
const decodedNode = decodeNode(serialized, verkleCrypto)
assert.deepEqual(node, decodedNode)

const defaultNode = await LeafNode.create(randomBytes(31), verkleCrypto)
assert.deepEqual(defaultNode, decodeNode(defaultNode.serialize(), verkleCrypto))
})
})
Loading
Loading