From 4ee69acf8588adb46d2e9369d5541fb04380c652 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 4 Sep 2024 09:25:15 -0300 Subject: [PATCH] fix: Split stores per component and split merkle tree operations (#8299) This PR packs two changesets: 1. Spins off a `MerkleTreeAdminOperations` from `MerkleTreeOperations`, to have an interface that does not expose methods that commit changes to the underlying store. World state now exposes a method to get an `ephemeralFork` without these operations, that can be used for answering public simulation requests. Note that we do not yet enforce that no changes go into the store, that requires more changes to have "readonly" versions of the trees and store all the way down. 2. Moves creation of the underlying data store to the factory of each component, so each creates a new db as needed, instead of sharing a single one. This allows to keep separate db files for p2p, archive, and world state. As a bonus, it makes forking world state cheaper. --- yarn-project/archiver/src/factory.ts | 8 ++-- .../aztec-node/src/aztec-node/server.test.ts | 10 ++--- .../aztec-node/src/aztec-node/server.ts | 31 ++++---------- .../aztec/src/cli/cmds/start_archiver.ts | 3 +- .../src/interfaces/merkle_tree_operations.ts | 7 ++-- .../end-to-end/src/e2e_block_building.test.ts | 4 +- .../src/e2e_prover/e2e_prover_test.ts | 4 -- .../src/fixtures/snapshot_manager.ts | 22 ++-------- .../e2e_public_testnet_transfer.test.ts | 7 +--- yarn-project/kv-store/src/utils.ts | 26 ++++++++---- yarn-project/p2p/src/client/index.ts | 15 +++++-- .../reqresp/p2p_client.integration.test.ts | 15 +++++-- .../prover-client/src/mocks/test_context.ts | 5 ++- yarn-project/prover-node/src/factory.ts | 10 +---- .../prover-node/src/prover-node.test.ts | 8 ++-- .../pxe/src/pxe_service/create_pxe_service.ts | 12 +++--- .../src/sequencer/sequencer.test.ts | 7 ++-- .../world-state/src/synchronizer/factory.ts | 7 ++-- .../server_world_state_synchronizer.test.ts | 4 +- .../server_world_state_synchronizer.ts | 28 ++++++++----- .../synchronizer/world_state_synchronizer.ts | 22 +++++----- .../world-state/src/world-state-db/index.ts | 2 +- .../src/world-state-db/merkle_tree_db.ts | 30 +++++++++----- .../merkle_tree_operations_facade.ts | 41 +++++++++++-------- .../src/world-state-db/merkle_trees.ts | 39 +++++++++++------- 25 files changed, 187 insertions(+), 180 deletions(-) diff --git a/yarn-project/archiver/src/factory.ts b/yarn-project/archiver/src/factory.ts index ad7d6cfcdb2..302af46afb1 100644 --- a/yarn-project/archiver/src/factory.ts +++ b/yarn-project/archiver/src/factory.ts @@ -1,4 +1,5 @@ -import { type AztecKVStore } from '@aztec/kv-store'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { createStore } from '@aztec/kv-store/utils'; import { type TelemetryClient } from '@aztec/telemetry-client'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; @@ -7,14 +8,13 @@ import { type ArchiverConfig } from './archiver/config.js'; import { KVArchiverDataStore } from './archiver/index.js'; import { createArchiverClient } from './rpc/archiver_client.js'; -export function createArchiver( +export async function createArchiver( config: ArchiverConfig, - store: AztecKVStore, telemetry: TelemetryClient = new NoopTelemetryClient(), opts: { blockUntilSync: boolean } = { blockUntilSync: true }, ) { if (!config.archiverUrl) { - // first create and sync the archiver + const store = await createStore('archiver', config, createDebugLogger('aztec:archiver:lmdb')); const archiverStore = new KVArchiverDataStore(store, config.maxLogs); return Archiver.createAndSync(config, archiverStore, telemetry, opts.blockUntilSync); } else { diff --git a/yarn-project/aztec-node/src/aztec-node/server.test.ts b/yarn-project/aztec-node/src/aztec-node/server.test.ts index 4a62d116f51..f753431b918 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.test.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.test.ts @@ -4,12 +4,11 @@ import { type L1ToL2MessageSource, type L2BlockSource, type L2LogsSource, + type MerkleTreeAdminOperations, MerkleTreeId, - type MerkleTreeOperations, mockTxForRollup, } from '@aztec/circuit-types'; import { AztecAddress, EthAddress, Fr, GasFees, GlobalVariables, MaxBlockNumber } from '@aztec/circuits.js'; -import { type AztecLmdbStore } from '@aztec/kv-store/lmdb'; import { type P2P } from '@aztec/p2p'; import { type GlobalVariableBuilder } from '@aztec/sequencer-client'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; @@ -24,7 +23,7 @@ import { AztecNodeService } from './server.js'; describe('aztec node', () => { let p2p: MockProxy; let globalVariablesBuilder: MockProxy; - let merkleTreeOps: MockProxy; + let merkleTreeOps: MockProxy; let lastBlockNumber: number; @@ -42,7 +41,7 @@ describe('aztec node', () => { p2p = mock(); globalVariablesBuilder = mock(); - merkleTreeOps = mock(); + merkleTreeOps = mock(); const worldState = mock({ getLatest: () => merkleTreeOps, @@ -59,8 +58,6 @@ describe('aztec node', () => { // all txs use the same allowed FPC class const contractSource = mock(); - const store = mock(); - const aztecNodeConfig: AztecNodeConfig = getConfigEnvVars(); node = new AztecNodeService( @@ -86,7 +83,6 @@ describe('aztec node', () => { 31337, 1, globalVariablesBuilder, - store, new TestCircuitVerifier(), new NoopTelemetryClient(), ); diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index da98547c0bf..ed438882142 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -50,10 +50,9 @@ import { AztecAddress } from '@aztec/foundation/aztec-address'; import { padArrayEnd } from '@aztec/foundation/collection'; import { createDebugLogger } from '@aztec/foundation/log'; import { Timer } from '@aztec/foundation/timer'; -import { type AztecKVStore } from '@aztec/kv-store'; -import { createStore, openTmpStore } from '@aztec/kv-store/utils'; +import { openTmpStore } from '@aztec/kv-store/utils'; import { SHA256Trunc, StandardTree, UnbalancedTree } from '@aztec/merkle-tree'; -import { AztecKVTxPool, InMemoryAttestationPool, type P2P, createP2PClient } from '@aztec/p2p'; +import { InMemoryAttestationPool, type P2P, createP2PClient } from '@aztec/p2p'; import { getCanonicalClassRegisterer } from '@aztec/protocol-contracts/class-registerer'; import { getCanonicalFeeJuice } from '@aztec/protocol-contracts/fee-juice'; import { getCanonicalInstanceDeployer } from '@aztec/protocol-contracts/instance-deployer'; @@ -77,7 +76,7 @@ import { type ProtocolContractAddresses, } from '@aztec/types/contracts'; import { createValidatorClient } from '@aztec/validator-client'; -import { MerkleTrees, type WorldStateSynchronizer, createWorldStateSynchronizer } from '@aztec/world-state'; +import { type WorldStateSynchronizer, createWorldStateSynchronizer } from '@aztec/world-state'; import { type AztecNodeConfig, getPackageInfo } from './config.js'; import { NodeMetrics } from './node_metrics.js'; @@ -104,7 +103,6 @@ export class AztecNodeService implements AztecNode { protected readonly l1ChainId: number, protected readonly version: number, protected readonly globalVariableBuilder: GlobalVariableBuilder, - protected readonly merkleTreesDb: AztecKVStore, private proofVerifier: ClientProtocolCircuitVerifier, private telemetry: TelemetryClient, private log = createDebugLogger('aztec:node'), @@ -131,7 +129,6 @@ export class AztecNodeService implements AztecNode { config: AztecNodeConfig, telemetry?: TelemetryClient, log = createDebugLogger('aztec:node'), - storeLog = createDebugLogger('aztec:node:lmdb'), ): Promise { telemetry ??= new NoopTelemetryClient(); const ethereumChain = createEthereumChain(config.l1RpcUrl, config.l1ChainId); @@ -142,25 +139,17 @@ export class AztecNodeService implements AztecNode { ); } - const store = await createStore(config, config.l1Contracts.rollupAddress, storeLog); - - const archiver = await createArchiver(config, store, telemetry, { blockUntilSync: true }); + const archiver = await createArchiver(config, telemetry, { blockUntilSync: true }); // we identify the P2P transaction protocol by using the rollup contract address. // this may well change in future config.transactionProtocol = `/aztec/tx/${config.l1Contracts.rollupAddress.toString()}`; // create the tx pool and the p2p client, which will need the l2 block source - const p2pClient = await createP2PClient( - config, - store, - new AztecKVTxPool(store, telemetry), - new InMemoryAttestationPool(), - archiver, - ); + const p2pClient = await createP2PClient(config, new InMemoryAttestationPool(), archiver, telemetry); // now create the merkle trees and the world state synchronizer - const worldStateSynchronizer = await createWorldStateSynchronizer(config, store, archiver, telemetry); + const worldStateSynchronizer = await createWorldStateSynchronizer(config, archiver, telemetry); // start both and wait for them to sync from the block source await Promise.all([p2pClient.start(), worldStateSynchronizer.start()]); @@ -199,7 +188,6 @@ export class AztecNodeService implements AztecNode { ethereumChain.chainInfo.id, config.version, new GlobalVariableBuilder(config), - store, proofVerifier, telemetry, log, @@ -726,13 +714,8 @@ export class AztecNodeService implements AztecNode { ); const prevHeader = (await this.blockSource.getBlock(-1))?.header; - // Instantiate merkle trees so uncommitted updates by this simulation are local to it. - // TODO we should be able to remove this after https://github.com/AztecProtocol/aztec-packages/issues/1869 - // So simulation of public functions doesn't affect the merkle trees. - const merkleTrees = await MerkleTrees.new(this.merkleTreesDb, new NoopTelemetryClient(), this.log); - const publicProcessorFactory = new PublicProcessorFactory( - merkleTrees.asLatest(), + await this.worldStateSynchronizer.ephemeralFork(), this.contractDataSource, new WASMSimulator(), this.telemetry, diff --git a/yarn-project/aztec/src/cli/cmds/start_archiver.ts b/yarn-project/aztec/src/cli/cmds/start_archiver.ts index a8675164ae1..d9e0c092aa7 100644 --- a/yarn-project/aztec/src/cli/cmds/start_archiver.ts +++ b/yarn-project/aztec/src/cli/cmds/start_archiver.ts @@ -21,8 +21,7 @@ export const startArchiver = async (options: any, signalHandlers: (() => Promise const archiverConfig = extractRelevantOptions(options, archiverConfigMappings, 'archiver'); const storeLog = createDebugLogger('aztec:archiver:lmdb'); - const rollupAddress = archiverConfig.l1Contracts.rollupAddress; - const store = await createStore(archiverConfig, rollupAddress, storeLog); + const store = await createStore('archiver', archiverConfig, storeLog); const archiverStore = new KVArchiverDataStore(store, archiverConfig.maxLogs); const telemetry = await createAndStartTelemetryClient(getTelemetryClientConfig()); diff --git a/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts b/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts index 2738dbc99e0..af16730d969 100644 --- a/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts +++ b/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts @@ -84,9 +84,7 @@ type LeafTypes = { export type MerkleTreeLeafType = LeafTypes[ID]; -/** - * Defines the interface for operations on a set of Merkle Trees. - */ +/** Defines the interface for operations on a set of Merkle Trees. */ export interface MerkleTreeOperations { /** * Appends leaves to a given tree. @@ -203,7 +201,10 @@ export interface MerkleTreeOperations { leaves: Buffer[], subtreeHeight: number, ): Promise>; +} +/** Operations on merkle trees world state that can modify the underlying store. */ +export interface MerkleTreeAdminOperations extends MerkleTreeOperations { /** * Handles a single L2 block (i.e. Inserts the new note hashes into the merkle tree). * @param block - The L2 block to handle. diff --git a/yarn-project/end-to-end/src/e2e_block_building.test.ts b/yarn-project/end-to-end/src/e2e_block_building.test.ts index 92f830001ad..cb4655f5561 100644 --- a/yarn-project/end-to-end/src/e2e_block_building.test.ts +++ b/yarn-project/end-to-end/src/e2e_block_building.test.ts @@ -339,7 +339,7 @@ describe('e2e_block_building', () => { }); // Regression for https://github.com/AztecProtocol/aztec-packages/issues/8306 - it.skip('can simulate public txs while building a block', async () => { + it('can simulate public txs while building a block', async () => { ({ teardown, pxe, @@ -368,7 +368,7 @@ describe('e2e_block_building', () => { } logger.info('Waiting for txs to be mined'); - await Promise.all(txs.map(tx => tx.wait())); + await Promise.all(txs.map(tx => tx.wait({ proven: false, timeout: 600 }))); }); }); }); diff --git a/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts b/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts index b75763aa6dc..e53fb312ec5 100644 --- a/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts +++ b/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts @@ -17,7 +17,6 @@ import { deployL1Contract, } from '@aztec/aztec.js'; import { BBCircuitVerifier } from '@aztec/bb-prover'; -import { createStore } from '@aztec/kv-store/utils'; import { RollupAbi } from '@aztec/l1-artifacts'; import { TokenContract } from '@aztec/noir-contracts.js'; import { type ProverNode, type ProverNodeConfig, createProverNode } from '@aztec/prover-node'; @@ -225,11 +224,8 @@ export class FullProverTest { // Creating temp store and archiver for fully proven prover node this.logger.verbose('Starting archiver for new prover node'); - const store = await createStore({ dataDirectory: undefined }, this.l1Contracts.l1ContractAddresses.rollupAddress); - const archiver = await createArchiver( { ...this.context.aztecNodeConfig, dataDirectory: undefined }, - store, new NoopTelemetryClient(), { blockUntilSync: true }, ); diff --git a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts index 6b821026179..dac9a054a04 100644 --- a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts +++ b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts @@ -8,7 +8,6 @@ import { type CompleteAddress, type DebugLogger, type DeployL1Contracts, - type EthAddress, EthCheatCodes, Fr, GrumpkinScalar, @@ -23,7 +22,6 @@ import { asyncMap } from '@aztec/foundation/async-map'; import { type Logger, createDebugLogger } from '@aztec/foundation/log'; import { makeBackoff, retry } from '@aztec/foundation/retry'; import { resolver, reviver } from '@aztec/foundation/serialize'; -import { createStore } from '@aztec/kv-store/utils'; import { type ProverNode, type ProverNodeConfig, createProverNode } from '@aztec/prover-node'; import { type PXEService, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; @@ -235,21 +233,13 @@ async function teardown(context: SubsystemsContext | undefined) { } export async function createAndSyncProverNode( - rollupAddress: EthAddress, proverNodePrivateKey: `0x${string}`, aztecNodeConfig: AztecNodeConfig, aztecNode: AztecNode, ) { // Creating temp store and archiver for simulated prover node - - const store = await createStore({ dataDirectory: undefined }, rollupAddress); - - const archiver = await createArchiver( - { ...aztecNodeConfig, dataDirectory: undefined }, - store, - new NoopTelemetryClient(), - { blockUntilSync: true }, - ); + const archiverConfig = { ...aztecNodeConfig, dataDirectory: undefined }; + const archiver = await createArchiver(archiverConfig, new NoopTelemetryClient(), { blockUntilSync: true }); // Prover node config is for simulated proofs const proverConfig: ProverNodeConfig = { @@ -335,7 +325,6 @@ async function setupFromFresh( logger.verbose('Creating and syncing a simulated prover node...'); const proverNode = await createAndSyncProverNode( - deployL1ContractsValues.l1ContractAddresses.rollupAddress, `0x${proverNodePrivateKey!.toString('hex')}`, aztecNodeConfig, aztecNode, @@ -433,12 +422,7 @@ async function setupFromState(statePath: string, logger: Logger): Promise { ? `0x${proverNodePrivateKey?.toString('hex')}` : proverConfig.publisherPrivateKey; - proverNode = await createAndSyncProverNode( - config.l1Contracts.rollupAddress, - proverConfig.publisherPrivateKey, - config, - aztecNode, - ); + proverNode = await createAndSyncProverNode(proverConfig.publisherPrivateKey, config, aztecNode); }, 600_000); afterEach(async () => { diff --git a/yarn-project/kv-store/src/utils.ts b/yarn-project/kv-store/src/utils.ts index 3c33d571e77..640f50932c3 100644 --- a/yarn-project/kv-store/src/utils.ts +++ b/yarn-project/kv-store/src/utils.ts @@ -1,17 +1,25 @@ import { type EthAddress } from '@aztec/foundation/eth-address'; import { type Logger, createDebugLogger } from '@aztec/foundation/log'; +import { join } from 'path'; + import { type AztecKVStore } from './interfaces/store.js'; import { AztecLmdbStore } from './lmdb/store.js'; -export function createStore( - config: { dataDirectory: string | undefined } | (string | undefined), - rollupAddress: EthAddress, - log: Logger = createDebugLogger('aztec:kv-store'), -) { - const dataDirectory = typeof config === 'string' ? config : config?.dataDirectory; - log.info(dataDirectory ? `Creating data store at directory ${dataDirectory}` : 'Creating ephemeral data store'); - return initStoreForRollup(AztecLmdbStore.open(dataDirectory, false), rollupAddress, log); +export type DataStoreConfig = { dataDirectory: string | undefined; l1Contracts: { rollupAddress: EthAddress } }; + +export function createStore(name: string, config: DataStoreConfig, log: Logger = createDebugLogger('aztec:kv-store')) { + let { dataDirectory } = config; + if (typeof dataDirectory !== 'undefined') { + dataDirectory = join(dataDirectory, name); + } + + log.info( + dataDirectory + ? `Creating ${name} data store at directory ${dataDirectory}` + : `Creating ${name} ephemeral data store`, + ); + return initStoreForRollup(AztecLmdbStore.open(dataDirectory, false), config.l1Contracts.rollupAddress, log); } /** @@ -21,7 +29,7 @@ export function createStore( * @param rollupAddress - The ETH address of the rollup contract * @returns A promise that resolves when the store is cleared, or rejects if the rollup address does not match */ -export async function initStoreForRollup( +async function initStoreForRollup( store: T, rollupAddress: EthAddress, log?: Logger, diff --git a/yarn-project/p2p/src/client/index.ts b/yarn-project/p2p/src/client/index.ts index d2316c4db76..518387edfa9 100644 --- a/yarn-project/p2p/src/client/index.ts +++ b/yarn-project/p2p/src/client/index.ts @@ -1,5 +1,9 @@ import { type L2BlockSource } from '@aztec/circuit-types'; +import { createDebugLogger } from '@aztec/foundation/log'; import { type AztecKVStore } from '@aztec/kv-store'; +import { type DataStoreConfig, createStore } from '@aztec/kv-store/utils'; +import { type TelemetryClient } from '@aztec/telemetry-client'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { type AttestationPool } from '../attestation_pool/attestation_pool.js'; import { P2PClient } from '../client/p2p_client.js'; @@ -7,18 +11,21 @@ import { type P2PConfig } from '../config.js'; import { DiscV5Service } from '../service/discV5_service.js'; import { DummyP2PService } from '../service/dummy_service.js'; import { LibP2PService, createLibP2PPeerId } from '../service/index.js'; -import { type TxPool } from '../tx_pool/index.js'; +import { AztecKVTxPool, type TxPool } from '../tx_pool/index.js'; import { getPublicIp, resolveAddressIfNecessary, splitAddressPort } from '../util.js'; export * from './p2p_client.js'; export const createP2PClient = async ( - config: P2PConfig, - store: AztecKVStore, - txPool: TxPool, + config: P2PConfig & DataStoreConfig, attestationsPool: AttestationPool, l2BlockSource: L2BlockSource, + telemetry: TelemetryClient = new NoopTelemetryClient(), + deps: { txPool?: TxPool; store?: AztecKVStore } = {}, ) => { + const store = deps.store ?? (await createStore('p2p', config, createDebugLogger('aztec:p2p:lmdb'))); + const txPool = deps.txPool ?? new AztecKVTxPool(store, telemetry); + let p2pService; if (config.p2pEnabled) { diff --git a/yarn-project/p2p/src/service/reqresp/p2p_client.integration.test.ts b/yarn-project/p2p/src/service/reqresp/p2p_client.integration.test.ts index 307324f783f..8ec358ee2f4 100644 --- a/yarn-project/p2p/src/service/reqresp/p2p_client.integration.test.ts +++ b/yarn-project/p2p/src/service/reqresp/p2p_client.integration.test.ts @@ -1,9 +1,10 @@ // An integration test for the p2p client to test req resp protocols import { mockTx } from '@aztec/circuit-types'; +import { EthAddress } from '@aztec/circuits.js'; import { createDebugLogger } from '@aztec/foundation/log'; import { sleep } from '@aztec/foundation/sleep'; import { type AztecKVStore } from '@aztec/kv-store'; -import { openTmpStore } from '@aztec/kv-store/utils'; +import { type DataStoreConfig, openTmpStore } from '@aztec/kv-store/utils'; import { describe, expect, it, jest } from '@jest/globals'; import { generatePrivateKey } from 'viem/accounts'; @@ -73,7 +74,7 @@ describe('Req Resp p2p client integration', () => { // Note these bindings are important const addr = `127.0.0.1:${i + 1 + BOOT_NODE_UDP_PORT}`; const listenAddr = `0.0.0.0:${i + 1 + BOOT_NODE_UDP_PORT}`; - const config: P2PConfig = { + const config: P2PConfig & DataStoreConfig = { p2pEnabled: true, peerIdPrivateKey: peerIdPrivateKeys[i], tcpListenAddress: listenAddr, // run on port 0 @@ -89,6 +90,8 @@ describe('Req Resp p2p client integration', () => { maxPeerCount: 10, keepProvenTxsInPoolFor: 0, queryForIp: false, + dataDirectory: undefined, + l1Contracts: { rollupAddress: EthAddress.ZERO }, }; txPool = { @@ -112,12 +115,16 @@ describe('Req Resp p2p client integration', () => { blockSource = new MockBlockSource(); kvStore = openTmpStore(); + const deps = { + txPool: txPool as unknown as TxPool, + store: kvStore, + }; const client = await createP2PClient( config, - kvStore, - txPool as unknown as TxPool, attestationPool as unknown as AttestationPool, blockSource, + undefined, + deps, ); await client.start(); diff --git a/yarn-project/prover-client/src/mocks/test_context.ts b/yarn-project/prover-client/src/mocks/test_context.ts index 46b4f30a68a..16e19745b0a 100644 --- a/yarn-project/prover-client/src/mocks/test_context.ts +++ b/yarn-project/prover-client/src/mocks/test_context.ts @@ -1,6 +1,7 @@ import { type BBProverConfig } from '@aztec/bb-prover'; import { type BlockProver, + type MerkleTreeAdminOperations, type ProcessedTx, type PublicExecutionRequest, type ServerCircuitProver, @@ -23,7 +24,7 @@ import { type WorldStatePublicDB, } from '@aztec/simulator'; import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; -import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; +import { MerkleTrees } from '@aztec/world-state'; import * as fs from 'fs/promises'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -42,7 +43,7 @@ export class TestContext { public publicProcessor: PublicProcessor, public simulationProvider: SimulationProvider, public globalVariables: GlobalVariables, - public actualDb: MerkleTreeOperations, + public actualDb: MerkleTreeAdminOperations, public prover: ServerCircuitProver, public proverAgent: ProverAgent, public orchestrator: ProvingOrchestrator, diff --git a/yarn-project/prover-node/src/factory.ts b/yarn-project/prover-node/src/factory.ts index 48045ef33da..7e9c31e8cbf 100644 --- a/yarn-project/prover-node/src/factory.ts +++ b/yarn-project/prover-node/src/factory.ts @@ -1,7 +1,6 @@ import { type Archiver, createArchiver } from '@aztec/archiver'; import { type AztecNode } from '@aztec/circuit-types'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; -import { createStore } from '@aztec/kv-store/utils'; import { createProverClient } from '@aztec/prover-client'; import { L1Publisher } from '@aztec/sequencer-client'; import { createSimulationProvider } from '@aztec/simulator'; @@ -20,22 +19,17 @@ export async function createProverNode( deps: { telemetry?: TelemetryClient; log?: DebugLogger; - storeLog?: DebugLogger; aztecNodeTxProvider?: AztecNode; archiver?: Archiver; } = {}, ) { const telemetry = deps.telemetry ?? new NoopTelemetryClient(); const log = deps.log ?? createDebugLogger('aztec:prover'); - const storeLog = deps.storeLog ?? createDebugLogger('aztec:prover:lmdb'); - - const store = await createStore(config, config.l1Contracts.rollupAddress, storeLog); - - const archiver = deps.archiver ?? (await createArchiver(config, store, telemetry, { blockUntilSync: true })); + const archiver = deps.archiver ?? (await createArchiver(config, telemetry, { blockUntilSync: true })); log.verbose(`Created archiver and synced to block ${await archiver.getBlockNumber()}`); const worldStateConfig = { ...config, worldStateProvenBlocksOnly: true }; - const worldStateSynchronizer = await createWorldStateSynchronizer(worldStateConfig, store, archiver, telemetry); + const worldStateSynchronizer = await createWorldStateSynchronizer(worldStateConfig, archiver, telemetry); await worldStateSynchronizer.start(); const simulationProvider = await createSimulationProvider(config, log); diff --git a/yarn-project/prover-node/src/prover-node.test.ts b/yarn-project/prover-node/src/prover-node.test.ts index 97fdda86c1b..b8330eff75c 100644 --- a/yarn-project/prover-node/src/prover-node.test.ts +++ b/yarn-project/prover-node/src/prover-node.test.ts @@ -1,7 +1,7 @@ import { type L1ToL2MessageSource, type L2BlockSource, - type MerkleTreeOperations, + type MerkleTreeAdminOperations, type ProverClient, type TxProvider, } from '@aztec/circuit-types'; @@ -32,7 +32,7 @@ describe('prover-node', () => { let jobs: { job: MockProxy; cleanUp: (job: BlockProvingJob) => Promise; - db: MerkleTreeOperations; + db: MerkleTreeAdminOperations; }[]; beforeEach(() => { @@ -47,7 +47,7 @@ describe('prover-node', () => { const telemetryClient = new NoopTelemetryClient(); // World state returns a new mock db every time it is asked to fork - worldState.syncImmediateAndFork.mockImplementation(() => Promise.resolve(mock())); + worldState.syncImmediateAndFork.mockImplementation(() => Promise.resolve(mock())); jobs = []; proverNode = new TestProverNode( @@ -161,7 +161,7 @@ describe('prover-node', () => { class TestProverNode extends ProverNode { protected override doCreateBlockProvingJob( - db: MerkleTreeOperations, + db: MerkleTreeAdminOperations, _publicProcessorFactory: PublicProcessorFactory, cleanUp: (job: BlockProvingJob) => Promise, ): BlockProvingJob { diff --git a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts index 30d96288cca..e50e14f5339 100644 --- a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts @@ -11,8 +11,6 @@ import { getCanonicalInstanceDeployer } from '@aztec/protocol-contracts/instance import { getCanonicalKeyRegistry } from '@aztec/protocol-contracts/key-registry'; import { getCanonicalMultiCallEntrypointContract } from '@aztec/protocol-contracts/multi-call-entrypoint'; -import { join } from 'path'; - import { type PXEServiceConfig } from '../config/index.js'; import { KVPxeDatabase } from '../database/kv_pxe_database.js'; import { TestPrivateKernelProver } from '../kernel_prover/test/test_circuit_prover.js'; @@ -38,12 +36,12 @@ export async function createPXEService( const logSuffix = typeof useLogSuffix === 'boolean' ? (useLogSuffix ? randomBytes(3).toString('hex') : undefined) : useLogSuffix; - const pxeDbPath = config.dataDirectory ? join(config.dataDirectory, 'pxe_data') : undefined; - const keyStorePath = config.dataDirectory ? join(config.dataDirectory, 'pxe_key_store') : undefined; const l1Contracts = await aztecNode.getL1ContractAddresses(); - - const keyStore = new KeyStore(await createStore(keyStorePath, l1Contracts.rollupAddress)); - const db = new KVPxeDatabase(await createStore(pxeDbPath, l1Contracts.rollupAddress)); + const storeConfig = { dataDirectory: config.dataDirectory, l1Contracts }; + const keyStore = new KeyStore( + await createStore('pxe_key_store', storeConfig, createDebugLogger('aztec:pxe:keystore:lmdb')), + ); + const db = new KVPxeDatabase(await createStore('pxe_data', storeConfig, createDebugLogger('aztec:pxe:data:lmdb'))); const prover = proofCreator ?? (await createProver(config, logSuffix)); const server = new PXEService(keyStore, aztecNode, db, prover, config, logSuffix); diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts index d3a8ec1fc82..462d6687dad 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts @@ -5,6 +5,7 @@ import { type L1ToL2MessageSource, L2Block, type L2BlockSource, + type MerkleTreeAdminOperations, MerkleTreeId, PROVING_STATUS, type ProvingSuccess, @@ -34,7 +35,7 @@ import { type PublicProcessor, type PublicProcessorFactory } from '@aztec/simula import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { type ContractDataSource } from '@aztec/types/contracts'; import { type ValidatorClient } from '@aztec/validator-client'; -import { type MerkleTreeOperations, WorldStateRunningState, type WorldStateSynchronizer } from '@aztec/world-state'; +import { WorldStateRunningState, type WorldStateSynchronizer } from '@aztec/world-state'; import { type MockProxy, mock, mockFn } from 'jest-mock-extended'; @@ -51,7 +52,7 @@ describe('sequencer', () => { let p2p: MockProxy; let worldState: MockProxy; let blockSimulator: MockProxy; - let merkleTreeOps: MockProxy; + let merkleTreeOps: MockProxy; let publicProcessor: MockProxy; let l2BlockSource: MockProxy; let l1ToL2MessageSource: MockProxy; @@ -112,7 +113,7 @@ describe('sequencer', () => { publisher.validateBlockForSubmission.mockResolvedValue(); globalVariableBuilder = mock(); - merkleTreeOps = mock(); + merkleTreeOps = mock(); blockSimulator = mock(); p2p = mock({ diff --git a/yarn-project/world-state/src/synchronizer/factory.ts b/yarn-project/world-state/src/synchronizer/factory.ts index e2d37d4a542..323fe6382c2 100644 --- a/yarn-project/world-state/src/synchronizer/factory.ts +++ b/yarn-project/world-state/src/synchronizer/factory.ts @@ -1,5 +1,6 @@ import { type L1ToL2MessageSource, type L2BlockSource } from '@aztec/circuit-types'; -import { type AztecKVStore } from '@aztec/kv-store'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { type DataStoreConfig, createStore } from '@aztec/kv-store/utils'; import { type TelemetryClient } from '@aztec/telemetry-client'; import { MerkleTrees } from '../world-state-db/merkle_trees.js'; @@ -7,11 +8,11 @@ import { type WorldStateConfig } from './config.js'; import { ServerWorldStateSynchronizer } from './server_world_state_synchronizer.js'; export async function createWorldStateSynchronizer( - config: WorldStateConfig, - store: AztecKVStore, + config: WorldStateConfig & DataStoreConfig, l2BlockSource: L2BlockSource & L1ToL2MessageSource, client: TelemetryClient, ) { + const store = await createStore('world-state', config, createDebugLogger('aztec:world-state:lmdb')); const merkleTrees = await MerkleTrees.new(store, client); return new ServerWorldStateSynchronizer(store, merkleTrees, l2BlockSource, config); } diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts index 7e585c12ae7..f46aa23879d 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts @@ -11,7 +11,7 @@ import { INITIAL_LEAF, Pedersen, SHA256Trunc, StandardTree } from '@aztec/merkle import { jest } from '@jest/globals'; import { mock } from 'jest-mock-extended'; -import { type MerkleTreeDb, type MerkleTrees, type WorldStateConfig } from '../index.js'; +import { type MerkleTreeAdminDb, type MerkleTrees, type WorldStateConfig } from '../index.js'; import { ServerWorldStateSynchronizer } from './server_world_state_synchronizer.js'; import { WorldStateRunningState } from './world_state_synchronizer.js'; @@ -39,7 +39,7 @@ describe('server_world_state_synchronizer', () => { getL1ToL2Messages: jest.fn(() => Promise.resolve(l1ToL2Messages)), }); - const merkleTreeDb = mock({ + const merkleTreeDb = mock({ getTreeInfo: jest.fn(() => Promise.resolve({ depth: 8, treeId: MerkleTreeId.NOTE_HASH_TREE, root: Buffer.alloc(32, 0), size: 0n }), ), diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts index a4512b41048..ce188d1eaa1 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts @@ -4,6 +4,7 @@ import { type L2Block, L2BlockDownloader, type L2BlockSource, + type MerkleTreeAdminOperations, } from '@aztec/circuit-types'; import { type L2BlockHandledStats } from '@aztec/circuit-types/stats'; import { L1_TO_L2_MSG_SUBTREE_HEIGHT } from '@aztec/circuits.js/constants'; @@ -16,8 +17,11 @@ import { type AztecKVStore, type AztecSingleton } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/utils'; import { SHA256Trunc, StandardTree } from '@aztec/merkle-tree'; -import { type MerkleTreeOperations, type MerkleTrees } from '../world-state-db/index.js'; -import { MerkleTreeOperationsFacade } from '../world-state-db/merkle_tree_operations_facade.js'; +import { type MerkleTrees } from '../world-state-db/index.js'; +import { + MerkleTreeAdminOperationsFacade, + MerkleTreeOperationsFacade, +} from '../world-state-db/merkle_tree_operations_facade.js'; import { MerkleTreeSnapshotOperationsFacade } from '../world-state-db/merkle_tree_snapshot_operations_facade.js'; import { type WorldStateConfig } from './config.js'; import { @@ -62,21 +66,25 @@ export class ServerWorldStateSynchronizer implements WorldStateSynchronizer { }); } - public getLatest(): MerkleTreeOperations { - return new MerkleTreeOperationsFacade(this.merkleTreeDb, true); + public getLatest(): MerkleTreeAdminOperations { + return new MerkleTreeAdminOperationsFacade(this.merkleTreeDb, true); } - public getCommitted(): MerkleTreeOperations { - return new MerkleTreeOperationsFacade(this.merkleTreeDb, false); + public getCommitted(): MerkleTreeAdminOperations { + return new MerkleTreeAdminOperationsFacade(this.merkleTreeDb, false); } - public getSnapshot(blockNumber: number): MerkleTreeOperations { + public getSnapshot(blockNumber: number): MerkleTreeAdminOperations { return new MerkleTreeSnapshotOperationsFacade(this.merkleTreeDb, blockNumber); } - private async getFork(includeUncommitted: boolean): Promise { + public async ephemeralFork(): Promise { + return new MerkleTreeOperationsFacade(await this.merkleTreeDb.ephemeralFork(), true); + } + + private async getFork(includeUncommitted: boolean): Promise { this.log.verbose(`Forking world state at ${this.blockNumber.get()}`); - return new MerkleTreeOperationsFacade(await this.merkleTreeDb.fork(), includeUncommitted); + return new MerkleTreeAdminOperationsFacade(await this.merkleTreeDb.fork(), includeUncommitted); } public async start() { @@ -212,7 +220,7 @@ export class ServerWorldStateSynchronizer implements WorldStateSynchronizer { public async syncImmediateAndFork( targetBlockNumber: number, forkIncludeUncommitted: boolean, - ): Promise { + ): Promise { try { await this.pause(); await this.syncImmediate(targetBlockNumber); diff --git a/yarn-project/world-state/src/synchronizer/world_state_synchronizer.ts b/yarn-project/world-state/src/synchronizer/world_state_synchronizer.ts index 0411827e10d..0fb17dbe508 100644 --- a/yarn-project/world-state/src/synchronizer/world_state_synchronizer.ts +++ b/yarn-project/world-state/src/synchronizer/world_state_synchronizer.ts @@ -1,4 +1,4 @@ -import { type MerkleTreeOperations } from '../world-state-db/index.js'; +import { type MerkleTreeAdminOperations, type MerkleTreeOperations } from '../world-state-db/index.js'; /** * Defines the possible states of the world state synchronizer. @@ -58,24 +58,26 @@ export interface WorldStateSynchronizer { * @param forkIncludeUncommitted - Whether to include uncommitted data in the fork. * @returns The db forked at the requested target block number. */ - syncImmediateAndFork(targetBlockNumber: number, forkIncludeUncommitted: boolean): Promise; + syncImmediateAndFork(targetBlockNumber: number, forkIncludeUncommitted: boolean): Promise; /** - * Returns an instance of MerkleTreeOperations that will include uncommitted data. - * @returns An instance of MerkleTreeOperations that will include uncommitted data. + * Forks the current in-memory state based off the current committed state, and returns an instance that cannot modify the underlying data store. */ - getLatest(): MerkleTreeOperations; + ephemeralFork(): Promise; /** - * Returns an instance of MerkleTreeOperations that will not include uncommitted data. - * @returns An instance of MerkleTreeOperations that will not include uncommitted data. + * Returns an instance of MerkleTreeAdminOperations that will include uncommitted data. */ - getCommitted(): MerkleTreeOperations; + getLatest(): MerkleTreeAdminOperations; /** - * Returns a readonly instance of MerkleTreeOperations where the state is as it was at the given block number + * Returns an instance of MerkleTreeAdminOperations that will not include uncommitted data. + */ + getCommitted(): MerkleTreeAdminOperations; + + /** + * Returns a readonly instance of MerkleTreeAdminOperations where the state is as it was at the given block number * @param block - The block number to look at - * @returns An instance of MerkleTreeOperations */ getSnapshot(block: number): MerkleTreeOperations; } diff --git a/yarn-project/world-state/src/world-state-db/index.ts b/yarn-project/world-state/src/world-state-db/index.ts index 63ec2e7ba65..44e00549965 100644 --- a/yarn-project/world-state/src/world-state-db/index.ts +++ b/yarn-project/world-state/src/world-state-db/index.ts @@ -3,4 +3,4 @@ export * from './merkle_tree_db.js'; export * from './merkle_tree_operations_facade.js'; export * from './merkle_tree_snapshot_operations_facade.js'; -export { MerkleTreeOperations } from '@aztec/circuit-types/interfaces'; +export { MerkleTreeOperations, MerkleTreeAdminOperations } from '@aztec/circuit-types/interfaces'; diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts index 6471a3e6e03..bf4d97e3924 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts @@ -1,5 +1,5 @@ import { type MerkleTreeId } from '@aztec/circuit-types'; -import { type MerkleTreeOperations } from '@aztec/circuit-types/interfaces'; +import { type MerkleTreeAdminOperations, type MerkleTreeOperations } from '@aztec/circuit-types/interfaces'; import { type Fr, MAX_NULLIFIERS_PER_TX, MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX } from '@aztec/circuits.js'; import { type IndexedTreeSnapshot, type TreeSnapshot } from '@aztec/merkle-tree'; @@ -32,13 +32,9 @@ type WithIncludeUncommitted = F extends (...args: [...infer Rest]) => infer R /** * Defines the names of the setters on Merkle Trees. */ -type MerkleTreeSetters = - | 'appendLeaves' - | 'updateLeaf' - | 'commit' - | 'rollback' - | 'handleL2BlockAndMessages' - | 'batchInsert'; +type MerkleTreeSetters = 'appendLeaves' | 'updateLeaf' | 'batchInsert'; + +type MerkleTreeAdmin = 'commit' | 'rollback' | 'handleL2BlockAndMessages'; export type TreeSnapshots = { [MerkleTreeId.NULLIFIER_TREE]: IndexedTreeSnapshot; @@ -48,9 +44,7 @@ export type TreeSnapshots = { [MerkleTreeId.ARCHIVE]: TreeSnapshot; }; -/** - * Defines the interface for operations on a set of Merkle Trees configuring whether to return committed or uncommitted data. - */ +/** Defines the interface for operations on a set of Merkle Trees configuring whether to return committed or uncommitted data. */ export type MerkleTreeDb = { [Property in keyof MerkleTreeOperations as Exclude]: WithIncludeUncommitted< MerkleTreeOperations[Property] @@ -61,6 +55,20 @@ export type MerkleTreeDb = { * @param block - The block number to take the snapshot at. */ getSnapshot(block: number): Promise; + }; + +/** Extends operations on MerkleTreeDb to include modifying the underlying store */ +export type MerkleTreeAdminDb = { + [Property in keyof MerkleTreeAdminOperations as Exclude< + Property, + MerkleTreeSetters | MerkleTreeAdmin + >]: WithIncludeUncommitted; +} & Pick & { + /** + * Returns a snapshot of the current state of the trees. + * @param block - The block number to take the snapshot at. + */ + getSnapshot(block: number): Promise; /** * Forks the database at its current state. diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts index b96fc441178..39975c30551 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts @@ -2,6 +2,7 @@ import { type BatchInsertionResult, type L2Block, type MerkleTreeId, type Siblin import { type HandleL2BlockAndMessagesResult, type IndexedTreeId, + type MerkleTreeAdminOperations, type MerkleTreeLeafType, type MerkleTreeOperations, type TreeInfo, @@ -9,13 +10,13 @@ import { import { type Fr, type Header, type NullifierLeafPreimage, type StateReference } from '@aztec/circuits.js'; import { type IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; -import { type MerkleTreeDb } from './merkle_tree_db.js'; +import { type MerkleTreeAdminDb, type MerkleTreeDb } from './merkle_tree_db.js'; /** * Wraps a MerkleTreeDbOperations to call all functions with a preset includeUncommitted flag. */ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { - constructor(private trees: MerkleTreeDb, private includeUncommitted: boolean) {} + constructor(protected trees: MerkleTreeDb, protected includeUncommitted: boolean) {} /** * Returns the tree info for the specified tree id. @@ -164,6 +165,27 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { return this.trees.updateArchive(header, this.includeUncommitted); } + /** + * Batch insert multiple leaves into the tree. + * @param treeId - The ID of the tree. + * @param leaves - Leaves to insert into the tree. + * @param subtreeHeight - Height of the subtree. + * @returns The data for the leaves to be updated when inserting the new ones. + */ + public batchInsert( + treeId: IndexedTreeId, + leaves: Buffer[], + subtreeHeight: number, + ): Promise> { + return this.trees.batchInsert(treeId, leaves, subtreeHeight); + } +} + +export class MerkleTreeAdminOperationsFacade extends MerkleTreeOperationsFacade implements MerkleTreeAdminOperations { + constructor(protected override trees: MerkleTreeAdminDb, includeUncommitted: boolean) { + super(trees, includeUncommitted); + } + /** * Handles a single L2 block (i.e. Inserts the new note hashes into the merkle tree). * @param block - The L2 block to handle. @@ -190,21 +212,6 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { return await this.trees.rollback(); } - /** - * Batch insert multiple leaves into the tree. - * @param treeId - The ID of the tree. - * @param leaves - Leaves to insert into the tree. - * @param subtreeHeight - Height of the subtree. - * @returns The data for the leaves to be updated when inserting the new ones. - */ - public batchInsert( - treeId: IndexedTreeId, - leaves: Buffer[], - subtreeHeight: number, - ): Promise> { - return this.trees.batchInsert(treeId, leaves, subtreeHeight); - } - public delete(): Promise { return this.trees.delete(); } diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index 1c9bd5b6877..f94cbec7dde 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -3,8 +3,8 @@ import { type BatchInsertionResult, type HandleL2BlockAndMessagesResult, type IndexedTreeId, + type MerkleTreeAdminOperations, type MerkleTreeLeafType, - type MerkleTreeOperations, type TreeInfo, } from '@aztec/circuit-types/interfaces'; import { @@ -56,7 +56,7 @@ import { type TreeSnapshots, } from './merkle_tree_db.js'; import { type MerkleTreeMap } from './merkle_tree_map.js'; -import { MerkleTreeOperationsFacade } from './merkle_tree_operations_facade.js'; +import { MerkleTreeAdminOperationsFacade } from './merkle_tree_operations_facade.js'; import { WorldStateMetrics } from './metrics.js'; /** @@ -122,8 +122,8 @@ export class MerkleTrees implements MerkleTreeDb { /** * Initializes the collection of Merkle Trees. */ - async #init() { - const fromDb = this.#isDbPopulated(); + async #init(loadFromDb?: boolean) { + const fromDb = loadFromDb === undefined ? this.#isDbPopulated() : loadFromDb; const initializeTree = fromDb ? loadTree : newTree; const hasher = new Poseidon(); @@ -180,17 +180,14 @@ export class MerkleTrees implements MerkleTreeDb { const initialState = await this.getStateReference(true); await this.#saveInitialStateReference(initialState); await this.#updateArchive(this.getInitialHeader(), true); - } - await this.#commit(); + // And commit anything we did to initialize this set of trees + await this.#commit(); + } } public async fork(): Promise { const [ms, db] = await elapsed(async () => { - // TODO(palla/prover-node): If the underlying store is being shared with other components, we're unnecessarily - // copying a lot of data unrelated to merkle trees. This may be fine for now, and we may be able to ditch backup-based - // forking in favor of a more elegant proposal. But if we see this operation starts taking a lot of time, we may want - // to open separate stores for merkle trees and other components. const forked = await this.store.fork(); return MerkleTrees.new(forked, this.telemetryClient, this.log); }); @@ -199,6 +196,20 @@ export class MerkleTrees implements MerkleTreeDb { return db; } + // REFACTOR: We're hiding the `commit` operations in the tree behind a type check only, but + // we should make sure it's not accidentally called elsewhere by splitting this class into one + // that can work on a read-only store and one that actually writes to the store. This implies + // having read-only versions of the kv-stores, all kv-containers, and all trees. + public async ephemeralFork(): Promise { + const forked = new MerkleTrees( + this.store, + this.telemetryClient, + createDebugLogger('aztec:merkle_trees:ephemeral_fork'), + ); + await forked.#init(true); + return forked; + } + public async delete() { await this.store.delete(); } @@ -218,16 +229,16 @@ export class MerkleTrees implements MerkleTreeDb { * Gets a view of this db that returns uncommitted data. * @returns - A facade for this instance. */ - public asLatest(): MerkleTreeOperations { - return new MerkleTreeOperationsFacade(this, true); + public asLatest(): MerkleTreeAdminOperations { + return new MerkleTreeAdminOperationsFacade(this, true); } /** * Gets a view of this db that returns committed data only. * @returns - A facade for this instance. */ - public asCommitted(): MerkleTreeOperations { - return new MerkleTreeOperationsFacade(this, false); + public asCommitted(): MerkleTreeAdminOperations { + return new MerkleTreeAdminOperationsFacade(this, false); } /**