From 2218f38e4c93acc45b66654bae2b3a3c208e2a84 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 24 Apr 2024 09:14:15 +0000 Subject: [PATCH 1/3] feat: configure prover as separate process --- .../aztec-node/src/aztec-node/config.ts | 14 +++- .../aztec-node/src/aztec-node/server.ts | 7 +- yarn-project/aztec/package.json | 1 + yarn-project/aztec/src/cli/cli.ts | 3 + yarn-project/aztec/src/cli/cmds/start_node.ts | 9 +++ .../aztec/src/cli/cmds/start_prover.ts | 50 +++++++++++++ yarn-project/aztec/src/cli/util.ts | 5 ++ yarn-project/aztec/tsconfig.json | 3 + .../src/interfaces/prover-client.ts | 3 + .../kernel/kernel_circuit_public_inputs.ts | 8 ++ .../public_kernel_circuit_public_inputs.ts | 8 ++ ...kernel_tail_circuit_private_inputs.test.ts | 7 ++ ...blic_kernel_tail_circuit_private_inputs.ts | 8 ++ .../composed/integration_l1_publisher.test.ts | 3 +- yarn-project/prover-client/package.json | 5 +- .../prover-client/src/dummy-prover.ts | 23 ++++++ .../prover-client/src/prover-pool/index.ts | 1 + .../src/prover-pool/prover-agent.ts | 3 +- .../src/prover-pool/prover-pool.ts | 33 ++++++++- .../src/prover-pool/proving-error.ts | 9 +++ .../prover-client/src/prover-pool/rpc.ts | 73 +++++++++++++++++++ .../src/prover/test_circuit_prover.ts | 5 +- .../prover-client/src/tx-prover/tx-prover.ts | 39 ++++------ yarn-project/yarn.lock | 1 + 24 files changed, 287 insertions(+), 34 deletions(-) create mode 100644 yarn-project/aztec/src/cli/cmds/start_prover.ts create mode 100644 yarn-project/prover-client/src/prover-pool/proving-error.ts create mode 100644 yarn-project/prover-client/src/prover-pool/rpc.ts diff --git a/yarn-project/aztec-node/src/aztec-node/config.ts b/yarn-project/aztec-node/src/aztec-node/config.ts index 125ec88b287..4b644ca3b78 100644 --- a/yarn-project/aztec-node/src/aztec-node/config.ts +++ b/yarn-project/aztec-node/src/aztec-node/config.ts @@ -17,6 +17,8 @@ export type AztecNodeConfig = ArchiverConfig & /** A URL for an archiver service that the node will use. */ archiverUrl?: string; + + proverAgents: number; }; /** @@ -24,15 +26,21 @@ export type AztecNodeConfig = ArchiverConfig & * @returns A valid aztec node config. */ export function getConfigEnvVars(): AztecNodeConfig { - const { SEQ_DISABLED, PROVER_DISABLED } = process.env; + const { SEQ_DISABLED, PROVER_DISABLED, ARCHIVER_URL, PROVER_AGENTS = '1' } = process.env; + let proverAgents = parseInt(PROVER_AGENTS, 10); + if (Number.isNaN(proverAgents)) { + proverAgents = 1; + } + const allEnvVars: AztecNodeConfig = { ...getSequencerVars(), ...getArchiverVars(), ...getP2PConfigEnvVars(), ...getWorldStateVars(), disableSequencer: !!SEQ_DISABLED, - disableProver: !!PROVER_DISABLED, - archiverUrl: process.env.ARCHIVER_URL, + archiverUrl: ARCHIVER_URL, + disableProver: PROVER_DISABLED === '1', + proverAgents, }; return allEnvVars; diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 5a57d935a31..7ec441d29a4 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -52,6 +52,7 @@ import { initStoreForRollup, openTmpStore } from '@aztec/kv-store/utils'; import { SHA256Trunc, StandardTree } from '@aztec/merkle-tree'; import { AztecKVTxPool, type P2P, createP2PClient } from '@aztec/p2p'; import { DummyProver, TxProver } from '@aztec/prover-client'; +import { ProverPool } from '@aztec/prover-client/prover-pool'; import { type GlobalVariableBuilder, SequencerClient, getGlobalVariableBuilder } from '@aztec/sequencer-client'; import { PublicProcessorFactory, WASMSimulator } from '@aztec/simulator'; import { @@ -151,7 +152,7 @@ export class AztecNodeService implements AztecNode { const simulationProvider = await getSimulationProvider(config, log); const prover = config.disableProver ? await DummyProver.new() - : await TxProver.new(config, worldStateSynchronizer, simulationProvider); + : await TxProver.new(worldStateSynchronizer, ProverPool.testPool(simulationProvider, config.proverAgents)); // now create the sequencer const sequencer = config.disableSequencer @@ -194,6 +195,10 @@ export class AztecNodeService implements AztecNode { return this.sequencer; } + public getProver(): ProverClient { + return this.prover; + } + /** * Method to return the currently deployed L1 contract addresses. * @returns - The currently deployed L1 contract addresses. diff --git a/yarn-project/aztec/package.json b/yarn-project/aztec/package.json index d9f79c69f3b..9f91f14b3ea 100644 --- a/yarn-project/aztec/package.json +++ b/yarn-project/aztec/package.json @@ -42,6 +42,7 @@ "@aztec/noir-contracts.js": "workspace:^", "@aztec/p2p": "workspace:^", "@aztec/protocol-contracts": "workspace:^", + "@aztec/prover-client": "workspace:^", "@aztec/pxe": "workspace:^", "abitype": "^0.8.11", "commander": "^11.1.0", diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts index 525b3e5d741..ec6a48eb709 100644 --- a/yarn-project/aztec/src/cli/cli.ts +++ b/yarn-project/aztec/src/cli/cli.ts @@ -56,6 +56,9 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command { } else if (options.p2pBootstrap) { const { startP2PBootstrap } = await import('./cmds/start_p2p_bootstrap.js'); await startP2PBootstrap(options, signalHandlers, userLog, debugLogger); + } else if (options.prover) { + const { startProver } = await import('./cmds/start_prover.js'); + services = await startProver(options, signalHandlers, userLog); } if (services.length) { const rpcServer = createNamespacedJsonRpcServer(services, debugLogger); diff --git a/yarn-project/aztec/src/cli/cmds/start_node.ts b/yarn-project/aztec/src/cli/cmds/start_node.ts index 63a5ab7aa9b..35fb127f607 100644 --- a/yarn-project/aztec/src/cli/cmds/start_node.ts +++ b/yarn-project/aztec/src/cli/cmds/start_node.ts @@ -6,6 +6,7 @@ import { import { NULL_KEY } from '@aztec/ethereum'; import { type ServerList } from '@aztec/foundation/json-rpc/server'; import { type LogFn } from '@aztec/foundation/log'; +import { createProvingJobSourceServer } from '@aztec/prover-client/prover-pool'; import { type PXEServiceConfig, createPXERpcServer, getPXEServiceConfig } from '@aztec/pxe'; import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; @@ -64,7 +65,10 @@ export const startNode = async ( } if (!options.prover) { + userLog(`Prover is disabled, using mocked proofs`); nodeConfig.disableProver = true; + } else { + nodeConfig = mergeEnvVarsAndCliOptions(nodeConfig, parseModuleOptions(options.prover)); } if (!nodeConfig.disableSequencer && nodeConfig.disableProver) { @@ -78,6 +82,11 @@ export const startNode = async ( // Add node to services list services.push({ node: nodeServer }); + if (!nodeConfig.disableProver) { + const provingJobSource = createProvingJobSourceServer(node.getProver().getProvingJobSource()); + services.push({ provingJobSource }); + } + // Add node stop function to signal handlers signalHandlers.push(node.stop); diff --git a/yarn-project/aztec/src/cli/cmds/start_prover.ts b/yarn-project/aztec/src/cli/cmds/start_prover.ts new file mode 100644 index 00000000000..103ca97c8df --- /dev/null +++ b/yarn-project/aztec/src/cli/cmds/start_prover.ts @@ -0,0 +1,50 @@ +import { type ProvingJobSource } from '@aztec/circuit-types'; +import { ProverPool, createProvingJobSourceClient } from '@aztec/prover-client/prover-pool'; + +import { type ServiceStarter, parseModuleOptions } from '../util.js'; + +type ProverOptions = Partial<{ + proverUrl: string; + agents: string; + acvmBinaryPath?: string; + bbBinaryPath?: string; + simulate?: string; +}>; + +export const startProver: ServiceStarter = async (options, signalHandlers, logger) => { + const proverOptions: ProverOptions = parseModuleOptions(options.prover); + let source: ProvingJobSource; + + if (typeof proverOptions.proverUrl === 'string') { + logger(`Connecting to prover at ${proverOptions.proverUrl}`); + source = createProvingJobSourceClient(proverOptions.proverUrl, 'provingJobSource'); + } else { + throw new Error('Starting prover without an orchestrator is not supported'); + } + + const agentCount = proverOptions.agents ? parseInt(proverOptions.agents, 10) : 1; + if (agentCount === 0 || !Number.isSafeInteger(agentCount)) { + throw new Error('Cannot start prover without agents'); + } + + let pool: ProverPool; + if (proverOptions.simulate) { + pool = ProverPool.testPool(undefined, agentCount); + } else if (proverOptions.acvmBinaryPath && proverOptions.bbBinaryPath) { + pool = ProverPool.nativePool( + { + acvmBinaryPath: proverOptions.acvmBinaryPath, + bbBinaryPath: proverOptions.bbBinaryPath, + }, + agentCount, + ); + } else { + throw new Error('Cannot start prover without simulation or native prover options'); + } + + logger(`Starting ${agentCount} prover agents`); + await pool.start(source); + signalHandlers.push(() => pool.stop()); + + return Promise.resolve([]); +}; diff --git a/yarn-project/aztec/src/cli/util.ts b/yarn-project/aztec/src/cli/util.ts index 1a14868638b..db16f546c7b 100644 --- a/yarn-project/aztec/src/cli/util.ts +++ b/yarn-project/aztec/src/cli/util.ts @@ -3,10 +3,15 @@ import { type AztecNodeConfig } from '@aztec/aztec-node'; import { type AccountManager, type Fr } from '@aztec/aztec.js'; import { type L1ContractAddresses, l1ContractsNames } from '@aztec/ethereum'; import { EthAddress } from '@aztec/foundation/eth-address'; +import { type ServerList } from '@aztec/foundation/json-rpc/server'; import { type LogFn, createConsoleLogger } from '@aztec/foundation/log'; import { type P2PConfig } from '@aztec/p2p'; import { type PXEService, type PXEServiceConfig } from '@aztec/pxe'; +export interface ServiceStarter { + (options: T, signalHandlers: (() => Promise)[], logger: LogFn): Promise; +} + /** * Checks if the object has l1Contracts property * @param obj - The object to check diff --git a/yarn-project/aztec/tsconfig.json b/yarn-project/aztec/tsconfig.json index 457e0776787..bc45846794b 100644 --- a/yarn-project/aztec/tsconfig.json +++ b/yarn-project/aztec/tsconfig.json @@ -51,6 +51,9 @@ { "path": "../protocol-contracts" }, + { + "path": "../prover-client" + }, { "path": "../pxe" } diff --git a/yarn-project/circuit-types/src/interfaces/prover-client.ts b/yarn-project/circuit-types/src/interfaces/prover-client.ts index 5c2ef55b8d5..8e55d3a2dbb 100644 --- a/yarn-project/circuit-types/src/interfaces/prover-client.ts +++ b/yarn-project/circuit-types/src/interfaces/prover-client.ts @@ -1,4 +1,5 @@ import { type BlockProver } from './block-prover.js'; +import { type ProvingJobSource } from './proving-job.js'; /** * The interface to the prover client. @@ -8,4 +9,6 @@ export interface ProverClient extends BlockProver { start(): Promise; stop(): Promise; + + getProvingJobSource(): ProvingJobSource; } diff --git a/yarn-project/circuits.js/src/structs/kernel/kernel_circuit_public_inputs.ts b/yarn-project/circuits.js/src/structs/kernel/kernel_circuit_public_inputs.ts index 84637a61f91..8c3f4ad3eb0 100644 --- a/yarn-project/circuits.js/src/structs/kernel/kernel_circuit_public_inputs.ts +++ b/yarn-project/circuits.js/src/structs/kernel/kernel_circuit_public_inputs.ts @@ -78,4 +78,12 @@ export class KernelCircuitPublicInputs { RevertCode.OK, ); } + + toString() { + return this.toBuffer().toString('hex'); + } + + static fromString(str: string) { + return KernelCircuitPublicInputs.fromBuffer(Buffer.from(str, 'hex')); + } } diff --git a/yarn-project/circuits.js/src/structs/kernel/public_kernel_circuit_public_inputs.ts b/yarn-project/circuits.js/src/structs/kernel/public_kernel_circuit_public_inputs.ts index 207a1db74d5..87e2e9545e9 100644 --- a/yarn-project/circuits.js/src/structs/kernel/public_kernel_circuit_public_inputs.ts +++ b/yarn-project/circuits.js/src/structs/kernel/public_kernel_circuit_public_inputs.ts @@ -51,6 +51,14 @@ export class PublicKernelCircuitPublicInputs { ); } + toString() { + return this.toBuffer().toString('hex'); + } + + static fromString(str: string) { + return PublicKernelCircuitPublicInputs.fromBuffer(Buffer.from(str, 'hex')); + } + get needsSetup() { return !this.endNonRevertibleData.publicCallStack[1].isEmpty(); } diff --git a/yarn-project/circuits.js/src/structs/kernel/public_kernel_tail_circuit_private_inputs.test.ts b/yarn-project/circuits.js/src/structs/kernel/public_kernel_tail_circuit_private_inputs.test.ts index e53f3f2df21..3e2767cbda3 100644 --- a/yarn-project/circuits.js/src/structs/kernel/public_kernel_tail_circuit_private_inputs.test.ts +++ b/yarn-project/circuits.js/src/structs/kernel/public_kernel_tail_circuit_private_inputs.test.ts @@ -14,4 +14,11 @@ describe('PublicKernelTailCircuitPrivateInputs', () => { expect(original).toEqual(serialized); expect(original).not.toBe(serialized); }); + + it('serializes to string and back', () => { + const original = makePublicKernelTailCircuitPrivateInputs(123); + const str = original.toString(); + const deserialized = PublicKernelTailCircuitPrivateInputs.fromString(str); + expect(original).toEqual(deserialized); + }); }); diff --git a/yarn-project/circuits.js/src/structs/kernel/public_kernel_tail_circuit_private_inputs.ts b/yarn-project/circuits.js/src/structs/kernel/public_kernel_tail_circuit_private_inputs.ts index 5399146b7e5..a1cf6b9d8ff 100644 --- a/yarn-project/circuits.js/src/structs/kernel/public_kernel_tail_circuit_private_inputs.ts +++ b/yarn-project/circuits.js/src/structs/kernel/public_kernel_tail_circuit_private_inputs.ts @@ -41,6 +41,14 @@ export class PublicKernelTailCircuitPrivateInputs { ); } + toString() { + return this.toBuffer().toString('hex'); + } + + static fromString(str: string) { + return PublicKernelTailCircuitPrivateInputs.fromBuffer(Buffer.from(str, 'hex')); + } + static fromBuffer(buffer: Buffer | BufferReader) { const reader = BufferReader.asReader(buffer); return new PublicKernelTailCircuitPrivateInputs( diff --git a/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts b/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts index 43f80b52ee1..8e66f351f66 100644 --- a/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts @@ -36,6 +36,7 @@ import { openTmpStore } from '@aztec/kv-store/utils'; import { AvailabilityOracleAbi, InboxAbi, OutboxAbi, RollupAbi } from '@aztec/l1-artifacts'; import { SHA256Trunc, StandardTree } from '@aztec/merkle-tree'; import { TxProver } from '@aztec/prover-client'; +import { ProverPool } from '@aztec/prover-client/prover-pool'; import { type L1Publisher, getL1Publisher } from '@aztec/sequencer-client'; import { WASMSimulator } from '@aztec/simulator'; import { MerkleTrees, ServerWorldStateSynchronizer, type WorldStateConfig } from '@aztec/world-state'; @@ -143,7 +144,7 @@ describe('L1Publisher integration', () => { }; const worldStateSynchronizer = new ServerWorldStateSynchronizer(tmpStore, builderDb, blockSource, worldStateConfig); await worldStateSynchronizer.start(); - builder = await TxProver.new({}, worldStateSynchronizer, new WASMSimulator()); + builder = await TxProver.new(worldStateSynchronizer, ProverPool.testPool(new WASMSimulator(), 4)); l2Proof = Buffer.alloc(0); publisher = getL1Publisher({ diff --git a/yarn-project/prover-client/package.json b/yarn-project/prover-client/package.json index 132403cbb7f..64580446844 100644 --- a/yarn-project/prover-client/package.json +++ b/yarn-project/prover-client/package.json @@ -2,7 +2,10 @@ "name": "@aztec/prover-client", "version": "0.1.0", "type": "module", - "exports": "./dest/index.js", + "exports": { + ".": "./dest/index.js", + "./prover-pool": "./dest/prover-pool/index.js" + }, "bin": { "bb-cli": "./dest/bb/index.js" }, diff --git a/yarn-project/prover-client/src/dummy-prover.ts b/yarn-project/prover-client/src/dummy-prover.ts index de61267c738..ed9b1fdf06a 100644 --- a/yarn-project/prover-client/src/dummy-prover.ts +++ b/yarn-project/prover-client/src/dummy-prover.ts @@ -4,6 +4,9 @@ import { PROVING_STATUS, type ProcessedTx, type ProverClient, + type ProvingJob, + type ProvingJobSource, + type ProvingRequest, type ProvingSuccess, type ProvingTicket, } from '@aztec/circuit-types'; @@ -11,6 +14,8 @@ import { type GlobalVariables, makeEmptyProof } from '@aztec/circuits.js'; import { type Fr } from '@aztec/foundation/fields'; export class DummyProver implements ProverClient { + jobs = new DummyProvingJobSource(); + public start(): Promise { return Promise.resolve(); } @@ -54,4 +59,22 @@ export class DummyProver implements ProverClient { setBlockCompleted(): Promise { return Promise.resolve(); } + + getProvingJobSource(): ProvingJobSource { + return this.jobs; + } +} + +class DummyProvingJobSource implements ProvingJobSource { + getProvingJob(): Promise | null> { + return Promise.resolve(null); + } + + rejectProvingJob(): Promise { + return Promise.resolve(); + } + + resolveProvingJob(): Promise { + return Promise.resolve(); + } } diff --git a/yarn-project/prover-client/src/prover-pool/index.ts b/yarn-project/prover-client/src/prover-pool/index.ts index eaae01068bc..353448bcbce 100644 --- a/yarn-project/prover-client/src/prover-pool/index.ts +++ b/yarn-project/prover-client/src/prover-pool/index.ts @@ -1,3 +1,4 @@ export * from './prover-agent.js'; export * from './memory-proving-queue.js'; export * from './prover-pool.js'; +export * from './rpc.js'; diff --git a/yarn-project/prover-client/src/prover-pool/prover-agent.ts b/yarn-project/prover-client/src/prover-pool/prover-agent.ts index b5dce971dbc..098359b49f0 100644 --- a/yarn-project/prover-client/src/prover-pool/prover-agent.ts +++ b/yarn-project/prover-client/src/prover-pool/prover-agent.ts @@ -11,6 +11,7 @@ import { RunningPromise } from '@aztec/foundation/running-promise'; import { elapsed } from '@aztec/foundation/timer'; import { type CircuitProver } from '../prover/interface.js'; +import { ProvingError } from './proving-error.js'; export class ProverAgent { private runningPromise?: RunningPromise; @@ -46,7 +47,7 @@ export class ProverAgent { this.log.error( `Error processing proving job id=${job.id} type=${ProvingRequestType[job.request.type]}: ${err}`, ); - await queue.rejectProvingJob(job.id, err as Error); + await queue.rejectProvingJob(job.id, new ProvingError(String(err))); } }, this.intervalMs); diff --git a/yarn-project/prover-client/src/prover-pool/prover-pool.ts b/yarn-project/prover-client/src/prover-pool/prover-pool.ts index 407b0a19fe6..0750b41520e 100644 --- a/yarn-project/prover-client/src/prover-pool/prover-pool.ts +++ b/yarn-project/prover-client/src/prover-pool/prover-pool.ts @@ -1,6 +1,11 @@ import { type ProvingJobSource } from '@aztec/circuit-types'; import { type SimulationProvider } from '@aztec/simulator'; +import { mkdtemp } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; + +import { BBNativeRollupProver, type BBProverConfig } from '../prover/bb_prover.js'; import { TestCircuitProver } from '../prover/test_circuit_prover.js'; import { ProverAgent } from './prover-agent.js'; @@ -43,10 +48,34 @@ export class ProverPool { this.running = false; } - static testPool(simulationProvider: SimulationProvider, size = 1, agentPollIntervalMS = 10): ProverPool { + static testPool(simulationProvider?: SimulationProvider, size = 1, agentPollIntervalMS = 10): ProverPool { return new ProverPool( size, - i => new ProverAgent(new TestCircuitProver(simulationProvider), agentPollIntervalMS, `${i}`), + i => new ProverAgent(new TestCircuitProver(simulationProvider), agentPollIntervalMS, `test-prover-${i}`), ); } + + static nativePool( + { acvmBinaryPath, bbBinaryPath }: Pick, + size: number, + agentPollIntervalMS = 10, + ): ProverPool { + // TODO generate keys ahead of time so that each agent doesn't have to do it + return new ProverPool(size, async i => { + const [acvmWorkingDirectory, bbWorkingDirectory] = await Promise.all([ + mkdtemp(join(tmpdir(), 'acvm-')), + mkdtemp(join(tmpdir(), 'bb-')), + ]); + return new ProverAgent( + await BBNativeRollupProver.new({ + acvmBinaryPath, + acvmWorkingDirectory, + bbBinaryPath, + bbWorkingDirectory, + }), + agentPollIntervalMS, + `bb-prover-${i}`, + ); + }); + } } diff --git a/yarn-project/prover-client/src/prover-pool/proving-error.ts b/yarn-project/prover-client/src/prover-pool/proving-error.ts new file mode 100644 index 00000000000..982ec0cdc0e --- /dev/null +++ b/yarn-project/prover-client/src/prover-pool/proving-error.ts @@ -0,0 +1,9 @@ +export class ProvingError extends Error { + override toString() { + return this.message; + } + + static fromString(message: string) { + return new ProvingError(message); + } +} diff --git a/yarn-project/prover-client/src/prover-pool/rpc.ts b/yarn-project/prover-client/src/prover-pool/rpc.ts new file mode 100644 index 00000000000..a7debf7bff2 --- /dev/null +++ b/yarn-project/prover-client/src/prover-pool/rpc.ts @@ -0,0 +1,73 @@ +import { type ProvingJobSource } from '@aztec/circuit-types'; +import { + BaseOrMergeRollupPublicInputs, + BaseParityInputs, + BaseRollupInputs, + KernelCircuitPublicInputs, + MergeRollupInputs, + ParityPublicInputs, + Proof, + PublicKernelCircuitPrivateInputs, + PublicKernelCircuitPublicInputs, + PublicKernelTailCircuitPrivateInputs, + RootParityInputs, + RootRollupInputs, + RootRollupPublicInputs, +} from '@aztec/circuits.js'; +import { createJsonRpcClient, makeFetch } from '@aztec/foundation/json-rpc/client'; +import { JsonRpcServer } from '@aztec/foundation/json-rpc/server'; + +import { ProvingError } from './proving-error.js'; + +export function createProvingJobSourceServer(queue: ProvingJobSource): JsonRpcServer { + return new JsonRpcServer( + queue, + { + BaseParityInputs, + BaseOrMergeRollupPublicInputs, + BaseRollupInputs, + MergeRollupInputs, + ParityPublicInputs, + Proof, + RootParityInputs, + RootRollupInputs, + RootRollupPublicInputs, + PublicKernelCircuitPrivateInputs, + PublicKernelCircuitPublicInputs, + PublicKernelTailCircuitPrivateInputs, + KernelCircuitPublicInputs, + ProvingError, + }, + {}, + ); +} + +export function createProvingJobSourceClient( + url: string, + namespace?: string, + fetch = makeFetch([1, 2, 3], false), +): ProvingJobSource { + return createJsonRpcClient( + url, + { + BaseParityInputs, + BaseOrMergeRollupPublicInputs, + BaseRollupInputs, + MergeRollupInputs, + ParityPublicInputs, + Proof, + RootParityInputs, + RootRollupInputs, + RootRollupPublicInputs, + PublicKernelCircuitPrivateInputs, + PublicKernelCircuitPublicInputs, + PublicKernelTailCircuitPrivateInputs, + KernelCircuitPublicInputs, + ProvingError, + }, + {}, + false, + namespace, + fetch, + ) as ProvingJobSource; +} diff --git a/yarn-project/prover-client/src/prover/test_circuit_prover.ts b/yarn-project/prover-client/src/prover/test_circuit_prover.ts index 47c15be17dc..6d2acdfa2e8 100644 --- a/yarn-project/prover-client/src/prover/test_circuit_prover.ts +++ b/yarn-project/prover-client/src/prover/test_circuit_prover.ts @@ -71,7 +71,7 @@ export class TestCircuitProver implements CircuitProver { private wasmSimulator = new WASMSimulator(); constructor( - private simulationProvider: SimulationProvider, + private simulationProvider?: SimulationProvider, private logger = createDebugLogger('aztec:test-prover'), ) {} @@ -131,7 +131,8 @@ export class TestCircuitProver implements CircuitProver { ): Promise> { const witnessMap = convertSimulatedBaseRollupInputsToWitnessMap(input); - const witness = await this.simulationProvider.simulateCircuit(witnessMap, SimulatedBaseRollupArtifact); + const simulationProvider = this.simulationProvider ?? this.wasmSimulator; + const witness = await simulationProvider.simulateCircuit(witnessMap, SimulatedBaseRollupArtifact); const result = convertSimulatedBaseRollupOutputsFromWitnessMap(witness); diff --git a/yarn-project/prover-client/src/tx-prover/tx-prover.ts b/yarn-project/prover-client/src/tx-prover/tx-prover.ts index 90daea5a7fa..bc45cc582bf 100644 --- a/yarn-project/prover-client/src/tx-prover/tx-prover.ts +++ b/yarn-project/prover-client/src/tx-prover/tx-prover.ts @@ -1,37 +1,30 @@ import { type ProcessedTx } from '@aztec/circuit-types'; -import { type BlockResult, type ProverClient, type ProvingTicket } from '@aztec/circuit-types/interfaces'; +import { + type BlockResult, + type ProverClient, + type ProvingJobSource, + type ProvingTicket, +} from '@aztec/circuit-types/interfaces'; import { type Fr, type GlobalVariables } from '@aztec/circuits.js'; -import { type SimulationProvider } from '@aztec/simulator'; import { type WorldStateSynchronizer } from '@aztec/world-state'; -import { type ProverConfig } from '../config.js'; import { type VerificationKeys, getVerificationKeys } from '../mocks/verification_keys.js'; import { ProvingOrchestrator } from '../orchestrator/orchestrator.js'; import { MemoryProvingQueue } from '../prover-pool/memory-proving-queue.js'; -import { ProverAgent } from '../prover-pool/prover-agent.js'; -import { ProverPool } from '../prover-pool/prover-pool.js'; -import { TestCircuitProver } from '../prover/test_circuit_prover.js'; +import { type ProverPool } from '../prover-pool/prover-pool.js'; /** * A prover accepting individual transaction requests */ export class TxProver implements ProverClient { private orchestrator: ProvingOrchestrator; - private proverPool: ProverPool; private queue = new MemoryProvingQueue(); constructor( private worldStateSynchronizer: WorldStateSynchronizer, - simulationProvider: SimulationProvider, protected vks: VerificationKeys, - agentCount = 4, - agentPollIntervalMS = 10, + private proverPool?: ProverPool, ) { - this.proverPool = new ProverPool( - agentCount, - i => new ProverAgent(new TestCircuitProver(simulationProvider), agentPollIntervalMS, `${i}`), - ); - this.orchestrator = new ProvingOrchestrator(worldStateSynchronizer.getLatest(), this.queue); } @@ -39,14 +32,14 @@ export class TxProver implements ProverClient { * Starts the prover instance */ public async start() { - await this.proverPool.start(this.queue); + await this.proverPool?.start(this.queue); } /** * Stops the prover instance */ public async stop() { - await this.proverPool.stop(); + await this.proverPool?.stop(); } /** @@ -55,12 +48,8 @@ export class TxProver implements ProverClient { * @param worldStateSynchronizer - An instance of the world state * @returns An instance of the prover, constructed and started. */ - public static async new( - config: ProverConfig, - worldStateSynchronizer: WorldStateSynchronizer, - simulationProvider: SimulationProvider, - ) { - const prover = new TxProver(worldStateSynchronizer, simulationProvider, getVerificationKeys()); + public static async new(worldStateSynchronizer: WorldStateSynchronizer, proverPool?: ProverPool) { + const prover = new TxProver(worldStateSynchronizer, getVerificationKeys(), proverPool); await prover.start(); return prover; } @@ -112,4 +101,8 @@ export class TxProver implements ProverClient { public setBlockCompleted(): Promise { return this.orchestrator.setBlockCompleted(); } + + getProvingJobSource(): ProvingJobSource { + return this.queue; + } } diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index f7a6a3e7d1c..9b79cfcc6a7 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -207,6 +207,7 @@ __metadata: "@aztec/noir-contracts.js": "workspace:^" "@aztec/p2p": "workspace:^" "@aztec/protocol-contracts": "workspace:^" + "@aztec/prover-client": "workspace:^" "@aztec/pxe": "workspace:^" "@jest/globals": ^29.5.0 "@types/jest": ^29.5.0 From f69572565839cc784c7f724ebee1df56b0f37b9c Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Fri, 26 Apr 2024 14:01:30 +0000 Subject: [PATCH 2/3] fix: pass error message along --- .../aztec-node/src/aztec-node/config.ts | 12 ++---- .../aztec-node/src/aztec-node/server.ts | 3 +- .../composed/integration_l1_publisher.test.ts | 3 +- yarn-project/prover-client/src/config.ts | 37 ++++++++++++++++--- .../src/prover-pool/prover-agent.ts | 2 +- .../prover-client/src/tx-prover/tx-prover.ts | 30 +++++++++++++-- 6 files changed, 65 insertions(+), 22 deletions(-) diff --git a/yarn-project/aztec-node/src/aztec-node/config.ts b/yarn-project/aztec-node/src/aztec-node/config.ts index 4b644ca3b78..d4c71c30421 100644 --- a/yarn-project/aztec-node/src/aztec-node/config.ts +++ b/yarn-project/aztec-node/src/aztec-node/config.ts @@ -1,5 +1,6 @@ import { type ArchiverConfig, getConfigEnvVars as getArchiverVars } from '@aztec/archiver'; import { type P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p'; +import { type ProverConfig, getProverEnvVars } from '@aztec/prover-client'; import { type SequencerClientConfig, getConfigEnvVars as getSequencerVars } from '@aztec/sequencer-client'; import { getConfigEnvVars as getWorldStateVars } from '@aztec/world-state'; @@ -8,6 +9,7 @@ import { getConfigEnvVars as getWorldStateVars } from '@aztec/world-state'; */ export type AztecNodeConfig = ArchiverConfig & SequencerClientConfig & + ProverConfig & P2PConfig & { /** Whether the sequencer is disabled for this node. */ disableSequencer: boolean; @@ -17,8 +19,6 @@ export type AztecNodeConfig = ArchiverConfig & /** A URL for an archiver service that the node will use. */ archiverUrl?: string; - - proverAgents: number; }; /** @@ -26,21 +26,17 @@ export type AztecNodeConfig = ArchiverConfig & * @returns A valid aztec node config. */ export function getConfigEnvVars(): AztecNodeConfig { - const { SEQ_DISABLED, PROVER_DISABLED, ARCHIVER_URL, PROVER_AGENTS = '1' } = process.env; - let proverAgents = parseInt(PROVER_AGENTS, 10); - if (Number.isNaN(proverAgents)) { - proverAgents = 1; - } + const { SEQ_DISABLED, PROVER_DISABLED, ARCHIVER_URL } = process.env; const allEnvVars: AztecNodeConfig = { ...getSequencerVars(), ...getArchiverVars(), ...getP2PConfigEnvVars(), ...getWorldStateVars(), + ...getProverEnvVars(), disableSequencer: !!SEQ_DISABLED, archiverUrl: ARCHIVER_URL, disableProver: PROVER_DISABLED === '1', - proverAgents, }; return allEnvVars; diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 7ec441d29a4..3bf5b97ab46 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -52,7 +52,6 @@ import { initStoreForRollup, openTmpStore } from '@aztec/kv-store/utils'; import { SHA256Trunc, StandardTree } from '@aztec/merkle-tree'; import { AztecKVTxPool, type P2P, createP2PClient } from '@aztec/p2p'; import { DummyProver, TxProver } from '@aztec/prover-client'; -import { ProverPool } from '@aztec/prover-client/prover-pool'; import { type GlobalVariableBuilder, SequencerClient, getGlobalVariableBuilder } from '@aztec/sequencer-client'; import { PublicProcessorFactory, WASMSimulator } from '@aztec/simulator'; import { @@ -152,7 +151,7 @@ export class AztecNodeService implements AztecNode { const simulationProvider = await getSimulationProvider(config, log); const prover = config.disableProver ? await DummyProver.new() - : await TxProver.new(worldStateSynchronizer, ProverPool.testPool(simulationProvider, config.proverAgents)); + : await TxProver.new(config, simulationProvider, worldStateSynchronizer); // now create the sequencer const sequencer = config.disableSequencer diff --git a/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts b/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts index 8e66f351f66..29432464114 100644 --- a/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts @@ -36,7 +36,6 @@ import { openTmpStore } from '@aztec/kv-store/utils'; import { AvailabilityOracleAbi, InboxAbi, OutboxAbi, RollupAbi } from '@aztec/l1-artifacts'; import { SHA256Trunc, StandardTree } from '@aztec/merkle-tree'; import { TxProver } from '@aztec/prover-client'; -import { ProverPool } from '@aztec/prover-client/prover-pool'; import { type L1Publisher, getL1Publisher } from '@aztec/sequencer-client'; import { WASMSimulator } from '@aztec/simulator'; import { MerkleTrees, ServerWorldStateSynchronizer, type WorldStateConfig } from '@aztec/world-state'; @@ -144,7 +143,7 @@ describe('L1Publisher integration', () => { }; const worldStateSynchronizer = new ServerWorldStateSynchronizer(tmpStore, builderDb, blockSource, worldStateConfig); await worldStateSynchronizer.start(); - builder = await TxProver.new(worldStateSynchronizer, ProverPool.testPool(new WASMSimulator(), 4)); + builder = await TxProver.new(config, new WASMSimulator(), worldStateSynchronizer); l2Proof = Buffer.alloc(0); publisher = getL1Publisher({ diff --git a/yarn-project/prover-client/src/config.ts b/yarn-project/prover-client/src/config.ts index a6d2cefb83c..6d840cac681 100644 --- a/yarn-project/prover-client/src/config.ts +++ b/yarn-project/prover-client/src/config.ts @@ -1,11 +1,21 @@ +import { tmpdir } from 'os'; + /** * The prover configuration. */ export interface ProverConfig { /** The working directory to use for simulation/proving */ - acvmWorkingDirectory?: string; + acvmWorkingDirectory: string; /** The path to the ACVM binary */ - acvmBinaryPath?: string; + acvmBinaryPath: string; + /** The working directory to for proving */ + bbWorkingDirectory: string; + /** The path to the bb binary */ + bbBinaryPath: string; + /** How many agents to start */ + proverAgents: number; + /** Enable proving. If true, must set bb env vars */ + realProofs: boolean; } /** @@ -13,10 +23,25 @@ export interface ProverConfig { * Note: If an environment variable is not set, the default value is used. * @returns The prover configuration. */ -export function getConfigEnvVars(): ProverConfig { - const { ACVM_WORKING_DIRECTORY, ACVM_BINARY_PATH } = process.env; +export function getProverEnvVars(): ProverConfig { + const { + ACVM_WORKING_DIRECTORY = tmpdir(), + ACVM_BINARY_PATH = '', + BB_WORKING_DIRECTORY = tmpdir(), + BB_BINARY_PATH = '', + PROVER_AGENTS = '1', + PROVER_REAL_PROOFS = '', + } = process.env; + + const parsedProverAgents = parseInt(PROVER_AGENTS, 10); + const proverAgents = Number.isSafeInteger(parsedProverAgents) ? parsedProverAgents : 0; + return { - acvmWorkingDirectory: ACVM_WORKING_DIRECTORY ? ACVM_WORKING_DIRECTORY : undefined, - acvmBinaryPath: ACVM_BINARY_PATH ? ACVM_BINARY_PATH : undefined, + acvmWorkingDirectory: ACVM_WORKING_DIRECTORY, + acvmBinaryPath: ACVM_BINARY_PATH, + bbBinaryPath: BB_BINARY_PATH, + bbWorkingDirectory: BB_WORKING_DIRECTORY, + proverAgents, + realProofs: PROVER_REAL_PROOFS === '1', }; } diff --git a/yarn-project/prover-client/src/prover-pool/prover-agent.ts b/yarn-project/prover-client/src/prover-pool/prover-agent.ts index 098359b49f0..e5ae9f156f9 100644 --- a/yarn-project/prover-client/src/prover-pool/prover-agent.ts +++ b/yarn-project/prover-client/src/prover-pool/prover-agent.ts @@ -47,7 +47,7 @@ export class ProverAgent { this.log.error( `Error processing proving job id=${job.id} type=${ProvingRequestType[job.request.type]}: ${err}`, ); - await queue.rejectProvingJob(job.id, new ProvingError(String(err))); + await queue.rejectProvingJob(job.id, new ProvingError((err as any)?.message ?? String(err))); } }, this.intervalMs); diff --git a/yarn-project/prover-client/src/tx-prover/tx-prover.ts b/yarn-project/prover-client/src/tx-prover/tx-prover.ts index bc45cc582bf..e5b08a74560 100644 --- a/yarn-project/prover-client/src/tx-prover/tx-prover.ts +++ b/yarn-project/prover-client/src/tx-prover/tx-prover.ts @@ -6,12 +6,14 @@ import { type ProvingTicket, } from '@aztec/circuit-types/interfaces'; import { type Fr, type GlobalVariables } from '@aztec/circuits.js'; +import { type SimulationProvider } from '@aztec/simulator'; import { type WorldStateSynchronizer } from '@aztec/world-state'; +import { type ProverConfig } from '../config.js'; import { type VerificationKeys, getVerificationKeys } from '../mocks/verification_keys.js'; import { ProvingOrchestrator } from '../orchestrator/orchestrator.js'; import { MemoryProvingQueue } from '../prover-pool/memory-proving-queue.js'; -import { type ProverPool } from '../prover-pool/prover-pool.js'; +import { ProverPool } from '../prover-pool/prover-pool.js'; /** * A prover accepting individual transaction requests @@ -48,8 +50,30 @@ export class TxProver implements ProverClient { * @param worldStateSynchronizer - An instance of the world state * @returns An instance of the prover, constructed and started. */ - public static async new(worldStateSynchronizer: WorldStateSynchronizer, proverPool?: ProverPool) { - const prover = new TxProver(worldStateSynchronizer, getVerificationKeys(), proverPool); + public static async new( + config: ProverConfig, + simulationProvider: SimulationProvider, + worldStateSynchronizer: WorldStateSynchronizer, + ) { + let pool: ProverPool | undefined; + if (config.proverAgents === 0) { + pool = undefined; + } else if (config.realProofs) { + if ( + !config.acvmBinaryPath || + !config.acvmWorkingDirectory || + !config.bbBinaryPath || + !config.bbWorkingDirectory + ) { + throw new Error(); + } + + pool = ProverPool.nativePool(config, config.proverAgents, 10); + } else { + pool = ProverPool.testPool(simulationProvider, config.proverAgents, 10); + } + + const prover = new TxProver(worldStateSynchronizer, getVerificationKeys(), pool); await prover.start(); return prover; } From 7bd3d27db6e8fd0a0078282ac6a548c186d359c4 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Mon, 29 Apr 2024 13:26:16 +0000 Subject: [PATCH 3/3] feat: accept 1,true for env vars --- yarn-project/aztec-node/src/aztec-node/config.ts | 4 ++-- yarn-project/prover-client/src/config.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn-project/aztec-node/src/aztec-node/config.ts b/yarn-project/aztec-node/src/aztec-node/config.ts index d4c71c30421..dba7d824025 100644 --- a/yarn-project/aztec-node/src/aztec-node/config.ts +++ b/yarn-project/aztec-node/src/aztec-node/config.ts @@ -26,7 +26,7 @@ export type AztecNodeConfig = ArchiverConfig & * @returns A valid aztec node config. */ export function getConfigEnvVars(): AztecNodeConfig { - const { SEQ_DISABLED, PROVER_DISABLED, ARCHIVER_URL } = process.env; + const { SEQ_DISABLED, PROVER_DISABLED = '', ARCHIVER_URL } = process.env; const allEnvVars: AztecNodeConfig = { ...getSequencerVars(), @@ -36,7 +36,7 @@ export function getConfigEnvVars(): AztecNodeConfig { ...getProverEnvVars(), disableSequencer: !!SEQ_DISABLED, archiverUrl: ARCHIVER_URL, - disableProver: PROVER_DISABLED === '1', + disableProver: ['1', 'true'].includes(PROVER_DISABLED), }; return allEnvVars; diff --git a/yarn-project/prover-client/src/config.ts b/yarn-project/prover-client/src/config.ts index 6d840cac681..0b8f7cce6ea 100644 --- a/yarn-project/prover-client/src/config.ts +++ b/yarn-project/prover-client/src/config.ts @@ -42,6 +42,6 @@ export function getProverEnvVars(): ProverConfig { bbBinaryPath: BB_BINARY_PATH, bbWorkingDirectory: BB_WORKING_DIRECTORY, proverAgents, - realProofs: PROVER_REAL_PROOFS === '1', + realProofs: ['1', 'true'].includes(PROVER_REAL_PROOFS), }; }