feat: configure prover as separate process

yarn-project/aztec-node/src/aztec-node/config.ts

@@ -17,22 +17,30 @@ export type AztecNodeConfig = ArchiverConfig &
 
     /** A URL for an archiver service that the node will use. */
     archiverUrl?: string;
+
+    proverAgents: number;
   };
 
 /**
  * Returns the config of the aztec node from environment variables with reasonable defaults.
  * @returns A valid aztec node config.
  */
 export function getConfigEnvVars(): AztecNodeConfig {
-  const { SEQ_DISABLED, PROVER_DISABLED } = process.env;
+  const { SEQ_DISABLED, PROVER_DISABLED, ARCHIVER_URL, PROVER_AGENTS = '1' } = process.env;
+  let proverAgents = parseInt(PROVER_AGENTS, 10);
+  if (Number.isNaN(proverAgents)) {
+    proverAgents = 1;
+  }
+
   const allEnvVars: AztecNodeConfig = {
     ...getSequencerVars(),
     ...getArchiverVars(),
     ...getP2PConfigEnvVars(),
     ...getWorldStateVars(),
     disableSequencer: !!SEQ_DISABLED,
-    disableProver: !!PROVER_DISABLED,
-    archiverUrl: process.env.ARCHIVER_URL,
+    archiverUrl: ARCHIVER_URL,
+    disableProver: PROVER_DISABLED === '1',
+    proverAgents,
   };
 
   return allEnvVars;

yarn-project/aztec-node/src/aztec-node/server.ts

@@ -52,6 +52,7 @@ import { initStoreForRollup, openTmpStore } from '@aztec/kv-store/utils';
 import { SHA256Trunc, StandardTree } from '@aztec/merkle-tree';
 import { AztecKVTxPool, type P2P, createP2PClient } from '@aztec/p2p';
 import { DummyProver, TxProver } from '@aztec/prover-client';
+import { ProverPool } from '@aztec/prover-client/prover-pool';
 import { type GlobalVariableBuilder, SequencerClient, getGlobalVariableBuilder } from '@aztec/sequencer-client';
 import { PublicProcessorFactory, WASMSimulator } from '@aztec/simulator';
 import {
@@ -151,7 +152,7 @@ export class AztecNodeService implements AztecNode {
     const simulationProvider = await getSimulationProvider(config, log);
     const prover = config.disableProver
       ? await DummyProver.new()
-      : await TxProver.new(config, worldStateSynchronizer, simulationProvider);
+      : await TxProver.new(worldStateSynchronizer, ProverPool.testPool(simulationProvider, config.proverAgents));
 
     // now create the sequencer
     const sequencer = config.disableSequencer
@@ -194,6 +195,10 @@ export class AztecNodeService implements AztecNode {
     return this.sequencer;
   }
 
+  public getProver(): ProverClient {
+    return this.prover;
+  }
+
   /**
    * Method to return the currently deployed L1 contract addresses.
    * @returns - The currently deployed L1 contract addresses.

yarn-project/aztec/package.json

@@ -42,6 +42,7 @@
     "@aztec/noir-contracts.js": "workspace:^",
     "@aztec/p2p": "workspace:^",
     "@aztec/protocol-contracts": "workspace:^",
+    "@aztec/prover-client": "workspace:^",
     "@aztec/pxe": "workspace:^",
     "abitype": "^0.8.11",
     "commander": "^11.1.0",

yarn-project/aztec/src/cli/cli.ts

@@ -56,6 +56,9 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command {
       } else if (options.p2pBootstrap) {
         const { startP2PBootstrap } = await import('./cmds/start_p2p_bootstrap.js');
         await startP2PBootstrap(options, signalHandlers, userLog, debugLogger);
+      } else if (options.prover) {
+        const { startProver } = await import('./cmds/start_prover.js');
+        services = await startProver(options, signalHandlers, userLog);
       }
       if (services.length) {
         const rpcServer = createNamespacedJsonRpcServer(services, debugLogger);

yarn-project/aztec/src/cli/cmds/start_node.ts

@@ -6,6 +6,7 @@ import {
 import { NULL_KEY } from '@aztec/ethereum';
 import { type ServerList } from '@aztec/foundation/json-rpc/server';
 import { type LogFn } from '@aztec/foundation/log';
+import { createProvingJobSourceServer } from '@aztec/prover-client/prover-pool';
 import { type PXEServiceConfig, createPXERpcServer, getPXEServiceConfig } from '@aztec/pxe';
 
 import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts';
@@ -64,7 +65,10 @@ export const startNode = async (
   }
 
   if (!options.prover) {
+    userLog(`Prover is disabled, using mocked proofs`);
     nodeConfig.disableProver = true;
+  } else {
+    nodeConfig = mergeEnvVarsAndCliOptions<AztecNodeConfig>(nodeConfig, parseModuleOptions(options.prover));
   }
 
   if (!nodeConfig.disableSequencer && nodeConfig.disableProver) {
@@ -78,6 +82,11 @@ export const startNode = async (
   // Add node to services list
   services.push({ node: nodeServer });
 
+  if (!nodeConfig.disableProver) {
+    const provingJobSource = createProvingJobSourceServer(node.getProver().getProvingJobSource());
+    services.push({ provingJobSource });
+  }
+
   // Add node stop function to signal handlers
   signalHandlers.push(node.stop);
 

yarn-project/aztec/src/cli/cmds/start_prover.ts

@@ -0,0 +1,50 @@
+import { type ProvingJobSource } from '@aztec/circuit-types';
+import { ProverPool, createProvingJobSourceClient } from '@aztec/prover-client/prover-pool';
+
+import { type ServiceStarter, parseModuleOptions } from '../util.js';
+
+type ProverOptions = Partial<{
+  proverUrl: string;
+  agents: string;
+  acvmBinaryPath?: string;
+  bbBinaryPath?: string;
+  simulate?: string;
+}>;
+
+export const startProver: ServiceStarter = async (options, signalHandlers, logger) => {
+  const proverOptions: ProverOptions = parseModuleOptions(options.prover);
+  let source: ProvingJobSource;
+
+  if (typeof proverOptions.proverUrl === 'string') {
+    logger(`Connecting to prover at ${proverOptions.proverUrl}`);
+    source = createProvingJobSourceClient(proverOptions.proverUrl, 'provingJobSource');
+  } else {
+    throw new Error('Starting prover without an orchestrator is not supported');
+  }
+
+  const agentCount = proverOptions.agents ? parseInt(proverOptions.agents, 10) : 1;
+  if (agentCount === 0 || !Number.isSafeInteger(agentCount)) {
+    throw new Error('Cannot start prover without agents');
+  }
+
+  let pool: ProverPool;
+  if (proverOptions.simulate) {
+    pool = ProverPool.testPool(undefined, agentCount);
+  } else if (proverOptions.acvmBinaryPath && proverOptions.bbBinaryPath) {
+    pool = ProverPool.nativePool(
+      {
+        acvmBinaryPath: proverOptions.acvmBinaryPath,
+        bbBinaryPath: proverOptions.bbBinaryPath,
+      },
+      agentCount,
+    );
+  } else {
+    throw new Error('Cannot start prover without simulation or native prover options');
+  }
+
+  logger(`Starting ${agentCount} prover agents`);
+  await pool.start(source);
+  signalHandlers.push(() => pool.stop());
+
+  return Promise.resolve([]);
+};

yarn-project/aztec/src/cli/util.ts

@@ -3,10 +3,15 @@ import { type AztecNodeConfig } from '@aztec/aztec-node';
 import { type AccountManager, type Fr } from '@aztec/aztec.js';
 import { type L1ContractAddresses, l1ContractsNames } from '@aztec/ethereum';
 import { EthAddress } from '@aztec/foundation/eth-address';
+import { type ServerList } from '@aztec/foundation/json-rpc/server';
 import { type LogFn, createConsoleLogger } from '@aztec/foundation/log';
 import { type P2PConfig } from '@aztec/p2p';
 import { type PXEService, type PXEServiceConfig } from '@aztec/pxe';
 
+export interface ServiceStarter<T = any> {
+  (options: T, signalHandlers: (() => Promise<void>)[], logger: LogFn): Promise<ServerList>;
+}
+
 /**
  * Checks if the object has l1Contracts property
  * @param obj - The object to check

yarn-project/aztec/tsconfig.json

@@ -51,6 +51,9 @@
     {
       "path": "../protocol-contracts"
     },
+    {
+      "path": "../prover-client"
+    },
     {
       "path": "../pxe"
     }

yarn-project/circuit-types/src/interfaces/prover-client.ts

@@ -1,4 +1,5 @@
 import { type BlockProver } from './block-prover.js';
+import { type ProvingJobSource } from './proving-job.js';
 
 /**
  * The interface to the prover client.
@@ -8,4 +9,6 @@ export interface ProverClient extends BlockProver {
   start(): Promise<void>;
 
   stop(): Promise<void>;
+
+  getProvingJobSource(): ProvingJobSource;
 }

yarn-project/circuits.js/src/structs/kernel/kernel_circuit_public_inputs.ts

@@ -78,4 +78,12 @@ export class KernelCircuitPublicInputs {
       RevertCode.OK,
     );
   }
+
+  toString() {
+    return this.toBuffer().toString('hex');
+  }
+
+  static fromString(str: string) {
+    return KernelCircuitPublicInputs.fromBuffer(Buffer.from(str, 'hex'));
+  }
 }

yarn-project/circuits.js/src/structs/kernel/public_kernel_circuit_public_inputs.ts

@@ -51,6 +51,14 @@ export class PublicKernelCircuitPublicInputs {
     );
   }
 
+  toString() {
+    return this.toBuffer().toString('hex');
+  }
+
+  static fromString(str: string) {
+    return PublicKernelCircuitPublicInputs.fromBuffer(Buffer.from(str, 'hex'));
+  }
+
   get needsSetup() {
     return !this.endNonRevertibleData.publicCallStack[1].isEmpty();
   }

yarn-project/circuits.js/src/structs/kernel/public_kernel_tail_circuit_private_inputs.test.ts

@@ -14,4 +14,11 @@ describe('PublicKernelTailCircuitPrivateInputs', () => {
     expect(original).toEqual(serialized);
     expect(original).not.toBe(serialized);
   });
+
+  it('serializes to string and back', () => {
+    const original = makePublicKernelTailCircuitPrivateInputs(123);
+    const str = original.toString();
+    const deserialized = PublicKernelTailCircuitPrivateInputs.fromString(str);
+    expect(original).toEqual(deserialized);
+  });
 });

yarn-project/circuits.js/src/structs/kernel/public_kernel_tail_circuit_private_inputs.ts

@@ -41,6 +41,14 @@ export class PublicKernelTailCircuitPrivateInputs {
     );
   }
 
+  toString() {
+    return this.toBuffer().toString('hex');
+  }
+
+  static fromString(str: string) {
+    return PublicKernelTailCircuitPrivateInputs.fromBuffer(Buffer.from(str, 'hex'));
+  }
+
   static fromBuffer(buffer: Buffer | BufferReader) {
     const reader = BufferReader.asReader(buffer);
     return new PublicKernelTailCircuitPrivateInputs(

yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts

@@ -36,6 +36,7 @@ import { openTmpStore } from '@aztec/kv-store/utils';
 import { AvailabilityOracleAbi, InboxAbi, OutboxAbi, RollupAbi } from '@aztec/l1-artifacts';
 import { SHA256Trunc, StandardTree } from '@aztec/merkle-tree';
 import { TxProver } from '@aztec/prover-client';
+import { ProverPool } from '@aztec/prover-client/prover-pool';
 import { type L1Publisher, getL1Publisher } from '@aztec/sequencer-client';
 import { WASMSimulator } from '@aztec/simulator';
 import { MerkleTrees, ServerWorldStateSynchronizer, type WorldStateConfig } from '@aztec/world-state';
@@ -143,7 +144,7 @@ describe('L1Publisher integration', () => {
     };
     const worldStateSynchronizer = new ServerWorldStateSynchronizer(tmpStore, builderDb, blockSource, worldStateConfig);
     await worldStateSynchronizer.start();
-    builder = await TxProver.new({}, worldStateSynchronizer, new WASMSimulator());
+    builder = await TxProver.new(worldStateSynchronizer, ProverPool.testPool(new WASMSimulator(), 4));
     l2Proof = Buffer.alloc(0);
 
     publisher = getL1Publisher({

yarn-project/prover-client/package.json

@@ -2,7 +2,10 @@
   "name": "@aztec/prover-client",
   "version": "0.1.0",
   "type": "module",
-  "exports": "./dest/index.js",
+  "exports": {
+    ".": "./dest/index.js",
+    "./prover-pool": "./dest/prover-pool/index.js"
+  },
   "bin": {
     "bb-cli": "./dest/bb/index.js"
   },

yarn-project/prover-client/src/dummy-prover.ts

@@ -4,13 +4,18 @@ import {
   PROVING_STATUS,
   type ProcessedTx,
   type ProverClient,
+  type ProvingJob,
+  type ProvingJobSource,
+  type ProvingRequest,
   type ProvingSuccess,
   type ProvingTicket,
 } from '@aztec/circuit-types';
 import { type GlobalVariables, makeEmptyProof } from '@aztec/circuits.js';
 import { type Fr } from '@aztec/foundation/fields';
 
 export class DummyProver implements ProverClient {
+  jobs = new DummyProvingJobSource();
+
   public start(): Promise<void> {
     return Promise.resolve();
   }
@@ -54,4 +59,22 @@ export class DummyProver implements ProverClient {
   setBlockCompleted(): Promise<void> {
     return Promise.resolve();
   }
+
+  getProvingJobSource(): ProvingJobSource {
+    return this.jobs;
+  }
+}
+
+class DummyProvingJobSource implements ProvingJobSource {
+  getProvingJob(): Promise<ProvingJob<ProvingRequest> | null> {
+    return Promise.resolve(null);
+  }
+
+  rejectProvingJob(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  resolveProvingJob(): Promise<void> {
+    return Promise.resolve();
+  }
 }

yarn-project/prover-client/src/prover-pool/index.ts

@@ -1,3 +1,4 @@
 export * from './prover-agent.js';
 export * from './memory-proving-queue.js';
 export * from './prover-pool.js';
+export * from './rpc.js';

yarn-project/prover-client/src/prover-pool/prover-agent.ts

@@ -10,6 +10,7 @@ import { RunningPromise } from '@aztec/foundation/running-promise';
 import { elapsed } from '@aztec/foundation/timer';
 
 import { type CircuitProver } from '../prover/interface.js';
+import { ProvingError } from './proving-error.js';
 
 export class ProverAgent {
   private runningPromise?: RunningPromise;
@@ -45,7 +46,7 @@ export class ProverAgent {
         this.log.error(
           `Error processing proving job id=${job.id} type=${ProvingRequestType[job.request.type]}: ${err}`,
         );
-        await queue.rejectProvingJob(job.id, err as Error);
+        await queue.rejectProvingJob(job.id, new ProvingError(String(err)));
       }
     }, this.intervalMs);
 

yarn-project/prover-client/src/prover-pool/prover-pool.ts

@@ -1,6 +1,11 @@
 import { type ProvingJobSource } from '@aztec/circuit-types';
 import { type SimulationProvider } from '@aztec/simulator';
 
+import { mkdtemp } from 'fs/promises';
+import { tmpdir } from 'os';
+import { join } from 'path';
+
+import { BBNativeRollupProver, type BBProverConfig } from '../prover/bb_prover.js';
 import { TestCircuitProver } from '../prover/test_circuit_prover.js';
 import { ProverAgent } from './prover-agent.js';
 
@@ -43,10 +48,34 @@ export class ProverPool {
     this.running = false;
   }
 
-  static testPool(simulationProvider: SimulationProvider, size = 1, agentPollIntervalMS = 10): ProverPool {
+  static testPool(simulationProvider?: SimulationProvider, size = 1, agentPollIntervalMS = 10): ProverPool {
     return new ProverPool(
       size,
-      i => new ProverAgent(new TestCircuitProver(simulationProvider), agentPollIntervalMS, `${i}`),
+      i => new ProverAgent(new TestCircuitProver(simulationProvider), agentPollIntervalMS, `test-prover-${i}`),
     );
   }
+
+  static nativePool(
+    { acvmBinaryPath, bbBinaryPath }: Pick<BBProverConfig, 'acvmBinaryPath' | 'bbBinaryPath'>,
+    size: number,
+    agentPollIntervalMS = 10,
+  ): ProverPool {
+    // TODO generate keys ahead of time so that each agent doesn't have to do it
+    return new ProverPool(size, async i => {
+      const [acvmWorkingDirectory, bbWorkingDirectory] = await Promise.all([
+        mkdtemp(join(tmpdir(), 'acvm-')),
+        mkdtemp(join(tmpdir(), 'bb-')),
+      ]);
+      return new ProverAgent(
+        await BBNativeRollupProver.new({
+          acvmBinaryPath,
+          acvmWorkingDirectory,
+          bbBinaryPath,
+          bbWorkingDirectory,
+        }),
+        agentPollIntervalMS,
+        `bb-prover-${i}`,
+      );
+    });
+  }
 }