feat: Dynamic proving (#5346)

This PR introduces a new proving orhestration system enabling a block to be proven concurrently with simulation. It also creates the `prover-client` package and moves some of the proving functionality to this package from `sequencer-client`.
AztecProtocol · Mar 22, 2024 · 6a7ccca · 6a7ccca
1 parent 8a76068
commit 6a7ccca
Show file tree

Hide file tree

Showing 64 changed files with 2,459 additions and 1,535 deletions.
diff --git a/docs/docs/developers/sandbox/references/sandbox-reference.md b/docs/docs/developers/sandbox/references/sandbox-reference.md
@@ -59,7 +59,7 @@ cd ~/.aztec && docker-compose up
 If you wish to run components of the Aztec network stack separately, you can use the `aztec start` command with various options for enabling components.
 
 ```bash
-aztec start --node [nodeOptions] --pxe [pxeOptions] --archiver [archiverOptions] --sequencer [sequencerOptions] ----p2p-bootstrap [p2pOptions]
+aztec start --node [nodeOptions] --pxe [pxeOptions] --archiver [archiverOptions] --sequencer [sequencerOptions] --prover [proverOptions] ----p2p-bootstrap [p2pOptions]
 ```
 
 Starting the aztec node alongside a PXE, sequencer or archiver, will attach the components to the node. If you want to e.g. run a PXE separately to a node, you can:

diff --git a/yarn-project/aztec-node/package.json b/yarn-project/aztec-node/package.json
@@ -42,7 +42,9 @@
     "@aztec/l1-artifacts": "workspace:^",
     "@aztec/merkle-tree": "workspace:^",
     "@aztec/p2p": "workspace:^",
+    "@aztec/prover-client": "workspace:^",
     "@aztec/sequencer-client": "workspace:^",
+    "@aztec/simulator": "workspace:^",
     "@aztec/types": "workspace:^",
     "@aztec/world-state": "workspace:^",
     "koa": "^2.14.2",

diff --git a/yarn-project/aztec-node/src/aztec-node/config.ts b/yarn-project/aztec-node/src/aztec-node/config.ts
@@ -12,6 +12,9 @@ export type AztecNodeConfig = ArchiverConfig &
     /** Whether the sequencer is disabled for this node. */
     disableSequencer: boolean;
 
+    /** Whether the prover is disabled for this node. */
+    disableProver: boolean;
+
     /** A URL for an archiver service that the node will use. */
     archiverUrl?: string;
   };
@@ -21,13 +24,14 @@ export type AztecNodeConfig = ArchiverConfig &
  * @returns A valid aztec node config.
  */
 export function getConfigEnvVars(): AztecNodeConfig {
-  const { SEQ_DISABLED } = process.env;
+  const { SEQ_DISABLED, PROVER_DISABLED } = process.env;
   const allEnvVars: AztecNodeConfig = {
     ...getSequencerVars(),
     ...getArchiverVars(),
     ...getP2PConfigEnvVars(),
     ...getWorldStateVars(),
     disableSequencer: !!SEQ_DISABLED,
+    disableProver: !!PROVER_DISABLED,
     archiverUrl: process.env.ARCHIVER_URL,
   };
 

diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts
@@ -12,6 +12,7 @@ import {
   LogType,
   MerkleTreeId,
   NullifierMembershipWitness,
+  ProverClient,
   PublicDataWitness,
   SequencerConfig,
   SiblingPath,
@@ -20,6 +21,7 @@ import {
   TxHash,
   TxReceipt,
   TxStatus,
+  partitionReverts,
 } from '@aztec/circuit-types';
 import {
   ARCHIVE_HEIGHT,
@@ -44,14 +46,14 @@ import { AztecLmdbStore } from '@aztec/kv-store/lmdb';
 import { initStoreForRollup, openTmpStore } from '@aztec/kv-store/utils';
 import { SHA256Trunc, StandardTree } from '@aztec/merkle-tree';
 import { AztecKVTxPool, P2P, createP2PClient } from '@aztec/p2p';
+import { DummyProver, TxProver } from '@aztec/prover-client';
 import {
   GlobalVariableBuilder,
   PublicProcessorFactory,
   SequencerClient,
-  WASMSimulator,
   getGlobalVariableBuilder,
-  partitionReverts,
 } from '@aztec/sequencer-client';
+import { WASMSimulator } from '@aztec/simulator';
 import { ContractClassPublic, ContractDataSource, ContractInstanceWithAddress } from '@aztec/types/contracts';
 import {
   MerkleTrees,
@@ -62,6 +64,7 @@ import {
 } from '@aztec/world-state';
 
 import { AztecNodeConfig } from './config.js';
+import { getSimulationProvider } from './simulator-factory.js';
 
 /**
  * The aztec node.
@@ -81,6 +84,7 @@ export class AztecNodeService implements AztecNode {
     protected readonly version: number,
     protected readonly globalVariableBuilder: GlobalVariableBuilder,
     protected readonly merkleTreesDb: AztecKVStore,
+    private readonly prover: ProverClient,
     private log = createDebugLogger('aztec:node'),
   ) {
     const message =
@@ -139,10 +143,25 @@ export class AztecNodeService implements AztecNode {
     // start both and wait for them to sync from the block source
     await Promise.all([p2pClient.start(), worldStateSynchronizer.start()]);
 
+    // start the prover if we have been told to
+    const simulationProvider = await getSimulationProvider(config, log);
+    const prover = config.disableProver
+      ? await DummyProver.new()
+      : await TxProver.new(config, worldStateSynchronizer, simulationProvider);
+
     // now create the sequencer
     const sequencer = config.disableSequencer
       ? undefined
-      : await SequencerClient.new(config, p2pClient, worldStateSynchronizer, archiver, archiver, archiver);
+      : await SequencerClient.new(
+          config,
+          p2pClient,
+          worldStateSynchronizer,
+          archiver,
+          archiver,
+          archiver,
+          prover,
+          simulationProvider,
+        );
 
     return new AztecNodeService(
       config,
@@ -158,6 +177,7 @@ export class AztecNodeService implements AztecNode {
       config.version,
       getGlobalVariableBuilder(config),
       store,
+      prover,
       log,
     );
   }
@@ -299,6 +319,7 @@ export class AztecNodeService implements AztecNode {
     await this.p2pClient.stop();
     await this.worldStateSynchronizer.stop();
     await this.blockSource.stop();
+    await this.prover.stop();
     this.log.info(`Stopped`);
   }
 

diff --git a/yarn-project/aztec-node/src/aztec-node/simulator-factory.ts b/yarn-project/aztec-node/src/aztec-node/simulator-factory.ts
@@ -0,0 +1,24 @@
+import { DebugLogger } from '@aztec/foundation/log';
+import { NativeACVMSimulator, SimulationProvider, WASMSimulator } from '@aztec/simulator';
+
+import * as fs from 'fs/promises';
+
+import { AztecNodeConfig } from './config.js';
+
+export async function getSimulationProvider(
+  config: AztecNodeConfig,
+  logger?: DebugLogger,
+): Promise<SimulationProvider> {
+  if (config.acvmBinaryPath && config.acvmWorkingDirectory) {
+    try {
+      await fs.access(config.acvmBinaryPath, fs.constants.R_OK);
+      await fs.mkdir(config.acvmWorkingDirectory, { recursive: true });
+      logger?.(`Using native ACVM at ${config.acvmBinaryPath} and working directory ${config.acvmWorkingDirectory}`);
+      return new NativeACVMSimulator(config.acvmWorkingDirectory, config.acvmBinaryPath);
+    } catch {
+      logger?.(`Failed to access ACVM at ${config.acvmBinaryPath}, falling back to WASM`);
+    }
+  }
+  logger?.('Using WASM ACVM simulation');
+  return new WASMSimulator();
+}
diff --git a/yarn-project/aztec-node/terraform/main.tf b/yarn-project/aztec-node/terraform/main.tf
@@ -155,7 +155,7 @@ resource "aws_ecs_task_definition" "aztec-node" {
   {
     "name": "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}",
     "image": "${var.DOCKERHUB_ACCOUNT}/aztec:${var.DEPLOY_TAG}",
-    "command": ["start", "--node", "--archiver", "--sequencer"],
+    "command": ["start", "--node", "--archiver", "--sequencer", "--prover"],
     "essential": true,
     "memoryReservation": 3776,
     "portMappings": [

diff --git a/yarn-project/aztec-node/tsconfig.json b/yarn-project/aztec-node/tsconfig.json
@@ -33,9 +33,15 @@
     {
       "path": "../p2p"
     },
+    {
+      "path": "../prover-client"
+    },
     {
       "path": "../sequencer-client"
     },
+    {
+      "path": "../simulator"
+    },
     {
       "path": "../types"
     },

diff --git a/yarn-project/aztec/src/cli/cli.ts b/yarn-project/aztec/src/cli/cli.ts
@@ -36,6 +36,7 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command {
     .option('-px, --pxe [options]', cliTexts.pxe)
     .option('-a, --archiver [options]', cliTexts.archiver)
     .option('-s, --sequencer [options]', cliTexts.sequencer)
+    .option('-r, --prover [options]', cliTexts.prover)
     .option('-p2p, --p2p-bootstrap [options]', cliTexts.p2pBootstrap)
     .action(async options => {
       // list of 'stop' functions to call when process ends

diff --git a/yarn-project/aztec/src/cli/cmds/start_node.ts b/yarn-project/aztec/src/cli/cmds/start_node.ts
@@ -59,6 +59,10 @@ export const startNode = async (
     nodeConfig.publisherPrivateKey = `0x${Buffer.from(privKey!).toString('hex')}`;
   }
 
+  if (!options.prover) {
+    nodeConfig.disableProver = true;
+  }
+
   // Create and start Aztec Node.
   const node = await createAztecNode(nodeConfig);
   const nodeServer = createAztecNodeRpcServer(node);

diff --git a/yarn-project/aztec/src/cli/texts.ts b/yarn-project/aztec/src/cli/texts.ts
@@ -63,7 +63,14 @@ export const cliTexts = {
     'requiredConfirmations:SEQ_REQUIRED_CONFIRMATIONS - number - The number of confirmations required before publishing a block. Default: 1\n' +
     'l1BlockPublishRetryIntervalMS:SEQ_PUBLISH_RETRY_INTERVAL_MS - number - The interval in ms to wait before retrying to publish a block. Default: 1000\n' +
     'transactionPollingIntervalMS:SEQ_TX_POLLING_INTERVAL_MS - number - The interval in ms to wait before polling for new transactions. Default: 1000\n' +
+    'acvmBinaryPath:ACVM_BINARY_PATH - string - The full path to an instance of the acvm cli application. If not provided will fallback to WASM circuit simulation\n' +
+    'acvmWorkingDirectory:ACVM_WORKING_DIRECTORY - string - A directory to use for temporary files used by the acvm application. If not provided WASM circuit simulation will be used\n' +
     contractAddresses,
+  prover:
+    'Starts a Prover with options. If started additionally to --node, the Prover will attach to that node.\n' +
+    'Available options are listed below as cliProperty:ENV_VARIABLE_NAME.\n' +
+    'acvmBinaryPath:ACVM_BINARY_PATH - string - The full path to an instance of the acvm cli application. If not provided will fallback to WASM circuit simulation\n' +
+    'acvmWorkingDirectory:ACVM_WORKING_DIRECTORY - string - A directory to use for temporary files used by the acvm application. If not provided WASM circuit simulation will be used\n',
   p2pBootstrap:
     'Starts a P2P bootstrap node with options.\n' +
     'Available options are listed below as cliProperty:ENV_VARIABLE_NAME.\n' +

diff --git a/yarn-project/circuit-types/src/body.ts b/yarn-project/circuit-types/src/body.ts
@@ -95,4 +95,8 @@ export class Body {
 
     return new Body(txEffects);
   }
+
+  static empty() {
+    return new Body([]);
+  }
 }
diff --git a/yarn-project/circuit-types/src/interfaces/block-prover.ts b/yarn-project/circuit-types/src/interfaces/block-prover.ts
@@ -0,0 +1,41 @@
+import { Fr, GlobalVariables, Proof } from '@aztec/circuits.js';
+
+import { L2Block } from '../l2_block.js';
+import { ProcessedTx } from '../tx/processed_tx.js';
+
+export enum PROVING_STATUS {
+  SUCCESS,
+  FAILURE,
+}
+
+export type ProvingSuccess = {
+  status: PROVING_STATUS.SUCCESS;
+  block: L2Block;
+  proof: Proof;
+};
+
+export type ProvingFailure = {
+  status: PROVING_STATUS.FAILURE;
+  reason: string;
+};
+
+export type ProvingResult = ProvingSuccess | ProvingFailure;
+
+export type ProvingTicket = {
+  provingPromise: Promise<ProvingResult>;
+};
+
+/**
+ * The interface to the block prover.
+ * Provides the ability to generate proofs and build rollups.
+ */
+export interface BlockProver {
+  startNewBlock(
+    numTxs: number,
+    globalVariables: GlobalVariables,
+    l1ToL2Messages: Fr[],
+    emptyTx: ProcessedTx,
+  ): Promise<ProvingTicket>;
+
+  addNewTx(tx: ProcessedTx): Promise<void>;
+}
diff --git a/yarn-project/circuit-types/src/interfaces/index.ts b/yarn-project/circuit-types/src/interfaces/index.ts
@@ -5,3 +5,5 @@ export * from './sync-status.js';
 export * from './configs.js';
 export * from './nullifier_tree.js';
 export * from './public_data_tree.js';
+export * from './prover-client.js';
+export * from './block-prover.js';
diff --git a/yarn-project/circuit-types/src/interfaces/prover-client.ts b/yarn-project/circuit-types/src/interfaces/prover-client.ts
@@ -0,0 +1,11 @@
+import { BlockProver } from './block-prover.js';
+
+/**
+ * The interface to the prover client.
+ * Provides the ability to generate proofs and build rollups.
+ */
+export interface ProverClient extends BlockProver {
+  start(): Promise<void>;
+
+  stop(): Promise<void>;
+}
diff --git a/yarn-project/circuit-types/src/l2_block.ts b/yarn-project/circuit-types/src/l2_block.ts
@@ -114,6 +114,18 @@ export class L2Block {
     });
   }
 
+  /**
+   * Creates an L2 block containing empty data.
+   * @returns The L2 block.
+   */
+  static empty(): L2Block {
+    return L2Block.fromFields({
+      archive: AppendOnlyTreeSnapshot.zero(),
+      header: Header.empty(),
+      body: Body.empty(),
+    });
+  }
+
   get number(): number {
     return Number(this.header.globalVariables.blockNumber.toBigInt());
   }

diff --git a/yarn-project/circuit-types/src/tx/index.ts b/yarn-project/circuit-types/src/tx/index.ts
@@ -1,3 +1,4 @@
 export * from './tx.js';
 export * from './tx_hash.js';
 export * from './tx_receipt.js';
+export * from './processed_tx.js';
diff --git a/...ncer-client/src/sequencer/processed_tx.ts → ...ject/circuit-types/src/tx/processed_tx.ts b/...ncer-client/src/sequencer/processed_tx.ts → ...ject/circuit-types/src/tx/processed_tx.ts
diff --git a/yarn-project/circuit-types/src/tx_effect.ts b/yarn-project/circuit-types/src/tx_effect.ts
@@ -142,6 +142,18 @@ export class TxEffect {
     );
   }
 
+  static empty(): TxEffect {
+    return new TxEffect(
+      RevertCode.OK,
+      makeTuple(MAX_NEW_NOTE_HASHES_PER_TX, Fr.zero),
+      makeTuple(MAX_NEW_NULLIFIERS_PER_TX, Fr.zero),
+      makeTuple(MAX_NEW_L2_TO_L1_MSGS_PER_TX, Fr.zero),
+      makeTuple(MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, PublicDataWrite.empty),
+      TxL2Logs.empty(),
+      TxL2Logs.empty(),
+    );
+  }
+
   /**
    * Returns a string representation of the TxEffect object.
    */

diff --git a/yarn-project/circuits.js/package.json b/yarn-project/circuits.js/package.json
@@ -11,7 +11,8 @@
     "./types": "./dest/types/index.js",
     "./constants": "./dest/constants.gen.js",
     "./contract": "./dest/contract/index.js",
-    "./merkle": "./dest/merkle/index.js"
+    "./merkle": "./dest/merkle/index.js",
+    "./simulation": "./dest/simulator/index.js"
   },
   "typedocOptions": {
     "entryPoints": [

diff --git a/yarn-project/circuits.js/src/structs/proof.ts b/yarn-project/circuits.js/src/structs/proof.ts
@@ -1,5 +1,7 @@
 import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize';
 
+const EMPTY_PROOF_SIZE = 42;
+
 /**
  * The Proof class is a wrapper around the circuits proof.
  * Underlying it is a buffer of proof data in a form a barretenberg prover understands.
@@ -47,5 +49,5 @@ export class Proof {
  * @returns The empty "proof".
  */
 export function makeEmptyProof() {
-  return new Proof(Buffer.alloc(0));
+  return new Proof(Buffer.alloc(EMPTY_PROOF_SIZE, 0));
 }
diff --git a/yarn-project/end-to-end/package.json b/yarn-project/end-to-end/package.json
@@ -32,8 +32,10 @@
     "@aztec/noir-contracts.js": "workspace:^",
     "@aztec/p2p": "workspace:^",
     "@aztec/protocol-contracts": "workspace:^",
+    "@aztec/prover-client": "workspace:^",
     "@aztec/pxe": "workspace:^",
     "@aztec/sequencer-client": "workspace:^",
+    "@aztec/simulator": "workspace:^",
     "@aztec/types": "workspace:^",
     "@aztec/world-state": "workspace:^",
     "@jest/globals": "^29.5.0",
@@ -50,6 +52,7 @@
     "crypto-browserify": "^3.12.0",
     "glob": "^10.3.10",
     "jest": "^29.5.0",
+    "jest-mock-extended": "^3.0.5",
     "koa": "^2.14.2",
     "koa-static": "^5.0.0",
     "levelup": "^5.1.1",