From 3eef111a906c04af2c110007a0bc4057086172d0 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Fri, 26 Apr 2024 10:39:37 +0000 Subject: [PATCH] feat: proving benchmark --- yarn-project/Earthfile | 5 +- .../aztec-node/src/aztec-node/config.ts | 4 +- .../aztec-node/src/aztec-node/server.ts | 5 +- .../aztec/src/cli/cmds/start_prover.ts | 4 + .../src/interfaces/aztec-node.ts | 3 +- .../src/interfaces/prover-client.ts | 10 ++ yarn-project/end-to-end/Earthfile | 8 ++ .../src/benchmarks/bench_proving.test.ts | 124 ++++++++++++++++++ .../client_prover_test.ts | 19 ++- .../src/fixtures/get_acvm_config.ts | 17 ++- .../end-to-end/src/fixtures/get_bb_config.ts | 46 +++++++ .../src/fixtures/snapshot_manager.ts | 4 +- yarn-project/end-to-end/src/fixtures/utils.ts | 78 +++-------- yarn-project/prover-client/src/bb/execute.ts | 8 +- yarn-project/prover-client/src/config.ts | 18 ++- .../prover-client/src/dummy-prover.ts | 5 + yarn-project/prover-client/src/index.ts | 2 + .../src/prover-pool/prover-agent.ts | 7 +- .../src/prover-pool/prover-pool.ts | 31 +++-- .../prover-client/src/tx-prover/tx-prover.ts | 15 ++- 20 files changed, 299 insertions(+), 114 deletions(-) create mode 100644 yarn-project/end-to-end/src/benchmarks/bench_proving.test.ts create mode 100644 yarn-project/end-to-end/src/fixtures/get_bb_config.ts diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile index 0c6c1f9a1e7..fc3b21deccf 100644 --- a/yarn-project/Earthfile +++ b/yarn-project/Earthfile @@ -5,7 +5,8 @@ deps: LET packages = $(git ls-files "**/package*.json" package*.json) LET tsconfigs = $(git ls-files "**/tsconfig*.json" tsconfig*.json) FROM ../build-images+build - # copy bb-js and noir-packages + # copy bb, bb-js and noir-packages + COPY ../barretenberg/cpp/+preset-release/bin /usr/src/barretenberg/cpp/build/ COPY ../barretenberg/ts/+build/build /usr/src/barretenberg/ts COPY ../noir/+packages/packages /usr/src/noir/packages WORKDIR /usr/src/yarn-project @@ -100,7 +101,7 @@ end-to-end: RUN apt-get update && apt-get install -y wget gnupg \ && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ && echo "deb [arch=$(dpkg --print-architecture)] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list \ - && apt update && apt install nodejs jq google-chrome-stable netcat-openbsd -y \ + && apt update && apt install curl nodejs jq google-chrome-stable netcat-openbsd -y \ && rm -rf /var/lib/apt/lists/* ENV CHROME_BIN="/usr/bin/google-chrome-stable" ENV PATH=/opt/foundry/bin:$PATH diff --git a/yarn-project/aztec-node/src/aztec-node/config.ts b/yarn-project/aztec-node/src/aztec-node/config.ts index dba7d824025..8c00246d608 100644 --- a/yarn-project/aztec-node/src/aztec-node/config.ts +++ b/yarn-project/aztec-node/src/aztec-node/config.ts @@ -1,6 +1,6 @@ import { type ArchiverConfig, getConfigEnvVars as getArchiverVars } from '@aztec/archiver'; import { type P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p'; -import { type ProverConfig, getProverEnvVars } from '@aztec/prover-client'; +import { type ProverClientConfig, getProverEnvVars } from '@aztec/prover-client'; import { type SequencerClientConfig, getConfigEnvVars as getSequencerVars } from '@aztec/sequencer-client'; import { getConfigEnvVars as getWorldStateVars } from '@aztec/world-state'; @@ -9,7 +9,7 @@ import { getConfigEnvVars as getWorldStateVars } from '@aztec/world-state'; */ export type AztecNodeConfig = ArchiverConfig & SequencerClientConfig & - ProverConfig & + ProverClientConfig & P2PConfig & { /** Whether the sequencer is disabled for this node. */ disableSequencer: boolean; diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 343130a1b38..4028ddd3594 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -15,6 +15,7 @@ import { NullifierMembershipWitness, type ProcessOutput, type ProverClient, + type ProverConfig, PublicDataWitness, type SequencerConfig, type SiblingPath, @@ -688,9 +689,9 @@ export class AztecNodeService implements AztecNode { }; } - public setConfig(config: Partial): Promise { + public async setConfig(config: Partial): Promise { this.sequencer?.updateSequencerConfig(config); - return Promise.resolve(); + await this.prover.updateProverConfig(config); } /** diff --git a/yarn-project/aztec/src/cli/cmds/start_prover.ts b/yarn-project/aztec/src/cli/cmds/start_prover.ts index 103ca97c8df..7c39fe6e16a 100644 --- a/yarn-project/aztec/src/cli/cmds/start_prover.ts +++ b/yarn-project/aztec/src/cli/cmds/start_prover.ts @@ -1,6 +1,8 @@ import { type ProvingJobSource } from '@aztec/circuit-types'; import { ProverPool, createProvingJobSourceClient } from '@aztec/prover-client/prover-pool'; +import { tmpdir } from 'node:os'; + import { type ServiceStarter, parseModuleOptions } from '../util.js'; type ProverOptions = Partial<{ @@ -35,6 +37,8 @@ export const startProver: ServiceStarter = async (options, signalHandlers, logge { acvmBinaryPath: proverOptions.acvmBinaryPath, bbBinaryPath: proverOptions.bbBinaryPath, + acvmWorkingDirectory: tmpdir(), + bbWorkingDirectory: tmpdir(), }, agentCount, ); diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.ts index 95cc81d5bc2..fd8b71c1126 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.ts @@ -26,6 +26,7 @@ import { type TxEffect } from '../tx_effect.js'; import { type SequencerConfig } from './configs.js'; import { type L2BlockNumber } from './l2_block_number.js'; import { type NullifierMembershipWitness } from './nullifier_tree.js'; +import { type ProverConfig } from './prover-client.js'; import { type PublicDataWitness } from './public_data_tree.js'; /** @@ -288,7 +289,7 @@ export interface AztecNode { * Updates the configuration of this node. * @param config - Updated configuration to be merged with the current one. */ - setConfig(config: Partial): Promise; + setConfig(config: Partial): Promise; /** * Returns a registered contract class given its id. diff --git a/yarn-project/circuit-types/src/interfaces/prover-client.ts b/yarn-project/circuit-types/src/interfaces/prover-client.ts index 8e55d3a2dbb..6ce183fc385 100644 --- a/yarn-project/circuit-types/src/interfaces/prover-client.ts +++ b/yarn-project/circuit-types/src/interfaces/prover-client.ts @@ -1,6 +1,14 @@ import { type BlockProver } from './block-prover.js'; import { type ProvingJobSource } from './proving-job.js'; +/** + * The prover configuration. + */ +export type ProverConfig = { + /** How many agents to run */ + proverAgents: number; +}; + /** * The interface to the prover client. * Provides the ability to generate proofs and build rollups. @@ -11,4 +19,6 @@ export interface ProverClient extends BlockProver { stop(): Promise; getProvingJobSource(): ProvingJobSource; + + updateProverConfig(config: Partial): Promise; } diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index 7085b410cc5..c7b91115513 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -103,3 +103,11 @@ bench-tx-size: ARG COMMIT_HASH DO +E2E_COMPOSE_TEST --test=benchmarks/bench_tx_size_fees.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --enable_gas=1 --compose_file=./scripts/docker-compose-no-sandbox.yml DO +UPLOAD_LOGS --e2e_mode=$e2e_mode --PULL_REQUEST=$PULL_REQUEST --BRANCH=$BRANCH --COMMIT_HASH=$COMMIT_HASH + +bench-proving: + ARG e2e_mode=local + ARG PULL_REQUEST + ARG BRANCH + ARG COMMIT_HASH + DO +E2E_COMPOSE_TEST --test=bench_proving --debug="aztec:benchmarks:*,aztec:prover*,aztec:bb*" --e2e_mode=$e2e_mode --enable_gas=1 --compose_file=./scripts/docker-compose-no-sandbox.yml + DO +UPLOAD_LOGS --e2e_mode=$e2e_mode --PULL_REQUEST=$PULL_REQUEST --BRANCH=$BRANCH --COMMIT_HASH=$COMMIT_HASH diff --git a/yarn-project/end-to-end/src/benchmarks/bench_proving.test.ts b/yarn-project/end-to-end/src/benchmarks/bench_proving.test.ts new file mode 100644 index 00000000000..954c88c2653 --- /dev/null +++ b/yarn-project/end-to-end/src/benchmarks/bench_proving.test.ts @@ -0,0 +1,124 @@ +import { type AztecNodeService } from '@aztec/aztec-node'; +import { type AccountWallet, EthAddress, PublicFeePaymentMethod, TxStatus } from '@aztec/aztec.js'; +import { GasSettings } from '@aztec/circuits.js'; +import { FPCContract, GasTokenContract, TestContract, TokenContract } from '@aztec/noir-contracts.js'; +import { getCanonicalGasTokenAddress } from '@aztec/protocol-contracts/gas-token'; +import { ProverPool } from '@aztec/prover-client/prover-pool'; + +import { jest } from '@jest/globals'; + +import { getACVMConfig } from '../fixtures/get_acvm_config.js'; +import { getBBConfig } from '../fixtures/get_bb_config.js'; +import { type EndToEndContext, publicDeployAccounts, setup } from '../fixtures/utils.js'; + +jest.setTimeout(600_000); + +const txTimeoutSec = 600; + +describe('benchmarks/proving', () => { + let ctx: EndToEndContext; + let wallet: AccountWallet; + let testContract: TestContract; + let tokenContract: TokenContract; + let fpContract: FPCContract; + let acvmCleanup: () => Promise; + let bbCleanup: () => Promise; + let proverPool: ProverPool; + + // setup the environment quickly using fake proofs + beforeAll(async () => { + ctx = await setup( + 1, + { + // do setup with fake proofs + realProofs: false, + proverAgents: 4, + proverAgentPollInterval: 10, + minTxsPerBlock: 1, + }, + {}, + true, // enable gas + ); + + wallet = ctx.wallet; + + await publicDeployAccounts(wallet, ctx.wallets); + + testContract = await TestContract.deploy(wallet).send().deployed(); + tokenContract = await TokenContract.deploy(wallet, wallet.getAddress(), 'test', 't', 18).send().deployed(); + const gas = await GasTokenContract.at( + getCanonicalGasTokenAddress(ctx.deployL1ContractsValues.l1ContractAddresses.gasPortalAddress), + wallet, + ); + fpContract = await FPCContract.deploy(wallet, tokenContract.address, gas.address).send().deployed(); + + await Promise.all([ + gas.methods.mint_public(fpContract.address, 1e12).send().wait(), + tokenContract.methods.mint_public(wallet.getAddress(), 1e12).send().wait(), + ]); + }); + + // remove the fake prover and setup the real one + beforeAll(async () => { + const [acvmConfig, bbConfig] = await Promise.all([getACVMConfig(ctx.logger), getBBConfig(ctx.logger)]); + if (!acvmConfig || !bbConfig) { + throw new Error('Missing ACVM or BB config'); + } + + acvmCleanup = acvmConfig.cleanup; + bbCleanup = bbConfig.cleanup; + + proverPool = ProverPool.nativePool( + { + ...acvmConfig, + ...bbConfig, + }, + 4, + 10, + ); + + ctx.logger.info('Stopping fake provers'); + await ctx.aztecNode.setConfig({ + // stop the fake provers + proverAgents: 0, + // 4-tx blocks so that we have at least one merge level + minTxsPerBlock: 4, + }); + + ctx.logger.info('Starting real provers'); + await proverPool.start((ctx.aztecNode as AztecNodeService).getProver().getProvingJobSource()); + }); + + afterAll(async () => { + await proverPool.stop(); + await ctx.teardown(); + await acvmCleanup(); + await bbCleanup(); + }); + + it('builds a full block', async () => { + const txs = [ + // fully private tx + testContract.methods.emit_nullifier(42).send(), + // tx with setup, app, teardown + testContract.methods.emit_unencrypted(43).send({ + fee: { + gasSettings: GasSettings.default(), + paymentMethod: new PublicFeePaymentMethod(tokenContract.address, fpContract.address, wallet), + }, + }), + // tx with messages + testContract.methods.create_l2_to_l1_message_public(45, 46, EthAddress.random()).send(), + // tx with private and public exec + testContract.methods.set_tx_max_block_number(100, true).send({ + fee: { + gasSettings: GasSettings.default(), + paymentMethod: new PublicFeePaymentMethod(tokenContract.address, fpContract.address, wallet), + }, + }), + ]; + + const receipts = await Promise.all(txs.map(tx => tx.wait({ timeout: txTimeoutSec }))); + expect(receipts.every(r => r.status === TxStatus.MINED)).toBe(true); + }); +}); diff --git a/yarn-project/end-to-end/src/client_prover_integration/client_prover_test.ts b/yarn-project/end-to-end/src/client_prover_integration/client_prover_test.ts index 346147d92d0..0303dcecdab 100644 --- a/yarn-project/end-to-end/src/client_prover_integration/client_prover_test.ts +++ b/yarn-project/end-to-end/src/client_prover_integration/client_prover_test.ts @@ -15,9 +15,8 @@ import { import { TokenContract } from '@aztec/noir-contracts.js'; import { BBNativeProofCreator, type PXEService } from '@aztec/pxe'; -import * as fs from 'fs/promises'; - import { waitRegisteredAccountSynced } from '../benchmarks/utils.js'; +import { getBBConfig } from '../fixtures/get_bb_config.js'; import { type ISnapshotManager, type SubsystemsContext, @@ -25,7 +24,7 @@ import { createSnapshotManager, publicDeployAccounts, } from '../fixtures/snapshot_manager.js'; -import { getBBConfig, setupPXEService } from '../fixtures/utils.js'; +import { setupPXEService } from '../fixtures/utils.js'; import { TokenSimulator } from '../simulators/token_simulator.js'; const { E2E_DATA_PATH: dataPath } = process.env; @@ -55,7 +54,7 @@ export class ClientProverTest { fullProverPXE!: PXEService; provenAsset!: TokenContract; provenPXETeardown?: () => Promise; - private directoryToCleanup?: string; + private bbConfigCleanup?: () => Promise; proofCreator?: BBNativeProofCreator; constructor(testName: string) { @@ -121,13 +120,13 @@ export class ClientProverTest { // Configure a full prover PXE const bbConfig = await getBBConfig(this.logger); - this.directoryToCleanup = bbConfig?.directoryToCleanup; + this.bbConfigCleanup = bbConfig?.cleanup; - if (!bbConfig?.bbWorkingDirectory || !bbConfig?.expectedBBPath) { + if (!bbConfig?.bbWorkingDirectory || !bbConfig?.bbBinaryPath) { throw new Error(`Test must be run with BB native configuration`); } - this.proofCreator = new BBNativeProofCreator(bbConfig?.expectedBBPath, bbConfig?.bbWorkingDirectory); + this.proofCreator = new BBNativeProofCreator(bbConfig.bbBinaryPath, bbConfig.bbWorkingDirectory); this.logger.debug(`Main setup completed, initializing full prover PXE...`); ({ pxe: this.fullProverPXE, teardown: this.provenPXETeardown } = await setupPXEService( @@ -135,7 +134,7 @@ export class ClientProverTest { this.aztecNode, { proverEnabled: false, - bbBinaryPath: bbConfig?.expectedBBPath, + bbBinaryPath: bbConfig?.bbBinaryPath, bbWorkingDirectory: bbConfig?.bbWorkingDirectory, }, undefined, @@ -180,9 +179,7 @@ export class ClientProverTest { // Cleanup related to the second 'full prover' PXE await this.provenPXETeardown?.(); - if (this.directoryToCleanup) { - await fs.rm(this.directoryToCleanup, { recursive: true, force: true }); - } + await this.bbConfigCleanup?.(); } async addPendingShieldNoteToPXE(accountIndex: number, amount: bigint, secretHash: Fr, txHash: TxHash) { diff --git a/yarn-project/end-to-end/src/fixtures/get_acvm_config.ts b/yarn-project/end-to-end/src/fixtures/get_acvm_config.ts index 55672641113..a8c8349a6ce 100644 --- a/yarn-project/end-to-end/src/fixtures/get_acvm_config.ts +++ b/yarn-project/end-to-end/src/fixtures/get_acvm_config.ts @@ -13,14 +13,21 @@ const { } = process.env; // Determines if we have access to the acvm binary and a tmp folder for temp files -export async function getACVMConfig(logger: DebugLogger) { +export async function getACVMConfig(logger: DebugLogger): Promise< + | { + acvmWorkingDirectory: string; + acvmBinaryPath: string; + cleanup: () => Promise; + } + | undefined +> { try { - const expectedAcvmPath = ACVM_BINARY_PATH ? ACVM_BINARY_PATH : `../../noir/${NOIR_RELEASE_DIR}/acvm`; - await fs.access(expectedAcvmPath, fs.constants.R_OK); + const acvmBinaryPath = ACVM_BINARY_PATH ? ACVM_BINARY_PATH : `../../noir/${NOIR_RELEASE_DIR}/acvm`; + await fs.access(acvmBinaryPath, fs.constants.R_OK); const tempWorkingDirectory = `${TEMP_DIR}/${randomBytes(4).toString('hex')}`; const acvmWorkingDirectory = ACVM_WORKING_DIRECTORY ? ACVM_WORKING_DIRECTORY : `${tempWorkingDirectory}/acvm`; await fs.mkdir(acvmWorkingDirectory, { recursive: true }); - logger.verbose(`Using native ACVM binary at ${expectedAcvmPath} with working directory ${acvmWorkingDirectory}`); + logger.verbose(`Using native ACVM binary at ${acvmBinaryPath} with working directory ${acvmWorkingDirectory}`); const directoryToCleanup = ACVM_WORKING_DIRECTORY ? undefined : tempWorkingDirectory; @@ -33,7 +40,7 @@ export async function getACVMConfig(logger: DebugLogger) { return { acvmWorkingDirectory, - expectedAcvmPath, + acvmBinaryPath, cleanup, }; } catch (err) { diff --git a/yarn-project/end-to-end/src/fixtures/get_bb_config.ts b/yarn-project/end-to-end/src/fixtures/get_bb_config.ts new file mode 100644 index 00000000000..412c9316457 --- /dev/null +++ b/yarn-project/end-to-end/src/fixtures/get_bb_config.ts @@ -0,0 +1,46 @@ +import { type DebugLogger, fileURLToPath } from '@aztec/aztec.js'; + +import fs from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'path'; + +const { + BB_RELEASE_DIR = 'barretenberg/cpp/build/bin', + BB_BINARY_PATH, + TEMP_DIR = tmpdir(), + BB_WORKING_DIRECTORY = '', +} = process.env; + +export const getBBConfig = async ( + logger: DebugLogger, +): Promise<{ bbBinaryPath: string; bbWorkingDirectory: string; cleanup: () => Promise } | undefined> => { + try { + const bbBinaryPath = + BB_BINARY_PATH ?? + path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../../', BB_RELEASE_DIR, 'bb'); + await fs.access(bbBinaryPath, fs.constants.R_OK); + + let bbWorkingDirectory: string; + let directoryToCleanup: string | undefined; + + if (BB_WORKING_DIRECTORY) { + bbWorkingDirectory = BB_WORKING_DIRECTORY; + } else { + bbWorkingDirectory = await fs.mkdtemp(path.join(TEMP_DIR, 'bb-')); + directoryToCleanup = bbWorkingDirectory; + } + + await fs.mkdir(bbWorkingDirectory, { recursive: true }); + + const cleanup = async () => { + if (directoryToCleanup) { + await fs.rm(directoryToCleanup, { recursive: true, force: true }); + } + }; + + return { bbBinaryPath, bbWorkingDirectory, cleanup }; + } catch (err) { + logger.error(`Native BB not available, error: ${err}`); + return undefined; + } +}; diff --git a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts index 2f1e6740527..72651f79e27 100644 --- a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts +++ b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts @@ -253,7 +253,7 @@ async function setupFromFresh(statePath: string | undefined, logger: Logger): Pr const acvmConfig = await getACVMConfig(logger); if (acvmConfig) { aztecNodeConfig.acvmWorkingDirectory = acvmConfig.acvmWorkingDirectory; - aztecNodeConfig.acvmBinaryPath = acvmConfig.expectedAcvmPath; + aztecNodeConfig.acvmBinaryPath = acvmConfig.acvmBinaryPath; } logger.verbose('Creating and synching an aztec node...'); @@ -305,7 +305,7 @@ async function setupFromState(statePath: string, logger: Logger): Promise { return PXE_URL; }; -// Determines if we have access to the acvm binary and a tmp folder for temp files -const getACVMConfig = async (logger: DebugLogger) => { - try { - const expectedAcvmPath = ACVM_BINARY_PATH - ? ACVM_BINARY_PATH - : `${path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../../noir/', NOIR_RELEASE_DIR)}/acvm`; - await fs.access(expectedAcvmPath, fs.constants.R_OK); - const tempWorkingDirectory = `${TEMP_DIR}/${randomBytes(4).toString('hex')}`; - const acvmWorkingDirectory = ACVM_WORKING_DIRECTORY ? ACVM_WORKING_DIRECTORY : `${tempWorkingDirectory}/acvm`; - await fs.mkdir(acvmWorkingDirectory, { recursive: true }); - logger.info(`Using native ACVM binary at ${expectedAcvmPath} with working directory ${acvmWorkingDirectory}`); - return { - acvmWorkingDirectory, - expectedAcvmPath, - directoryToCleanup: ACVM_WORKING_DIRECTORY ? undefined : tempWorkingDirectory, - }; - } catch (err) { - logger.error(`Native ACVM not available, error: ${err}`); - return undefined; - } -}; - -// Determines if we have access to the bb binary and a tmp folder for temp files -export const getBBConfig = async (logger: DebugLogger) => { - try { - const expectedBBPath = BB_BINARY_PATH - ? BB_BINARY_PATH - : `${path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../../barretenberg/', BB_RELEASE_DIR)}/bb`; - await fs.access(expectedBBPath, fs.constants.R_OK); - const tempWorkingDirectory = `${TEMP_DIR}/${randomBytes(4).toString('hex')}`; - const bbWorkingDirectory = BB_WORKING_DIRECTORY ? BB_WORKING_DIRECTORY : `${tempWorkingDirectory}/bb`; - await fs.mkdir(bbWorkingDirectory, { recursive: true }); - logger.info(`Using native BB binary at ${expectedBBPath} with working directory ${bbWorkingDirectory}`); - return { - bbWorkingDirectory, - expectedBBPath, - directoryToCleanup: BB_WORKING_DIRECTORY ? undefined : tempWorkingDirectory, - }; - } catch (err) { - logger.error(`Native BB not available, error: ${err}`); - return undefined; - } -}; - export const setupL1Contracts = async ( l1RpcUrl: string, account: HDAccount | PrivateKeyAccount, @@ -312,7 +258,7 @@ async function setupWithRemoteEnvironment( const { chainId, protocolVersion } = await pxeClient.getNodeInfo(); // this contract might already have been deployed - // the following deployin functions are idempotent + // the following deploying functions are idempotent await deployCanonicalKeyRegistry( new SignerlessWallet(pxeClient, new DefaultMultiCallEntrypoint(chainId, protocolVersion)), ); @@ -326,6 +272,7 @@ async function setupWithRemoteEnvironment( return { aztecNode, sequencer: undefined, + prover: undefined, pxe: pxeClient, deployL1ContractsValues, accounts: await pxeClient!.getRegisteredAccounts(), @@ -366,6 +313,8 @@ export type EndToEndContext = { logger: DebugLogger; /** The cheat codes. */ cheatCodes: CheatCodes; + /** Proving jobs */ + prover: ProverClient | undefined; /** Function to stop the started services. */ teardown: () => Promise; }; @@ -383,6 +332,7 @@ export async function setup( enableGas = false, ): Promise { const config = { ...getConfigEnvVars(), ...opts }; + const logger = getLogger(); let anvil: Anvil | undefined; @@ -413,6 +363,7 @@ export async function setup( // Enable logging metrics to a local file named after the test suite if (isMetricsLoggingRequested()) { const filename = path.join('log', getJobName() + '.jsonl'); + logger.info(`Logging metrics to ${filename}`); setupMetricsLogger(filename); } @@ -421,7 +372,6 @@ export async function setup( await ethCheatCodes.loadChainState(opts.stateLoad); } - const logger = getLogger(); const hdAccount = mnemonicToAccount(MNEMONIC); const privKeyRaw = hdAccount.getHdKey().privateKey; const publisherPrivKey = privKeyRaw === null ? null : Buffer.from(privKeyRaw); @@ -442,11 +392,12 @@ export async function setup( const acvmConfig = await getACVMConfig(logger); if (acvmConfig) { config.acvmWorkingDirectory = acvmConfig.acvmWorkingDirectory; - config.acvmBinaryPath = acvmConfig.expectedAcvmPath; + config.acvmBinaryPath = acvmConfig.acvmBinaryPath; } config.l1BlockPublishRetryIntervalMS = 100; const aztecNode = await AztecNodeService.createAndSync(config); const sequencer = aztecNode.getSequencer(); + const prover = aztecNode.getProver(); logger.verbose('Creating a pxe...'); const { pxe, wallets } = await setupPXEService(numberOfAccounts, aztecNode!, pxeOpts, logger); @@ -473,10 +424,10 @@ export async function setup( await pxe?.stop(); } - if (acvmConfig?.directoryToCleanup) { + if (acvmConfig?.cleanup) { // remove the temp directory created for the acvm - logger.verbose(`Cleaning up ACVM temp directory ${acvmConfig.directoryToCleanup}`); - await fs.rm(acvmConfig.directoryToCleanup, { recursive: true, force: true }); + logger.verbose(`Cleaning up ACVM state`); + await acvmConfig.cleanup(); } await anvil?.stop(); @@ -492,6 +443,7 @@ export async function setup( logger, cheatCodes, sequencer, + prover, teardown, }; } diff --git a/yarn-project/prover-client/src/bb/execute.ts b/yarn-project/prover-client/src/bb/execute.ts index f53950dd0f5..58a7cb968a6 100644 --- a/yarn-project/prover-client/src/bb/execute.ts +++ b/yarn-project/prover-client/src/bb/execute.ts @@ -50,16 +50,16 @@ export function executeBB( ) { return new Promise((resolve, reject) => { // spawn the bb process - const acvm = proc.spawn(pathToBB, [command, ...args]); - acvm.stdout.on('data', data => { + const bb = proc.spawn(pathToBB, [command, ...args]); + bb.stdout.on('data', data => { const message = data.toString('utf-8').replace(/\n$/, ''); logger(message); }); - acvm.stderr.on('data', data => { + bb.stderr.on('data', data => { const message = data.toString('utf-8').replace(/\n$/, ''); logger(message); }); - acvm.on('close', (code: number) => { + bb.on('close', (code: number) => { if (resultParser(code)) { resolve(BB_RESULT.SUCCESS); } else { diff --git a/yarn-project/prover-client/src/config.ts b/yarn-project/prover-client/src/config.ts index 0b8f7cce6ea..8549e60b2aa 100644 --- a/yarn-project/prover-client/src/config.ts +++ b/yarn-project/prover-client/src/config.ts @@ -1,9 +1,11 @@ +import { type ProverConfig } from '@aztec/circuit-types'; + import { tmpdir } from 'os'; /** * The prover configuration. */ -export interface ProverConfig { +export type ProverClientConfig = ProverConfig & { /** The working directory to use for simulation/proving */ acvmWorkingDirectory: string; /** The path to the ACVM binary */ @@ -12,29 +14,34 @@ export interface ProverConfig { bbWorkingDirectory: string; /** The path to the bb binary */ bbBinaryPath: string; - /** How many agents to start */ - proverAgents: number; /** Enable proving. If true, must set bb env vars */ realProofs: boolean; -} + /** The interval agents poll for jobs at */ + proverAgentPollInterval: number; +}; /** * Returns the prover configuration from the environment variables. * Note: If an environment variable is not set, the default value is used. * @returns The prover configuration. */ -export function getProverEnvVars(): ProverConfig { +export function getProverEnvVars(): ProverClientConfig { const { ACVM_WORKING_DIRECTORY = tmpdir(), ACVM_BINARY_PATH = '', BB_WORKING_DIRECTORY = tmpdir(), BB_BINARY_PATH = '', PROVER_AGENTS = '1', + PROVER_AGENT_POLL_INTERVAL_MS = '50', PROVER_REAL_PROOFS = '', } = process.env; const parsedProverAgents = parseInt(PROVER_AGENTS, 10); const proverAgents = Number.isSafeInteger(parsedProverAgents) ? parsedProverAgents : 0; + const parsedProverAgentPollInterval = parseInt(PROVER_AGENT_POLL_INTERVAL_MS, 10); + const proverAgentPollInterval = Number.isSafeInteger(parsedProverAgentPollInterval) + ? parsedProverAgentPollInterval + : 50; return { acvmWorkingDirectory: ACVM_WORKING_DIRECTORY, @@ -43,5 +50,6 @@ export function getProverEnvVars(): ProverConfig { bbWorkingDirectory: BB_WORKING_DIRECTORY, proverAgents, realProofs: ['1', 'true'].includes(PROVER_REAL_PROOFS), + proverAgentPollInterval, }; } diff --git a/yarn-project/prover-client/src/dummy-prover.ts b/yarn-project/prover-client/src/dummy-prover.ts index e8c76d009e5..4a912e09119 100644 --- a/yarn-project/prover-client/src/dummy-prover.ts +++ b/yarn-project/prover-client/src/dummy-prover.ts @@ -4,6 +4,7 @@ import { PROVING_STATUS, type ProcessedTx, type ProverClient, + type ProverConfig, type ProvingJob, type ProvingJobSource, type ProvingRequest, @@ -63,6 +64,10 @@ export class DummyProver implements ProverClient { getProvingJobSource(): ProvingJobSource { return this.jobs; } + + updateProverConfig(_config: Partial): Promise { + return Promise.resolve(); + } } class DummyProvingJobSource implements ProvingJobSource { diff --git a/yarn-project/prover-client/src/index.ts b/yarn-project/prover-client/src/index.ts index c47f1852f99..4331fcaff0e 100644 --- a/yarn-project/prover-client/src/index.ts +++ b/yarn-project/prover-client/src/index.ts @@ -1,3 +1,5 @@ +export { ProverClient } from '@aztec/circuit-types'; + export * from './tx-prover/tx-prover.js'; export * from './config.js'; export * from './dummy-prover.js'; diff --git a/yarn-project/prover-client/src/prover-pool/prover-agent.ts b/yarn-project/prover-client/src/prover-pool/prover-agent.ts index 40179596937..40ff2f22fcf 100644 --- a/yarn-project/prover-client/src/prover-pool/prover-agent.ts +++ b/yarn-project/prover-client/src/prover-pool/prover-agent.ts @@ -52,15 +52,18 @@ export class ProverAgent { }, this.intervalMs); this.runningPromise.start(); + this.log.info('Agent started'); } async stop(): Promise { - if (!this.runningPromise) { - throw new Error('Agent is not running'); + if (!this.runningPromise?.isRunning()) { + return; } await this.runningPromise.stop(); this.runningPromise = undefined; + + this.log.info('Agent stopped'); } private work(request: ProvingRequest): Promise> { diff --git a/yarn-project/prover-client/src/prover-pool/prover-pool.ts b/yarn-project/prover-client/src/prover-pool/prover-pool.ts index 4916e3e98a4..8dc7a320ff4 100644 --- a/yarn-project/prover-client/src/prover-pool/prover-pool.ts +++ b/yarn-project/prover-client/src/prover-pool/prover-pool.ts @@ -3,7 +3,6 @@ import { sleep } from '@aztec/foundation/sleep'; import { type SimulationProvider } from '@aztec/simulator'; import { mkdtemp } from 'fs/promises'; -import { tmpdir } from 'os'; import { join } from 'path'; import { BBNativeRollupProver, type BBProverConfig } from '../prover/bb_prover.js'; @@ -41,7 +40,7 @@ export class ProverPool { async stop(): Promise { if (!this.running) { - throw new Error('Prover pool is not running'); + return; } for (const agent of this.agents) { @@ -51,6 +50,20 @@ export class ProverPool { this.running = false; } + async rescale(newSize: number): Promise { + if (newSize > this.size) { + this.size = newSize; + for (let i = this.agents.length; i < newSize; i++) { + this.agents.push(await this.agentFactory(i)); + } + } else if (newSize < this.size) { + this.size = newSize; + while (this.agents.length > newSize) { + await this.agents.pop()?.stop(); + } + } + } + static testPool(simulationProvider?: SimulationProvider, size = 1, agentPollIntervalMS = 10): ProverPool { return new ProverPool( size, @@ -58,22 +71,18 @@ export class ProverPool { ); } - static nativePool( - { acvmBinaryPath, bbBinaryPath }: Pick, - size: number, - agentPollIntervalMS = 10, - ): ProverPool { + static nativePool(config: Omit, size: number, agentPollIntervalMS = 10): ProverPool { // TODO generate keys ahead of time so that each agent doesn't have to do it return new ProverPool(size, async i => { const [acvmWorkingDirectory, bbWorkingDirectory] = await Promise.all([ - mkdtemp(join(tmpdir(), 'acvm-')), - mkdtemp(join(tmpdir(), 'bb-')), + mkdtemp(join(config.acvmWorkingDirectory, 'agent-')), + mkdtemp(join(config.bbWorkingDirectory, 'agent-')), ]); return new ProverAgent( await BBNativeRollupProver.new({ - acvmBinaryPath, + acvmBinaryPath: config.acvmBinaryPath, acvmWorkingDirectory, - bbBinaryPath, + bbBinaryPath: config.bbBinaryPath, bbWorkingDirectory, }), agentPollIntervalMS, diff --git a/yarn-project/prover-client/src/tx-prover/tx-prover.ts b/yarn-project/prover-client/src/tx-prover/tx-prover.ts index 88f987e8124..a34716a682d 100644 --- a/yarn-project/prover-client/src/tx-prover/tx-prover.ts +++ b/yarn-project/prover-client/src/tx-prover/tx-prover.ts @@ -2,6 +2,7 @@ import { type ProcessedTx } from '@aztec/circuit-types'; import { type BlockResult, type ProverClient, + type ProverConfig, type ProvingJobSource, type ProvingTicket, } from '@aztec/circuit-types/interfaces'; @@ -9,7 +10,7 @@ import { type Fr, type GlobalVariables } from '@aztec/circuits.js'; import { type SimulationProvider } from '@aztec/simulator'; import { type WorldStateSynchronizer } from '@aztec/world-state'; -import { type ProverConfig } from '../config.js'; +import { type ProverClientConfig } from '../config.js'; import { type VerificationKeys, getVerificationKeys } from '../mocks/verification_keys.js'; import { ProvingOrchestrator } from '../orchestrator/orchestrator.js'; import { MemoryProvingQueue } from '../prover-pool/memory-proving-queue.js'; @@ -30,6 +31,12 @@ export class TxProver implements ProverClient { this.orchestrator = new ProvingOrchestrator(worldStateSynchronizer.getLatest(), this.queue); } + async updateProverConfig(config: Partial): Promise { + if (typeof config.proverAgents === 'number') { + await this.proverPool?.rescale(config.proverAgents); + } + } + /** * Starts the prover instance */ @@ -51,7 +58,7 @@ export class TxProver implements ProverClient { * @returns An instance of the prover, constructed and started. */ public static async new( - config: ProverConfig, + config: ProverClientConfig, simulationProvider: SimulationProvider, worldStateSynchronizer: WorldStateSynchronizer, ) { @@ -68,9 +75,9 @@ export class TxProver implements ProverClient { throw new Error(); } - pool = ProverPool.nativePool(config, config.proverAgents, 50); + pool = ProverPool.nativePool(config, config.proverAgents, config.proverAgentPollInterval); } else { - pool = ProverPool.testPool(simulationProvider, config.proverAgents, 50); + pool = ProverPool.testPool(simulationProvider, config.proverAgents, config.proverAgentPollInterval); } const prover = new TxProver(worldStateSynchronizer, getVerificationKeys(), pool);