From 400d3254eef183cf1e104cb46661484f04e4697a Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Fri, 26 Apr 2024 10:39:37 +0000 Subject: [PATCH] feat: proving benchmark --- build_manifest.yml | 1 + yarn-project/Earthfile | 5 +- .../aztec-node/src/aztec-node/config.ts | 4 +- .../aztec-node/src/aztec-node/server.ts | 5 +- .../aztec/src/cli/cmds/start_prover.ts | 4 + .../src/interfaces/aztec-node.ts | 3 +- .../src/interfaces/prover-client.ts | 22 +++ yarn-project/end-to-end/Dockerfile | 6 +- yarn-project/end-to-end/Earthfile | 8 ++ .../src/benchmarks/bench_proving.test.ts | 124 +++++++++++++++++ .../src/fixtures/get_acvm_config.ts | 17 ++- .../end-to-end/src/fixtures/get_bb_config.ts | 46 ++++++ .../src/fixtures/snapshot_manager.ts | 4 +- yarn-project/end-to-end/src/fixtures/utils.ts | 131 +++++++++--------- yarn-project/prover-client/src/bb/execute.ts | 8 +- yarn-project/prover-client/src/config.ts | 18 ++- .../prover-client/src/dummy-prover.ts | 5 + yarn-project/prover-client/src/index.ts | 2 + .../src/prover-pool/prover-agent.ts | 7 +- .../src/prover-pool/prover-pool.ts | 31 +++-- .../prover-client/src/tx-prover/tx-prover.ts | 12 +- 21 files changed, 354 insertions(+), 109 deletions(-) create mode 100644 yarn-project/end-to-end/src/benchmarks/bench_proving.test.ts create mode 100644 yarn-project/end-to-end/src/fixtures/get_bb_config.ts diff --git a/build_manifest.yml b/build_manifest.yml index 4257cd66e11a..b2df1eaf1ce0 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -230,6 +230,7 @@ end-to-end: - noir-projects - noir - yarn-project + - barretenberg-x86_64-linux-clang runDependencies: - aztec diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile index 9a98815ccfe9..245ca5e57d90 100644 --- a/yarn-project/Earthfile +++ b/yarn-project/Earthfile @@ -5,7 +5,8 @@ deps: LET packages = $(git ls-files "**/package*.json" package*.json) LET tsconfigs = $(git ls-files "**/tsconfig*.json" tsconfig*.json) FROM ../build-images+build - # copy bb-js and noir-packages + # copy bb, bb-js and noir-packages + COPY ../barretenberg/cpp/+preset-release/bin /usr/src/barretenberg/cpp/build/ COPY ../barretenberg/ts/+build/build /usr/src/barretenberg/ts COPY ../noir/+packages/packages /usr/src/noir/packages WORKDIR /usr/src/yarn-project @@ -92,7 +93,7 @@ end-to-end: RUN apt-get update && apt-get install -y wget gnupg \ && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ && echo "deb [arch=$(dpkg --print-architecture)] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list \ - && apt update && apt install nodejs jq google-chrome-stable netcat-openbsd -y \ + && apt update && apt install curl nodejs jq google-chrome-stable netcat-openbsd -y \ && rm -rf /var/lib/apt/lists/* ENV CHROME_BIN="/usr/bin/google-chrome-stable" ENV PATH=/opt/foundry/bin:$PATH diff --git a/yarn-project/aztec-node/src/aztec-node/config.ts b/yarn-project/aztec-node/src/aztec-node/config.ts index dba7d824025a..8c00246d6088 100644 --- a/yarn-project/aztec-node/src/aztec-node/config.ts +++ b/yarn-project/aztec-node/src/aztec-node/config.ts @@ -1,6 +1,6 @@ import { type ArchiverConfig, getConfigEnvVars as getArchiverVars } from '@aztec/archiver'; import { type P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p'; -import { type ProverConfig, getProverEnvVars } from '@aztec/prover-client'; +import { type ProverClientConfig, getProverEnvVars } from '@aztec/prover-client'; import { type SequencerClientConfig, getConfigEnvVars as getSequencerVars } from '@aztec/sequencer-client'; import { getConfigEnvVars as getWorldStateVars } from '@aztec/world-state'; @@ -9,7 +9,7 @@ import { getConfigEnvVars as getWorldStateVars } from '@aztec/world-state'; */ export type AztecNodeConfig = ArchiverConfig & SequencerClientConfig & - ProverConfig & + ProverClientConfig & P2PConfig & { /** Whether the sequencer is disabled for this node. */ disableSequencer: boolean; diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 3bf5b97ab46f..9c13115176f7 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -15,6 +15,7 @@ import { NullifierMembershipWitness, type ProcessOutput, type ProverClient, + type ProverConfig, PublicDataWitness, type SequencerConfig, type SiblingPath, @@ -688,9 +689,9 @@ export class AztecNodeService implements AztecNode { }; } - public setConfig(config: Partial): Promise { + public async setConfig(config: Partial): Promise { this.sequencer?.updateSequencerConfig(config); - return Promise.resolve(); + await this.prover.updateProverConfig(config); } /** diff --git a/yarn-project/aztec/src/cli/cmds/start_prover.ts b/yarn-project/aztec/src/cli/cmds/start_prover.ts index 103ca97c8df0..7c39fe6e16a7 100644 --- a/yarn-project/aztec/src/cli/cmds/start_prover.ts +++ b/yarn-project/aztec/src/cli/cmds/start_prover.ts @@ -1,6 +1,8 @@ import { type ProvingJobSource } from '@aztec/circuit-types'; import { ProverPool, createProvingJobSourceClient } from '@aztec/prover-client/prover-pool'; +import { tmpdir } from 'node:os'; + import { type ServiceStarter, parseModuleOptions } from '../util.js'; type ProverOptions = Partial<{ @@ -35,6 +37,8 @@ export const startProver: ServiceStarter = async (options, signalHandlers, logge { acvmBinaryPath: proverOptions.acvmBinaryPath, bbBinaryPath: proverOptions.bbBinaryPath, + acvmWorkingDirectory: tmpdir(), + bbWorkingDirectory: tmpdir(), }, agentCount, ); diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.ts index 95cc81d5bc24..fd8b71c1126f 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.ts @@ -26,6 +26,7 @@ import { type TxEffect } from '../tx_effect.js'; import { type SequencerConfig } from './configs.js'; import { type L2BlockNumber } from './l2_block_number.js'; import { type NullifierMembershipWitness } from './nullifier_tree.js'; +import { type ProverConfig } from './prover-client.js'; import { type PublicDataWitness } from './public_data_tree.js'; /** @@ -288,7 +289,7 @@ export interface AztecNode { * Updates the configuration of this node. * @param config - Updated configuration to be merged with the current one. */ - setConfig(config: Partial): Promise; + setConfig(config: Partial): Promise; /** * Returns a registered contract class given its id. diff --git a/yarn-project/circuit-types/src/interfaces/prover-client.ts b/yarn-project/circuit-types/src/interfaces/prover-client.ts index 8e55d3a2dbbb..9200cc58aa48 100644 --- a/yarn-project/circuit-types/src/interfaces/prover-client.ts +++ b/yarn-project/circuit-types/src/interfaces/prover-client.ts @@ -1,6 +1,26 @@ import { type BlockProver } from './block-prover.js'; import { type ProvingJobSource } from './proving-job.js'; +/** + * The prover configuration. + */ +export type ProverConfig = { + /** The working directory to use for simulation/proving */ + acvmWorkingDirectory: string; + /** The path to the ACVM binary */ + acvmBinaryPath: string; + /** The working directory to for proving */ + bbWorkingDirectory: string; + /** The path to the bb binary */ + bbBinaryPath: string; + /** How many agents to start */ + proverAgents: number; + /** Enable proving. If true, must set bb env vars */ + realProofs: boolean; + /** The interval agents poll for jobs at */ + proverAgentPollInterval: number; +}; + /** * The interface to the prover client. * Provides the ability to generate proofs and build rollups. @@ -11,4 +31,6 @@ export interface ProverClient extends BlockProver { stop(): Promise; getProvingJobSource(): ProvingJobSource; + + updateProverConfig(config: Partial): Promise; } diff --git a/yarn-project/end-to-end/Dockerfile b/yarn-project/end-to-end/Dockerfile index 215a0fb1f977..22e364841da1 100644 --- a/yarn-project/end-to-end/Dockerfile +++ b/yarn-project/end-to-end/Dockerfile @@ -2,6 +2,7 @@ FROM --platform=linux/amd64 aztecprotocol/bb.js as bb.js FROM --platform=linux/amd64 aztecprotocol/noir-packages as noir-packages FROM --platform=linux/amd64 aztecprotocol/l1-contracts as contracts FROM --platform=linux/amd64 aztecprotocol/noir-projects as noir-projects +FROM --platform=linux/amd64 aztecprotocol/barretenberg-x86_64-linux-clang as bb FROM aztecprotocol/noir as noir FROM node:18.19.0 as builder @@ -13,6 +14,7 @@ COPY --from=noir-packages /usr/src/noir/packages /usr/src/noir/packages COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects COPY --from=noir /usr/src/noir/noir-repo/target/release/acvm /usr/src/noir/noir-repo/target/release/acvm +COPY --from=bb /usr/src/barretenberg/cpp/build/bin/bb /usr/src/barretenberg/cpp/build/bin/bb WORKDIR /usr/src/yarn-project COPY . . @@ -43,7 +45,7 @@ RUN /root/.foundry/bin/foundryup --version nightly-de33b6af53005037b463318d2628b # Create minimal image. FROM node:18.19.1-slim -RUN apt-get update && apt-get install jq gnupg wget netcat-openbsd -y && \ +RUN apt-get update && apt-get install jq gnupg wget curl netcat-openbsd -y && \ wget --quiet --output-document=- https://dl-ssl.google.com/linux/linux_signing_key.pub | gpg --dearmor > /etc/apt/trusted.gpg.d/google-archive.gpg && \ sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' && \ apt-get update && \ @@ -52,4 +54,4 @@ RUN apt-get update && apt-get install jq gnupg wget netcat-openbsd -y && \ ENV CHROME_BIN="/usr/bin/google-chrome-stable" COPY --from=builder /usr/src /usr/src WORKDIR /usr/src/yarn-project/end-to-end -ENTRYPOINT ["yarn", "test"] \ No newline at end of file +ENTRYPOINT ["yarn", "test"] diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index 8ff7a7f9aa18..c73963b402c9 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -98,3 +98,11 @@ bench-tx-size: ARG COMMIT_HASH DO +E2E_COMPOSE_TEST --test=benchmarks/bench_tx_size_fees.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --enable_gas=1 --compose_file=./scripts/docker-compose-no-sandbox.yml DO +UPLOAD_LOGS --e2e_mode=$e2e_mode --PULL_REQUEST=$PULL_REQUEST --BRANCH=$BRANCH --COMMIT_HASH=$COMMIT_HASH + +bench-proving: + ARG e2e_mode=local + ARG PULL_REQUEST + ARG BRANCH + ARG COMMIT_HASH + DO +E2E_COMPOSE_TEST --test=bench_proving --debug="aztec:benchmarks:*,aztec:prover*,aztec:bb*" --e2e_mode=$e2e_mode --enable_gas=1 --compose_file=./scripts/docker-compose-no-sandbox.yml + DO +UPLOAD_LOGS --e2e_mode=$e2e_mode --PULL_REQUEST=$PULL_REQUEST --BRANCH=$BRANCH --COMMIT_HASH=$COMMIT_HASH diff --git a/yarn-project/end-to-end/src/benchmarks/bench_proving.test.ts b/yarn-project/end-to-end/src/benchmarks/bench_proving.test.ts new file mode 100644 index 000000000000..a1565060e976 --- /dev/null +++ b/yarn-project/end-to-end/src/benchmarks/bench_proving.test.ts @@ -0,0 +1,124 @@ +import { type AztecNodeService } from '@aztec/aztec-node'; +import { type AccountWallet, EthAddress, PublicFeePaymentMethod, TxStatus } from '@aztec/aztec.js'; +import { GasSettings } from '@aztec/circuits.js'; +import { FPCContract, GasTokenContract, TestContract, TokenContract } from '@aztec/noir-contracts.js'; +import { getCanonicalGasTokenAddress } from '@aztec/protocol-contracts/gas-token'; +import { ProverPool } from '@aztec/prover-client/prover-pool'; + +import { jest } from '@jest/globals'; + +import { getACVMConfig } from '../fixtures/get_acvm_config.js'; +import { getBBConfig } from '../fixtures/get_bb_config.js'; +import { type EndToEndContext, publicDeployAccounts, setup } from '../fixtures/utils.js'; + +jest.setTimeout(200_000); + +const txTimeoutSec = 200; + +describe('benchmarks/proving', () => { + let ctx: EndToEndContext; + let wallet: AccountWallet; + let testContract: TestContract; + let tokenContract: TokenContract; + let fpContract: FPCContract; + let acvmCleanup: () => Promise; + let bbCleanup: () => Promise; + let proverPool: ProverPool; + + // setup the environment quickly using fake proofs + beforeAll(async () => { + ctx = await setup( + 1, + { + // do setup with fake proofs + realProofs: false, + proverAgents: 4, + proverAgentPollInterval: 10, + minTxsPerBlock: 1, + }, + {}, + true, // enable gas + ); + + wallet = ctx.wallet; + + await publicDeployAccounts(wallet, ctx.wallets); + + testContract = await TestContract.deploy(wallet).send().deployed(); + tokenContract = await TokenContract.deploy(wallet, wallet.getAddress(), 'test', 't', 18).send().deployed(); + const gas = await GasTokenContract.at( + getCanonicalGasTokenAddress(ctx.deployL1ContractsValues.l1ContractAddresses.gasPortalAddress), + wallet, + ); + fpContract = await FPCContract.deploy(wallet, tokenContract.address, gas.address).send().deployed(); + + await Promise.all([ + gas.methods.mint_public(fpContract.address, 1e12).send().wait(), + tokenContract.methods.mint_public(wallet.getAddress(), 1e12).send().wait(), + ]); + }); + + // remove the fake prover and setup the real one + beforeAll(async () => { + const [acvmConfig, bbConfig] = await Promise.all([getACVMConfig(ctx.logger), getBBConfig(ctx.logger)]); + if (!acvmConfig || !bbConfig) { + throw new Error('Missing ACVM or BB config'); + } + + acvmCleanup = acvmConfig.cleanup; + bbCleanup = bbConfig.cleanup; + + proverPool = ProverPool.nativePool( + { + ...acvmConfig, + ...bbConfig, + }, + 4, + 10, + ); + + ctx.logger.info('Stopping fake provers'); + await ctx.aztecNode.setConfig({ + // stop the fake provers + proverAgents: 0, + // 4-tx blocks so that we have at least one merge level + minTxsPerBlock: 4, + }); + + ctx.logger.info('Starting real provers'); + await proverPool.start((ctx.aztecNode as AztecNodeService).getProver().getProvingJobSource()); + }); + + afterAll(async () => { + await proverPool.stop(); + await ctx.teardown(); + await acvmCleanup(); + await bbCleanup(); + }); + + it('builds a full block', async () => { + const txs = [ + // fully private tx + testContract.methods.emit_nullifier(42).send(), + // tx with setup, app, teardown + testContract.methods.emit_unencrypted(43).send({ + fee: { + gasSettings: GasSettings.default(), + paymentMethod: new PublicFeePaymentMethod(tokenContract.address, fpContract.address, wallet), + }, + }), + // tx with messages + testContract.methods.create_l2_to_l1_message_public(45, 46, EthAddress.random()).send(), + // tx with private and public exec + testContract.methods.set_tx_max_block_number(100, true).send({ + fee: { + gasSettings: GasSettings.default(), + paymentMethod: new PublicFeePaymentMethod(tokenContract.address, fpContract.address, wallet), + }, + }), + ]; + + const receipts = await Promise.all(txs.map(tx => tx.wait({ timeout: txTimeoutSec }))); + expect(receipts.every(r => r.status === TxStatus.MINED)).toBe(true); + }); +}); diff --git a/yarn-project/end-to-end/src/fixtures/get_acvm_config.ts b/yarn-project/end-to-end/src/fixtures/get_acvm_config.ts index 556726411137..a8c8349a6cec 100644 --- a/yarn-project/end-to-end/src/fixtures/get_acvm_config.ts +++ b/yarn-project/end-to-end/src/fixtures/get_acvm_config.ts @@ -13,14 +13,21 @@ const { } = process.env; // Determines if we have access to the acvm binary and a tmp folder for temp files -export async function getACVMConfig(logger: DebugLogger) { +export async function getACVMConfig(logger: DebugLogger): Promise< + | { + acvmWorkingDirectory: string; + acvmBinaryPath: string; + cleanup: () => Promise; + } + | undefined +> { try { - const expectedAcvmPath = ACVM_BINARY_PATH ? ACVM_BINARY_PATH : `../../noir/${NOIR_RELEASE_DIR}/acvm`; - await fs.access(expectedAcvmPath, fs.constants.R_OK); + const acvmBinaryPath = ACVM_BINARY_PATH ? ACVM_BINARY_PATH : `../../noir/${NOIR_RELEASE_DIR}/acvm`; + await fs.access(acvmBinaryPath, fs.constants.R_OK); const tempWorkingDirectory = `${TEMP_DIR}/${randomBytes(4).toString('hex')}`; const acvmWorkingDirectory = ACVM_WORKING_DIRECTORY ? ACVM_WORKING_DIRECTORY : `${tempWorkingDirectory}/acvm`; await fs.mkdir(acvmWorkingDirectory, { recursive: true }); - logger.verbose(`Using native ACVM binary at ${expectedAcvmPath} with working directory ${acvmWorkingDirectory}`); + logger.verbose(`Using native ACVM binary at ${acvmBinaryPath} with working directory ${acvmWorkingDirectory}`); const directoryToCleanup = ACVM_WORKING_DIRECTORY ? undefined : tempWorkingDirectory; @@ -33,7 +40,7 @@ export async function getACVMConfig(logger: DebugLogger) { return { acvmWorkingDirectory, - expectedAcvmPath, + acvmBinaryPath, cleanup, }; } catch (err) { diff --git a/yarn-project/end-to-end/src/fixtures/get_bb_config.ts b/yarn-project/end-to-end/src/fixtures/get_bb_config.ts new file mode 100644 index 000000000000..412c93164579 --- /dev/null +++ b/yarn-project/end-to-end/src/fixtures/get_bb_config.ts @@ -0,0 +1,46 @@ +import { type DebugLogger, fileURLToPath } from '@aztec/aztec.js'; + +import fs from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'path'; + +const { + BB_RELEASE_DIR = 'barretenberg/cpp/build/bin', + BB_BINARY_PATH, + TEMP_DIR = tmpdir(), + BB_WORKING_DIRECTORY = '', +} = process.env; + +export const getBBConfig = async ( + logger: DebugLogger, +): Promise<{ bbBinaryPath: string; bbWorkingDirectory: string; cleanup: () => Promise } | undefined> => { + try { + const bbBinaryPath = + BB_BINARY_PATH ?? + path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../../', BB_RELEASE_DIR, 'bb'); + await fs.access(bbBinaryPath, fs.constants.R_OK); + + let bbWorkingDirectory: string; + let directoryToCleanup: string | undefined; + + if (BB_WORKING_DIRECTORY) { + bbWorkingDirectory = BB_WORKING_DIRECTORY; + } else { + bbWorkingDirectory = await fs.mkdtemp(path.join(TEMP_DIR, 'bb-')); + directoryToCleanup = bbWorkingDirectory; + } + + await fs.mkdir(bbWorkingDirectory, { recursive: true }); + + const cleanup = async () => { + if (directoryToCleanup) { + await fs.rm(directoryToCleanup, { recursive: true, force: true }); + } + }; + + return { bbBinaryPath, bbWorkingDirectory, cleanup }; + } catch (err) { + logger.error(`Native BB not available, error: ${err}`); + return undefined; + } +}; diff --git a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts index 9fe93aafd776..a77f661f34c2 100644 --- a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts +++ b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts @@ -202,7 +202,7 @@ export class SnapshotManager { const acvmConfig = await getACVMConfig(this.logger); if (acvmConfig) { aztecNodeConfig.acvmWorkingDirectory = acvmConfig.acvmWorkingDirectory; - aztecNodeConfig.acvmBinaryPath = acvmConfig.expectedAcvmPath; + aztecNodeConfig.acvmBinaryPath = acvmConfig.acvmBinaryPath; } this.logger.verbose('Creating and synching an aztec node...'); @@ -254,7 +254,7 @@ export class SnapshotManager { const acvmConfig = await getACVMConfig(this.logger); if (acvmConfig) { aztecNodeConfig.acvmWorkingDirectory = acvmConfig.acvmWorkingDirectory; - aztecNodeConfig.acvmBinaryPath = acvmConfig.expectedAcvmPath; + aztecNodeConfig.acvmBinaryPath = acvmConfig.acvmBinaryPath; } this.logger.verbose('Creating aztec node...'); diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts index df58258d1f69..f951a181a0eb 100644 --- a/yarn-project/end-to-end/src/fixtures/utils.ts +++ b/yarn-project/end-to-end/src/fixtures/utils.ts @@ -22,7 +22,6 @@ import { createDebugLogger, createPXEClient, deployL1Contracts, - fileURLToPath, makeFetch, waitForPXE, } from '@aztec/aztec.js'; @@ -33,7 +32,6 @@ import { computeContractAddressFromInstance, getContractClassFromArtifact, } from '@aztec/circuits.js'; -import { randomBytes } from '@aztec/foundation/crypto'; import { makeBackoff, retry } from '@aztec/foundation/retry'; import { AvailabilityOracleAbi, @@ -55,11 +53,11 @@ import { KeyRegistryContract } from '@aztec/noir-contracts.js'; import { GasTokenContract } from '@aztec/noir-contracts.js/GasToken'; import { getCanonicalGasToken, getCanonicalGasTokenAddress } from '@aztec/protocol-contracts/gas-token'; import { getCanonicalKeyRegistry } from '@aztec/protocol-contracts/key-registry'; +import { type ProverClient } from '@aztec/prover-client'; import { PXEService, type PXEServiceConfig, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; import { type SequencerClient } from '@aztec/sequencer-client'; import { type Anvil, createAnvil } from '@viem/anvil'; -import * as fs from 'fs/promises'; import getPort from 'get-port'; import * as path from 'path'; import { @@ -77,44 +75,17 @@ import { mnemonicToAccount } from 'viem/accounts'; import { foundry } from 'viem/chains'; import { MNEMONIC } from './fixtures.js'; +import { getACVMConfig } from './get_acvm_config.js'; import { isMetricsLoggingRequested, setupMetricsLogger } from './logging.js'; export { deployAndInitializeTokenAndBridgeContracts } from '../shared/cross_chain_test_harness.js'; -const { - PXE_URL = '', - NOIR_RELEASE_DIR = 'noir-repo/target/release', - TEMP_DIR = '/tmp', - ACVM_BINARY_PATH = '', - ACVM_WORKING_DIRECTORY = '', -} = process.env; +const { PXE_URL = '' } = process.env; const getAztecUrl = () => { return PXE_URL; }; -// Determines if we have access to the acvm binary and a tmp folder for temp files -const getACVMConfig = async (logger: DebugLogger) => { - try { - const expectedAcvmPath = ACVM_BINARY_PATH - ? ACVM_BINARY_PATH - : `${path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../../noir/', NOIR_RELEASE_DIR)}/acvm`; - await fs.access(expectedAcvmPath, fs.constants.R_OK); - const tempWorkingDirectory = `${TEMP_DIR}/${randomBytes(4).toString('hex')}`; - const acvmWorkingDirectory = ACVM_WORKING_DIRECTORY ? ACVM_WORKING_DIRECTORY : `${tempWorkingDirectory}/acvm`; - await fs.mkdir(acvmWorkingDirectory, { recursive: true }); - logger.info(`Using native ACVM binary at ${expectedAcvmPath} with working directory ${acvmWorkingDirectory}`); - return { - acvmWorkingDirectory, - expectedAcvmPath, - directoryToCleanup: ACVM_WORKING_DIRECTORY ? undefined : tempWorkingDirectory, - }; - } catch (err) { - logger.error(`Native ACVM not available, error: ${err}`); - return undefined; - } -}; - export const setupL1Contracts = async ( l1RpcUrl: string, account: HDAccount | PrivateKeyAccount, @@ -279,7 +250,7 @@ async function setupWithRemoteEnvironment( const { chainId, protocolVersion } = await pxeClient.getNodeInfo(); // this contract might already have been deployed - // the following deployin functions are idempotent + // the following deploying functions are idempotent await deployCanonicalKeyRegistry( new SignerlessWallet(pxeClient, new DefaultMultiCallEntrypoint(chainId, protocolVersion)), ); @@ -293,6 +264,7 @@ async function setupWithRemoteEnvironment( return { aztecNode, sequencer: undefined, + prover: undefined, pxe: pxeClient, deployL1ContractsValues, accounts: await pxeClient!.getRegisteredAccounts(), @@ -333,22 +305,17 @@ export type EndToEndContext = { logger: DebugLogger; /** The cheat codes. */ cheatCodes: CheatCodes; + /** Proving jobs */ + prover: ProverClient | undefined; /** Function to stop the started services. */ teardown: () => Promise; }; -/** - * Sets up the environment for the end-to-end tests. - * @param numberOfAccounts - The number of new accounts to be created once the PXE is initiated. - * @param opts - Options to pass to the node initialization and to the setup script. - * @param pxeOpts - Options to pass to the PXE initialization. - */ -export async function setup( - numberOfAccounts = 1, +export async function setupNoL2Deploy( opts: SetupOptions = {}, pxeOpts: Partial = {}, - enableGas = false, -): Promise { +): Promise> { + const logger = getLogger(); const config = { ...getConfigEnvVars(), ...opts }; let anvil: Anvil | undefined; @@ -380,6 +347,7 @@ export async function setup( // Enable logging metrics to a local file named after the test suite if (isMetricsLoggingRequested()) { const filename = path.join('log', getJobName() + '.jsonl'); + logger.info(`Logging metrics to ${filename}`); setupMetricsLogger(filename); } @@ -388,14 +356,13 @@ export async function setup( await ethCheatCodes.loadChainState(opts.stateLoad); } - const logger = getLogger(); const hdAccount = mnemonicToAccount(MNEMONIC); const privKeyRaw = hdAccount.getHdKey().privateKey; const publisherPrivKey = privKeyRaw === null ? null : Buffer.from(privKeyRaw); if (PXE_URL) { // we are setting up against a remote environment, l1 contracts are assumed to already be deployed - return await setupWithRemoteEnvironment(hdAccount, config, logger, numberOfAccounts, enableGas); + return setupWithRemoteEnvironment(hdAccount, config, logger, 0, false); } const deployL1ContractsValues = @@ -409,26 +376,15 @@ export async function setup( const acvmConfig = await getACVMConfig(logger); if (acvmConfig) { config.acvmWorkingDirectory = acvmConfig.acvmWorkingDirectory; - config.acvmBinaryPath = acvmConfig.expectedAcvmPath; + config.acvmBinaryPath = acvmConfig.acvmBinaryPath; } config.l1BlockPublishRetryIntervalMS = 100; const aztecNode = await AztecNodeService.createAndSync(config); const sequencer = aztecNode.getSequencer(); + const prover = aztecNode.getProver(); logger.verbose('Creating a pxe...'); - const { pxe, wallets } = await setupPXEService(numberOfAccounts, aztecNode!, pxeOpts, logger); - - logger.verbose('Deploying key registry...'); - await deployCanonicalKeyRegistry( - new SignerlessWallet(pxe, new DefaultMultiCallEntrypoint(config.chainId, config.version)), - ); - - if (enableGas) { - logger.verbose('Deploying gas token...'); - await deployCanonicalGasToken( - new SignerlessWallet(pxe, new DefaultMultiCallEntrypoint(config.chainId, config.version)), - ); - } + const { pxe } = await setupPXEService(0, aztecNode!, pxeOpts, logger, false); const cheatCodes = CheatCodes.create(config.rpcUrl, pxe!); @@ -440,10 +396,10 @@ export async function setup( await pxe?.stop(); } - if (acvmConfig?.directoryToCleanup) { + if (acvmConfig?.cleanup) { // remove the temp directory created for the acvm - logger.verbose(`Cleaning up ACVM temp directory ${acvmConfig.directoryToCleanup}`); - await fs.rm(acvmConfig.directoryToCleanup, { recursive: true, force: true }); + logger.verbose(`Cleaning up ACVM state`); + await acvmConfig.cleanup(); } await anvil?.stop(); @@ -454,15 +410,54 @@ export async function setup( pxe, deployL1ContractsValues, config, - wallet: wallets[0], - wallets, logger, cheatCodes, sequencer, + prover, + wallets: [], teardown, }; } +/** + * Sets up the environment for the end-to-end tests. + * @param numberOfAccounts - The number of new accounts to be created once the PXE is initiated. + * @param opts - Options to pass to the node initialization and to the setup script. + * @param pxeOpts - Options to pass to the PXE initialization. + */ +export async function setup( + numberOfAccounts = 1, + opts: SetupOptions = {}, + pxeOpts: Partial = {}, + enableGas = false, +): Promise { + const ctx = await setupNoL2Deploy(opts, pxeOpts); + const wallets = await getDeployedTestAccountsWallets(ctx.pxe); + if (wallets.length < numberOfAccounts) { + const numNewAccounts = numberOfAccounts - wallets.length; + ctx.logger.verbose(`Deploying ${numNewAccounts} accounts...`); + wallets.push(...(await createAccounts(ctx.pxe, numNewAccounts))); + } + + ctx.logger.verbose('Deploying key registry...'); + await deployCanonicalKeyRegistry( + new SignerlessWallet(ctx.pxe, new DefaultMultiCallEntrypoint(ctx.config.chainId, ctx.config.version)), + ); + + if (enableGas) { + ctx.logger.verbose('Deploying gas token...'); + await deployCanonicalGasToken( + new SignerlessWallet(ctx.pxe, new DefaultMultiCallEntrypoint(ctx.config.chainId, ctx.config.version)), + ); + } + + return { + ...ctx, + wallets, + wallet: wallets[0], + }; +} + /** * Registers the contract class used for test accounts and publicly deploys the instances requested. * Use this when you need to make a public call to an account contract, such as for requesting a public authwit. @@ -594,7 +589,7 @@ export async function expectMapping( /** * Deploy the protocol contracts to a running instance. */ -export async function deployCanonicalGasToken(deployer: Wallet) { +export async function deployCanonicalGasToken(deployer: Wallet, timeout?: number) { // "deploy" the Gas token as it contains public functions const gasPortalAddress = (await deployer.getNodeInfo()).l1ContractAddresses.gasPortalAddress; const canonicalGasToken = getCanonicalGasToken(gasPortalAddress); @@ -605,14 +600,14 @@ export async function deployCanonicalGasToken(deployer: Wallet) { const gasToken = await GasTokenContract.deploy(deployer, gasPortalAddress) .send({ contractAddressSalt: canonicalGasToken.instance.salt, universalDeploy: true }) - .deployed(); + .deployed({ timeout }); await expect(deployer.isContractClassPubliclyRegistered(gasToken.instance.contractClassId)).resolves.toBe(true); await expect(deployer.getContractInstance(gasToken.address)).resolves.toBeDefined(); await expect(deployer.isContractPubliclyDeployed(gasToken.address)).resolves.toBe(true); } -async function deployCanonicalKeyRegistry(deployer: Wallet) { +async function deployCanonicalKeyRegistry(deployer: Wallet, timeout?: number) { const canonicalKeyRegistry = getCanonicalKeyRegistry(); // We check to see if there exists a contract at the canonical Key Registry address with the same contract class id as we expect. This means that @@ -628,7 +623,7 @@ async function deployCanonicalKeyRegistry(deployer: Wallet) { const keyRegistry = await KeyRegistryContract.deploy(deployer) .send({ contractAddressSalt: canonicalKeyRegistry.instance.salt, universalDeploy: true }) - .deployed(); + .deployed({ timeout }); if ( !keyRegistry.address.equals(canonicalKeyRegistry.address) || diff --git a/yarn-project/prover-client/src/bb/execute.ts b/yarn-project/prover-client/src/bb/execute.ts index f53950dd0f5c..58a7cb968a61 100644 --- a/yarn-project/prover-client/src/bb/execute.ts +++ b/yarn-project/prover-client/src/bb/execute.ts @@ -50,16 +50,16 @@ export function executeBB( ) { return new Promise((resolve, reject) => { // spawn the bb process - const acvm = proc.spawn(pathToBB, [command, ...args]); - acvm.stdout.on('data', data => { + const bb = proc.spawn(pathToBB, [command, ...args]); + bb.stdout.on('data', data => { const message = data.toString('utf-8').replace(/\n$/, ''); logger(message); }); - acvm.stderr.on('data', data => { + bb.stderr.on('data', data => { const message = data.toString('utf-8').replace(/\n$/, ''); logger(message); }); - acvm.on('close', (code: number) => { + bb.on('close', (code: number) => { if (resultParser(code)) { resolve(BB_RESULT.SUCCESS); } else { diff --git a/yarn-project/prover-client/src/config.ts b/yarn-project/prover-client/src/config.ts index 0b8f7cce6ea1..fdebb24ed7f6 100644 --- a/yarn-project/prover-client/src/config.ts +++ b/yarn-project/prover-client/src/config.ts @@ -1,9 +1,11 @@ +import { type ProverConfig } from '@aztec/circuit-types'; + import { tmpdir } from 'os'; /** * The prover configuration. */ -export interface ProverConfig { +export type ProverClientConfig = ProverConfig & { /** The working directory to use for simulation/proving */ acvmWorkingDirectory: string; /** The path to the ACVM binary */ @@ -12,29 +14,34 @@ export interface ProverConfig { bbWorkingDirectory: string; /** The path to the bb binary */ bbBinaryPath: string; - /** How many agents to start */ - proverAgents: number; /** Enable proving. If true, must set bb env vars */ realProofs: boolean; -} + /** The interval agents poll for jobs at */ + proverAgentPollInterval: number; +}; /** * Returns the prover configuration from the environment variables. * Note: If an environment variable is not set, the default value is used. * @returns The prover configuration. */ -export function getProverEnvVars(): ProverConfig { +export function getProverEnvVars(): ProverClientConfig { const { ACVM_WORKING_DIRECTORY = tmpdir(), ACVM_BINARY_PATH = '', BB_WORKING_DIRECTORY = tmpdir(), BB_BINARY_PATH = '', PROVER_AGENTS = '1', + PROVER_AGENT_POLL_INTERVAL_MS = '10', PROVER_REAL_PROOFS = '', } = process.env; const parsedProverAgents = parseInt(PROVER_AGENTS, 10); const proverAgents = Number.isSafeInteger(parsedProverAgents) ? parsedProverAgents : 0; + const parsedProverAgentPollInterval = parseInt(PROVER_AGENT_POLL_INTERVAL_MS, 10); + const proverAgentPollInterval = Number.isSafeInteger(parsedProverAgentPollInterval) + ? parsedProverAgentPollInterval + : 10; return { acvmWorkingDirectory: ACVM_WORKING_DIRECTORY, @@ -43,5 +50,6 @@ export function getProverEnvVars(): ProverConfig { bbWorkingDirectory: BB_WORKING_DIRECTORY, proverAgents, realProofs: ['1', 'true'].includes(PROVER_REAL_PROOFS), + proverAgentPollInterval, }; } diff --git a/yarn-project/prover-client/src/dummy-prover.ts b/yarn-project/prover-client/src/dummy-prover.ts index e8c76d009e56..4a912e09119a 100644 --- a/yarn-project/prover-client/src/dummy-prover.ts +++ b/yarn-project/prover-client/src/dummy-prover.ts @@ -4,6 +4,7 @@ import { PROVING_STATUS, type ProcessedTx, type ProverClient, + type ProverConfig, type ProvingJob, type ProvingJobSource, type ProvingRequest, @@ -63,6 +64,10 @@ export class DummyProver implements ProverClient { getProvingJobSource(): ProvingJobSource { return this.jobs; } + + updateProverConfig(_config: Partial): Promise { + return Promise.resolve(); + } } class DummyProvingJobSource implements ProvingJobSource { diff --git a/yarn-project/prover-client/src/index.ts b/yarn-project/prover-client/src/index.ts index c47f1852f991..4331fcaff0ea 100644 --- a/yarn-project/prover-client/src/index.ts +++ b/yarn-project/prover-client/src/index.ts @@ -1,3 +1,5 @@ +export { ProverClient } from '@aztec/circuit-types'; + export * from './tx-prover/tx-prover.js'; export * from './config.js'; export * from './dummy-prover.js'; diff --git a/yarn-project/prover-client/src/prover-pool/prover-agent.ts b/yarn-project/prover-client/src/prover-pool/prover-agent.ts index e5ae9f156f95..46a190085c32 100644 --- a/yarn-project/prover-client/src/prover-pool/prover-agent.ts +++ b/yarn-project/prover-client/src/prover-pool/prover-agent.ts @@ -52,15 +52,18 @@ export class ProverAgent { }, this.intervalMs); this.runningPromise.start(); + this.log.info('Agent started'); } async stop(): Promise { - if (!this.runningPromise) { - throw new Error('Agent is not running'); + if (!this.runningPromise?.isRunning()) { + return; } await this.runningPromise.stop(); this.runningPromise = undefined; + + this.log.info('Agent stopped'); } private work(request: ProvingRequest): Promise> { diff --git a/yarn-project/prover-client/src/prover-pool/prover-pool.ts b/yarn-project/prover-client/src/prover-pool/prover-pool.ts index 0750b41520e1..b81a37b6766e 100644 --- a/yarn-project/prover-client/src/prover-pool/prover-pool.ts +++ b/yarn-project/prover-client/src/prover-pool/prover-pool.ts @@ -2,7 +2,6 @@ import { type ProvingJobSource } from '@aztec/circuit-types'; import { type SimulationProvider } from '@aztec/simulator'; import { mkdtemp } from 'fs/promises'; -import { tmpdir } from 'os'; import { join } from 'path'; import { BBNativeRollupProver, type BBProverConfig } from '../prover/bb_prover.js'; @@ -38,7 +37,7 @@ export class ProverPool { async stop(): Promise { if (!this.running) { - throw new Error('Prover pool is not running'); + return; } for (const agent of this.agents) { @@ -48,6 +47,20 @@ export class ProverPool { this.running = false; } + async rescale(newSize: number): Promise { + if (newSize > this.size) { + this.size = newSize; + for (let i = this.agents.length; i < newSize; i++) { + this.agents.push(await this.agentFactory(i)); + } + } else if (newSize < this.size) { + this.size = newSize; + while (this.agents.length > newSize) { + await this.agents.pop()?.stop(); + } + } + } + static testPool(simulationProvider?: SimulationProvider, size = 1, agentPollIntervalMS = 10): ProverPool { return new ProverPool( size, @@ -55,22 +68,18 @@ export class ProverPool { ); } - static nativePool( - { acvmBinaryPath, bbBinaryPath }: Pick, - size: number, - agentPollIntervalMS = 10, - ): ProverPool { + static nativePool(config: Omit, size: number, agentPollIntervalMS = 10): ProverPool { // TODO generate keys ahead of time so that each agent doesn't have to do it return new ProverPool(size, async i => { const [acvmWorkingDirectory, bbWorkingDirectory] = await Promise.all([ - mkdtemp(join(tmpdir(), 'acvm-')), - mkdtemp(join(tmpdir(), 'bb-')), + mkdtemp(join(config.acvmWorkingDirectory, 'agent-')), + mkdtemp(join(config.bbWorkingDirectory, 'agent-')), ]); return new ProverAgent( await BBNativeRollupProver.new({ - acvmBinaryPath, + acvmBinaryPath: config.acvmBinaryPath, acvmWorkingDirectory, - bbBinaryPath, + bbBinaryPath: config.bbBinaryPath, bbWorkingDirectory, }), agentPollIntervalMS, diff --git a/yarn-project/prover-client/src/tx-prover/tx-prover.ts b/yarn-project/prover-client/src/tx-prover/tx-prover.ts index e5b08a74560d..ea6ecfcd0373 100644 --- a/yarn-project/prover-client/src/tx-prover/tx-prover.ts +++ b/yarn-project/prover-client/src/tx-prover/tx-prover.ts @@ -2,6 +2,7 @@ import { type ProcessedTx } from '@aztec/circuit-types'; import { type BlockResult, type ProverClient, + type ProverConfig, type ProvingJobSource, type ProvingTicket, } from '@aztec/circuit-types/interfaces'; @@ -9,7 +10,6 @@ import { type Fr, type GlobalVariables } from '@aztec/circuits.js'; import { type SimulationProvider } from '@aztec/simulator'; import { type WorldStateSynchronizer } from '@aztec/world-state'; -import { type ProverConfig } from '../config.js'; import { type VerificationKeys, getVerificationKeys } from '../mocks/verification_keys.js'; import { ProvingOrchestrator } from '../orchestrator/orchestrator.js'; import { MemoryProvingQueue } from '../prover-pool/memory-proving-queue.js'; @@ -30,6 +30,12 @@ export class TxProver implements ProverClient { this.orchestrator = new ProvingOrchestrator(worldStateSynchronizer.getLatest(), this.queue); } + async updateProverConfig(config: Partial): Promise { + if (typeof config.proverAgents === 'number') { + await this.proverPool?.rescale(config.proverAgents); + } + } + /** * Starts the prover instance */ @@ -68,9 +74,9 @@ export class TxProver implements ProverClient { throw new Error(); } - pool = ProverPool.nativePool(config, config.proverAgents, 10); + pool = ProverPool.nativePool(config, config.proverAgents, config.proverAgentPollInterval); } else { - pool = ProverPool.testPool(simulationProvider, config.proverAgents, 10); + pool = ProverPool.testPool(simulationProvider, config.proverAgents, config.proverAgentPollInterval); } const prover = new TxProver(worldStateSynchronizer, getVerificationKeys(), pool);