From dd28ad35d15b76d8549ecf2a50e955c9caa3c0a9 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Fri, 26 Apr 2024 07:04:23 +0000 Subject: [PATCH] feat: prover metrics --- .../circuit-types/src/stats/metrics.ts | 26 ++++- yarn-project/circuit-types/src/stats/stats.ts | 52 ++++++--- .../src/orchestrator/orchestrator.ts | 94 ++-------------- .../prover-client/src/prover/bb_prover.ts | 29 +++++ .../src/prover/test_circuit_prover.ts | 81 ++++++++++++-- yarn-project/prover-client/src/util.ts | 103 ++++++++++++++++++ 6 files changed, 273 insertions(+), 112 deletions(-) create mode 100644 yarn-project/prover-client/src/util.ts diff --git a/yarn-project/circuit-types/src/stats/metrics.ts b/yarn-project/circuit-types/src/stats/metrics.ts index 4ee8ea7e04c7..00c2a9a7bdbf 100644 --- a/yarn-project/circuit-types/src/stats/metrics.ts +++ b/yarn-project/circuit-types/src/stats/metrics.ts @@ -111,7 +111,13 @@ export const Metrics = [ events: ['note-processor-caught-up'], }, { - name: 'circuit_simulation_time_in_ms', + name: 'circuit_wasm_simulation_time_in_ms', + groupBy: 'circuit-name', + description: 'Time to run a circuit simulation using WASM.', + events: ['circuit-simulation'], + }, + { + name: 'circuit_native_simulation_time_in_ms', groupBy: 'circuit-name', description: 'Time to run a circuit simulation.', events: ['circuit-simulation'], @@ -128,6 +134,24 @@ export const Metrics = [ description: 'Size of the outputs (ie public inputs) from a circuit simulation.', events: ['circuit-simulation'], }, + { + name: 'circuit_proving_time_in_ms', + groupBy: 'circuit-name', + description: 'Time to prove circuit execution.', + events: ['circuit-proving'], + }, + { + name: 'circuit_proof_size_in_bytes', + groupBy: 'circuit-name', + description: 'Size of the proof produced by a circuit.', + events: ['circuit-proving'], + }, + { + name: 'proof_verification_time_in_ms', + groupBy: 'circuit-name', + description: 'Time to verify proof.', + events: ['proof-verification'], + }, { name: 'tx_size_in_bytes', groupBy: 'classes-registered', diff --git a/yarn-project/circuit-types/src/stats/stats.ts b/yarn-project/circuit-types/src/stats/stats.ts index 14d159a1e991..c14d3e1b28a1 100644 --- a/yarn-project/circuit-types/src/stats/stats.ts +++ b/yarn-project/circuit-types/src/stats/stats.ts @@ -48,30 +48,54 @@ export type NodeSyncedChainHistoryStats = { dbSize: number; }; +export type CircuitName = + | 'base-parity' + | 'root-parity' + | 'base-rollup' + | 'private-kernel-init' + | 'private-kernel-ordering' + | 'root-rollup' + | 'merge-rollup' + | 'private-kernel-inner' + | 'public-kernel-setup' + | 'public-kernel-app-logic' + | 'public-kernel-teardown' + | 'public-kernel-tail'; + /** Stats for circuit simulation. */ export type CircuitSimulationStats = { /** name of the event. */ eventName: 'circuit-simulation'; /** Name of the circuit. */ - circuitName: - | 'base-parity' - | 'root-parity' - | 'base-rollup' - | 'private-kernel-init' - | 'private-kernel-ordering' - | 'root-rollup' - | 'merge-rollup' - | 'private-kernel-inner' - | 'public-kernel-setup' - | 'public-kernel-app-logic' - | 'public-kernel-teardown' - | 'public-kernel-tail'; + circuitName: CircuitName; + /** Duration in ms. */ + duration: number; + /** Size in bytes of circuit inputs. */ + inputSize: number; + /** Size in bytes of circuit outputs (aka public inputs). */ + outputSize: number; +}; + +export type CircuitProvingStats = { + /** Name of the event. */ + eventName: 'circuit-proving'; + /** Name of the circuit. */ + circuitName: CircuitName; /** Duration in ms. */ duration: number; /** Size in bytes of circuit inputs. */ inputSize: number; /** Size in bytes of circuit outputs (aka public inputs). */ outputSize: number; + /** Size in bytes of the proof. */ + proofSize: number; +}; + +export type ProofVerificationStats = { + eventName: 'proof-verification'; + circuitName: CircuitName; + duration: number; + proofSize: number; }; /** Stats for an L2 block built by a sequencer. */ @@ -206,6 +230,8 @@ export type Stats = | L1PublishStats | NodeSyncedChainHistoryStats | CircuitSimulationStats + | CircuitProvingStats + | ProofVerificationStats | L2BlockBuiltStats | L2BlockHandledStats | NoteProcessorCaughtUpStats diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.ts index eb77bf3db937..a3a1602d399f 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.ts @@ -3,7 +3,6 @@ import { L2Block, MerkleTreeId, type ProcessedTx, - type PublicKernelRequest, PublicKernelType, type TxEffect, toTxEffect, @@ -14,7 +13,6 @@ import { type ProvingResult, type ProvingTicket, } from '@aztec/circuit-types/interfaces'; -import { type CircuitSimulationStats } from '@aztec/circuit-types/stats'; import { type BaseOrMergeRollupPublicInputs, BaseParityInputs, @@ -39,7 +37,6 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { promiseWithResolvers } from '@aztec/foundation/promise'; import { type Tuple } from '@aztec/foundation/serialize'; import { sleep } from '@aztec/foundation/sleep'; -import { Timer } from '@aztec/foundation/timer'; import { type MerkleTreeOperations } from '@aztec/world-state'; import { inspect } from 'util'; @@ -309,7 +306,7 @@ export class ProvingOrchestrator { private deferredProving( provingState: ProvingState | undefined, request: (signal: AbortSignal) => Promise, - callback: (result: T, durationMs: number) => void | Promise, + callback: (result: T) => void | Promise, ) { if (!provingState?.verifyState()) { logger.debug(`Not enqueuing job, state no longer valid`); @@ -328,10 +325,7 @@ export class ProvingOrchestrator { return; } - const timer = new Timer(); const result = await request(controller.signal); - const duration = timer.ms(); - if (!provingState?.verifyState()) { logger.debug(`State no longer valid, discarding result`); return; @@ -343,7 +337,7 @@ export class ProvingOrchestrator { return; } - await callback(result, duration); + await callback(result); } catch (err) { if (err instanceof AbortedError) { // operation was cancelled, probably because the block was cancelled @@ -365,39 +359,6 @@ export class ProvingOrchestrator { setImmediate(safeJob); } - private emitCircuitSimulationStats( - circuitName: CircuitSimulationStats['circuitName'] | null, - inputSize: number, - outputSize: number, - duration: number, - ) { - const stats: CircuitSimulationStats | undefined = circuitName - ? { - eventName: 'circuit-simulation', - circuitName, - duration, - inputSize, - outputSize, - } - : undefined; - logger.debug(`Simulated ${circuitName} circuit duration=${duration}ms`, stats); - } - - private getPublicKernelCircuitName(request: PublicKernelRequest) { - switch (request.type) { - case PublicKernelType.SETUP: - return 'public-kernel-setup'; - case PublicKernelType.APP_LOGIC: - return 'public-kernel-app-logic'; - case PublicKernelType.TEARDOWN: - return 'public-kernel-teardown'; - case PublicKernelType.TAIL: - return 'public-kernel-tail'; - default: - return null; - } - } - // Updates the merkle trees for a transaction. The first enqueued job for a transaction private async prepareBaseRollupInputs( provingState: ProvingState | undefined, @@ -473,14 +434,7 @@ export class ProvingOrchestrator { this.deferredProving( provingState, signal => this.prover.getBaseRollupProof(tx.baseRollupInputs, signal), - ([publicInputs, proof], duration) => { - this.emitCircuitSimulationStats( - 'base-rollup', - tx.baseRollupInputs.toBuffer().length, - publicInputs.toBuffer().length, - duration, - ); - + ([publicInputs, proof]) => { validatePartialState(publicInputs.end, tx.treeSnapshots); const currentLevel = provingState.numMergeLevels + 1n; this.storeAndExecuteNextMergeLevel(provingState, currentLevel, index, [publicInputs, proof]); @@ -504,13 +458,7 @@ export class ProvingOrchestrator { this.deferredProving( provingState, signal => this.prover.getMergeRollupProof(inputs, signal), - ([publicInputs, proof], duration) => { - this.emitCircuitSimulationStats( - 'merge-rollup', - inputs.toBuffer().length, - publicInputs.toBuffer().length, - duration, - ); + ([publicInputs, proof]) => { this.storeAndExecuteNextMergeLevel(provingState, level, index, [publicInputs, proof]); }, ); @@ -540,14 +488,7 @@ export class ProvingOrchestrator { this.deferredProving( provingState, signal => this.prover.getRootRollupProof(inputs, signal), - ([publicInputs, proof], duration) => { - this.emitCircuitSimulationStats( - 'root-rollup', - inputs.toBuffer().length, - publicInputs.toBuffer().length, - duration, - ); - + ([publicInputs, proof]) => { provingState.rootRollupPublicInputs = publicInputs; provingState.finalProof = proof; @@ -565,13 +506,7 @@ export class ProvingOrchestrator { this.deferredProving( provingState, signal => this.prover.getBaseParityProof(inputs, signal), - ([publicInputs, proof], duration) => { - this.emitCircuitSimulationStats( - 'base-parity', - inputs.toBuffer().length, - publicInputs.toBuffer().length, - duration, - ); + ([publicInputs, proof]) => { const rootInput = new RootParityInput(proof, publicInputs); provingState.setRootParityInputs(rootInput, index); const rootParityInputs = new RootParityInputs( @@ -588,13 +523,7 @@ export class ProvingOrchestrator { this.deferredProving( provingState, signal => this.prover.getRootParityProof(inputs, signal), - async ([publicInputs, proof], duration) => { - this.emitCircuitSimulationStats( - 'root-parity', - inputs.toBuffer().length, - publicInputs.toBuffer().length, - duration, - ); + async ([publicInputs, proof]) => { const rootInput = new RootParityInput(proof, publicInputs); provingState!.finalRootParityInput = rootInput; await this.checkAndEnqueueRootRollup(provingState); @@ -709,14 +638,7 @@ export class ProvingOrchestrator { return this.prover.getPublicKernelProof(request, signal); } }, - ([_, proof], duration) => { - this.emitCircuitSimulationStats( - this.getPublicKernelCircuitName(request), - request.inputs.toBuffer().length, - 0, - duration, - ); - + ([_, proof]) => { const nextKernelRequest = txProvingState.getNextPublicKernelFromKernelProof(functionIndex, proof); // What's the status of the next kernel? if (nextKernelRequest.code === TX_PROVING_CODE.NOT_READY) { diff --git a/yarn-project/prover-client/src/prover/bb_prover.ts b/yarn-project/prover-client/src/prover/bb_prover.ts index 524343b27a3b..9b29ab7c8665 100644 --- a/yarn-project/prover-client/src/prover/bb_prover.ts +++ b/yarn-project/prover-client/src/prover/bb_prover.ts @@ -4,6 +4,7 @@ import { type BaseOrMergeRollupPublicInputs, type BaseParityInputs, type BaseRollupInputs, + Fr, type KernelCircuitPublicInputs, type MergeRollupInputs, type ParityPublicInputs, @@ -17,6 +18,7 @@ import { } from '@aztec/circuits.js'; import { randomBytes } from '@aztec/foundation/crypto'; import { createDebugLogger } from '@aztec/foundation/log'; +import { Timer } from '@aztec/foundation/timer'; import { ServerCircuitArtifacts, type ServerProtocolArtifact, @@ -39,6 +41,12 @@ import { type WitnessMap } from '@noir-lang/types'; import * as fs from 'fs/promises'; import { BB_RESULT, generateKeyForNoirCircuit, generateProof, verifyProof } from '../bb/execute.js'; +import { + circuitTypeToCircuitName, + emitCircuitProvingStats, + emitCircuitSimulationStats, + emitProofVerificationStats, +} from '../util.js'; import { type CircuitProver, KernelArtifactMapping } from './interface.js'; const logger = createDebugLogger('aztec:bb-prover'); @@ -214,7 +222,15 @@ export class BBNativeRollupProver implements CircuitProver { logger.debug(`Generating witness data for ${circuitType}`); + const simTimer = new Timer(); const outputWitness = await simulator.simulateCircuit(witnessMap, artifact); + emitCircuitSimulationStats( + circuitTypeToCircuitName(circuitType), + simTimer.ms(), + witnessMap.size * Fr.SIZE_IN_BYTES, + outputWitness.size * Fr.SIZE_IN_BYTES, + logger, + ); // Now prove the circuit from the generated witness logger.debug(`Proving ${circuitType}...`); @@ -236,6 +252,16 @@ export class BBNativeRollupProver implements CircuitProver { // Read the proof and then cleanup up our temporary directory const proofBuffer = await fs.readFile(provingResult.path!); + // does not include reading the proof from disk above because duration comes from the bb wrapper + emitCircuitProvingStats( + circuitTypeToCircuitName(circuitType), + provingResult.duration, + witnessMap.size * Fr.SIZE_IN_BYTES, + outputWitness.size * Fr.SIZE_IN_BYTES, + proofBuffer.length, + logger, + ); + await fs.rm(bbWorkingDirectory, { recursive: true, force: true }); logger.info( @@ -268,6 +294,9 @@ export class BBNativeRollupProver implements CircuitProver { throw new Error(errorMessage); } + // does not include the fs.rm above because duration comes from the bb wrapper + emitProofVerificationStats(circuitTypeToCircuitName(circuitType), result.duration, proof.buffer.length, logger); + logger.info(`Successfully verified ${circuitType} proof in ${result.duration} ms`); } diff --git a/yarn-project/prover-client/src/prover/test_circuit_prover.ts b/yarn-project/prover-client/src/prover/test_circuit_prover.ts index 37e1d6b68208..ca347fcc5076 100644 --- a/yarn-project/prover-client/src/prover/test_circuit_prover.ts +++ b/yarn-project/prover-client/src/prover/test_circuit_prover.ts @@ -1,5 +1,4 @@ import { type PublicKernelNonTailRequest, type PublicKernelTailRequest, PublicKernelType } from '@aztec/circuit-types'; -import { type CircuitSimulationStats } from '@aztec/circuit-types/stats'; import { type BaseOrMergeRollupPublicInputs, type BaseParityInputs, @@ -15,7 +14,7 @@ import { makeEmptyProof, } from '@aztec/circuits.js'; import { createDebugLogger } from '@aztec/foundation/log'; -import { elapsed } from '@aztec/foundation/timer'; +import { Timer } from '@aztec/foundation/timer'; import { BaseParityArtifact, MergeRollupArtifact, @@ -39,6 +38,7 @@ import { } from '@aztec/noir-protocol-circuits-types'; import { type SimulationProvider, WASMSimulator } from '@aztec/simulator'; +import { emitCircuitSimulationStats, mapPublicKernelToCircuitName } from '../util.js'; import { type CircuitProver, KernelArtifactMapping } from './interface.js'; /** @@ -59,13 +59,21 @@ export class TestCircuitProver implements CircuitProver { * @returns The public inputs of the parity circuit. */ public async getBaseParityProof(inputs: BaseParityInputs): Promise<[ParityPublicInputs, Proof]> { + const timer = new Timer(); const witnessMap = convertBaseParityInputsToWitnessMap(inputs); // use WASM here as it is faster for small circuits const witness = await this.wasmSimulator.simulateCircuit(witnessMap, BaseParityArtifact); - const result = convertBaseParityOutputsFromWitnessMap(witness); + emitCircuitSimulationStats( + 'base-parity', + timer.ms(), + inputs.toBuffer().length, + result.toBuffer().length, + this.logger, + ); + return Promise.resolve([result, makeEmptyProof()]); } @@ -75,6 +83,7 @@ export class TestCircuitProver implements CircuitProver { * @returns The public inputs of the parity circuit. */ public async getRootParityProof(inputs: RootParityInputs): Promise<[ParityPublicInputs, Proof]> { + const timer = new Timer(); const witnessMap = convertRootParityInputsToWitnessMap(inputs); // use WASM here as it is faster for small circuits @@ -82,6 +91,14 @@ export class TestCircuitProver implements CircuitProver { const result = convertRootParityOutputsFromWitnessMap(witness); + emitCircuitSimulationStats( + 'root-parity', + timer.ms(), + inputs.toBuffer().length, + result.toBuffer().length, + this.logger, + ); + return Promise.resolve([result, makeEmptyProof()]); } @@ -91,6 +108,7 @@ export class TestCircuitProver implements CircuitProver { * @returns The public inputs as outputs of the simulation. */ public async getBaseRollupProof(input: BaseRollupInputs): Promise<[BaseOrMergeRollupPublicInputs, Proof]> { + const timer = new Timer(); const witnessMap = convertSimulatedBaseRollupInputsToWitnessMap(input); const simulationProvider = this.simulationProvider ?? this.wasmSimulator; @@ -98,6 +116,14 @@ export class TestCircuitProver implements CircuitProver { const result = convertSimulatedBaseRollupOutputsFromWitnessMap(witness); + emitCircuitSimulationStats( + 'base-rollup', + timer.ms(), + input.toBuffer().length, + result.toBuffer().length, + this.logger, + ); + return Promise.resolve([result, makeEmptyProof()]); } /** @@ -106,6 +132,7 @@ export class TestCircuitProver implements CircuitProver { * @returns The public inputs as outputs of the simulation. */ public async getMergeRollupProof(input: MergeRollupInputs): Promise<[BaseOrMergeRollupPublicInputs, Proof]> { + const timer = new Timer(); const witnessMap = convertMergeRollupInputsToWitnessMap(input); // use WASM here as it is faster for small circuits @@ -113,6 +140,14 @@ export class TestCircuitProver implements CircuitProver { const result = convertMergeRollupOutputsFromWitnessMap(witness); + emitCircuitSimulationStats( + 'merge-rollup', + timer.ms(), + input.toBuffer().length, + result.toBuffer().length, + this.logger, + ); + return Promise.resolve([result, makeEmptyProof()]); } @@ -122,26 +157,29 @@ export class TestCircuitProver implements CircuitProver { * @returns The public inputs as outputs of the simulation. */ public async getRootRollupProof(input: RootRollupInputs): Promise<[RootRollupPublicInputs, Proof]> { + const timer = new Timer(); const witnessMap = convertRootRollupInputsToWitnessMap(input); // use WASM here as it is faster for small circuits - const [duration, witness] = await elapsed(() => this.wasmSimulator.simulateCircuit(witnessMap, RootRollupArtifact)); + const witness = await this.wasmSimulator.simulateCircuit(witnessMap, RootRollupArtifact); const result = convertRootRollupOutputsFromWitnessMap(witness); - this.logger.debug(`Simulated root rollup circuit`, { - eventName: 'circuit-simulation', - circuitName: 'root-rollup', - duration, - inputSize: input.toBuffer().length, - outputSize: result.toBuffer().length, - } satisfies CircuitSimulationStats); + emitCircuitSimulationStats( + 'root-rollup', + timer.ms(), + input.toBuffer().length, + result.toBuffer().length, + this.logger, + ); + return Promise.resolve([result, makeEmptyProof()]); } public async getPublicKernelProof( kernelRequest: PublicKernelNonTailRequest, ): Promise<[PublicKernelCircuitPublicInputs, Proof]> { + const timer = new Timer(); const kernelOps = KernelArtifactMapping[kernelRequest.type]; if (kernelOps === undefined) { throw new Error(`Unable to prove for kernel type ${PublicKernelType[kernelRequest.type]}`); @@ -151,10 +189,20 @@ export class TestCircuitProver implements CircuitProver { const witness = await this.wasmSimulator.simulateCircuit(witnessMap, ServerCircuitArtifacts[kernelOps.artifact]); const result = kernelOps.convertOutputs(witness); + + emitCircuitSimulationStats( + mapPublicKernelToCircuitName(kernelRequest.type), + timer.ms(), + kernelRequest.inputs.toBuffer().length, + result.toBuffer().length, + this.logger, + ); + return [result, makeEmptyProof()]; } public async getPublicTailProof(kernelRequest: PublicKernelTailRequest): Promise<[KernelCircuitPublicInputs, Proof]> { + const timer = new Timer(); const witnessMap = convertPublicTailInputsToWitnessMap(kernelRequest.inputs); // use WASM here as it is faster for small circuits const witness = await this.wasmSimulator.simulateCircuit( @@ -163,11 +211,20 @@ export class TestCircuitProver implements CircuitProver { ); const result = convertPublicTailOutputFromWitnessMap(witness); + + emitCircuitSimulationStats( + 'public-kernel-tail', + timer.ms(), + kernelRequest.inputs.toBuffer().length, + result.toBuffer().length, + this.logger, + ); + return [result, makeEmptyProof()]; } // Not implemented for test circuits public verifyProof(_1: ServerProtocolArtifact, _2: Proof): Promise { - throw new Error('Method not implemented.'); + return Promise.reject(new Error('Method not implemented.')); } } diff --git a/yarn-project/prover-client/src/util.ts b/yarn-project/prover-client/src/util.ts new file mode 100644 index 000000000000..ec18d3e9e364 --- /dev/null +++ b/yarn-project/prover-client/src/util.ts @@ -0,0 +1,103 @@ +import { type PublicKernelRequest, PublicKernelType } from '@aztec/circuit-types'; +import type { + CircuitName, + CircuitProvingStats, + CircuitSimulationStats, + ProofVerificationStats, +} from '@aztec/circuit-types/stats'; +import { type Logger } from '@aztec/foundation/log'; +import { type ServerProtocolArtifact } from '@aztec/noir-protocol-circuits-types'; + +export function emitCircuitSimulationStats( + circuitName: CircuitName, + duration: number, + inputSize: number, + outputSize: number, + logger: Logger, +) { + const stats: CircuitSimulationStats = { + eventName: 'circuit-simulation', + circuitName, + inputSize, + outputSize, + duration, + }; + + logger.debug('Circuit simulation stats', stats); +} + +export function emitCircuitProvingStats( + circuitName: CircuitName, + duration: number, + inputSize: number, + outputSize: number, + proofSize: number, + logger: Logger, +) { + const stats: CircuitProvingStats = { + eventName: 'circuit-proving', + circuitName, + duration, + inputSize, + outputSize, + proofSize, + }; + + logger.debug('Circuit proving stats', stats); +} + +export function emitProofVerificationStats( + circuitName: CircuitName, + duration: number, + proofSize: number, + logger: Logger, +) { + const stats: ProofVerificationStats = { + eventName: 'proof-verification', + circuitName, + duration, + proofSize, + }; + + logger.debug('Proof verification stats', stats); +} + +export function mapPublicKernelToCircuitName(kernelType: PublicKernelRequest['type']): CircuitName { + switch (kernelType) { + case PublicKernelType.SETUP: + return 'public-kernel-setup'; + case PublicKernelType.APP_LOGIC: + return 'public-kernel-app-logic'; + case PublicKernelType.TEARDOWN: + return 'public-kernel-teardown'; + case PublicKernelType.TAIL: + return 'public-kernel-tail'; + default: + throw new Error(`Unknown kernel type: ${kernelType}`); + } +} + +export function circuitTypeToCircuitName(circuitType: ServerProtocolArtifact): CircuitName { + switch (circuitType) { + case 'BaseParityArtifact': + return 'base-parity'; + case 'RootParityArtifact': + return 'root-parity'; + case 'BaseRollupArtifact': + return 'base-rollup'; + case 'MergeRollupArtifact': + return 'merge-rollup'; + case 'RootRollupArtifact': + return 'root-rollup'; + case 'PublicKernelSetupArtifact': + return 'public-kernel-setup'; + case 'PublicKernelAppLogicArtifact': + return 'public-kernel-app-logic'; + case 'PublicKernelTeardownArtifact': + return 'public-kernel-teardown'; + case 'PublicKernelTailArtifact': + return 'public-kernel-tail'; + default: + throw new Error(`Unknown circuit type: ${circuitType}`); + } +}