From 5f6b52f7363d8e59366c0fff07f91595452219d6 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Fri, 26 Apr 2024 07:04:23 +0000 Subject: [PATCH] feat: prover metrics --- .../circuit-types/src/stats/metrics.ts | 20 +++- yarn-project/circuit-types/src/stats/stats.ts | 44 ++++++--- .../src/orchestrator/orchestrator.ts | 94 ++----------------- .../prover-client/src/prover/bb_prover.ts | 20 ++++ .../src/prover/test_circuit_prover.ts | 76 ++++++++++++--- yarn-project/prover-client/src/util.ts | 82 ++++++++++++++++ 6 files changed, 224 insertions(+), 112 deletions(-) create mode 100644 yarn-project/prover-client/src/util.ts diff --git a/yarn-project/circuit-types/src/stats/metrics.ts b/yarn-project/circuit-types/src/stats/metrics.ts index 4ee8ea7e04c7..d8d5726e54ff 100644 --- a/yarn-project/circuit-types/src/stats/metrics.ts +++ b/yarn-project/circuit-types/src/stats/metrics.ts @@ -113,7 +113,13 @@ export const Metrics = [ { name: 'circuit_simulation_time_in_ms', groupBy: 'circuit-name', - description: 'Time to run a circuit simulation.', + description: 'Time to run a circuit simulation using WASM.', + events: ['circuit-simulation'], + }, + { + name: 'circuit_partial_witness_generation_time_in_ms', + groupBy: 'circuit-name', + description: 'Time to generate the partial witness for a circuit', events: ['circuit-simulation'], }, { @@ -128,6 +134,18 @@ export const Metrics = [ description: 'Size of the outputs (ie public inputs) from a circuit simulation.', events: ['circuit-simulation'], }, + { + name: 'circuit_proving_time_in_ms', + groupBy: 'circuit-name', + description: 'Time to prove circuit execution.', + events: ['circuit-proving'], + }, + { + name: 'circuit_proof_size_in_bytes', + groupBy: 'circuit-name', + description: 'Size of the proof produced by a circuit.', + events: ['circuit-proving'], + }, { name: 'tx_size_in_bytes', groupBy: 'classes-registered', diff --git a/yarn-project/circuit-types/src/stats/stats.ts b/yarn-project/circuit-types/src/stats/stats.ts index 14d159a1e991..2386869c7af2 100644 --- a/yarn-project/circuit-types/src/stats/stats.ts +++ b/yarn-project/circuit-types/src/stats/stats.ts @@ -48,30 +48,47 @@ export type NodeSyncedChainHistoryStats = { dbSize: number; }; +export type CircuitName = + | 'base-parity' + | 'root-parity' + | 'base-rollup' + | 'private-kernel-init' + | 'private-kernel-ordering' + | 'root-rollup' + | 'merge-rollup' + | 'private-kernel-inner' + | 'public-kernel-setup' + | 'public-kernel-app-logic' + | 'public-kernel-teardown' + | 'public-kernel-tail'; + /** Stats for circuit simulation. */ export type CircuitSimulationStats = { /** name of the event. */ eventName: 'circuit-simulation'; /** Name of the circuit. */ - circuitName: - | 'base-parity' - | 'root-parity' - | 'base-rollup' - | 'private-kernel-init' - | 'private-kernel-ordering' - | 'root-rollup' - | 'merge-rollup' - | 'private-kernel-inner' - | 'public-kernel-setup' - | 'public-kernel-app-logic' - | 'public-kernel-teardown' - | 'public-kernel-tail'; + circuitName: CircuitName; + /** Duration in ms. */ + duration: number; + /** Size in bytes of circuit inputs. */ + inputSize: number; + /** Size in bytes of circuit outputs (aka public inputs). */ + outputSize: number; +}; + +export type CircuitProvingStats = { + /** Name of the event. */ + eventName: 'circuit-proving'; + /** Name of the circuit. */ + circuitName: CircuitName; /** Duration in ms. */ duration: number; /** Size in bytes of circuit inputs. */ inputSize: number; /** Size in bytes of circuit outputs (aka public inputs). */ outputSize: number; + /** Size in bytes of the proof. */ + proofSize: number; }; /** Stats for an L2 block built by a sequencer. */ @@ -206,6 +223,7 @@ export type Stats = | L1PublishStats | NodeSyncedChainHistoryStats | CircuitSimulationStats + | CircuitProvingStats | L2BlockBuiltStats | L2BlockHandledStats | NoteProcessorCaughtUpStats diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.ts index e79ec164e445..bcbcf85fe862 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.ts @@ -3,7 +3,6 @@ import { L2Block, MerkleTreeId, type ProcessedTx, - type PublicKernelRequest, PublicKernelType, type TxEffect, toTxEffect, @@ -15,7 +14,6 @@ import { type ProvingTicket, type PublicInputsAndProof, } from '@aztec/circuit-types/interfaces'; -import { type CircuitSimulationStats } from '@aztec/circuit-types/stats'; import { type BaseOrMergeRollupPublicInputs, BaseParityInputs, @@ -41,7 +39,6 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { promiseWithResolvers } from '@aztec/foundation/promise'; import { type Tuple } from '@aztec/foundation/serialize'; import { sleep } from '@aztec/foundation/sleep'; -import { Timer } from '@aztec/foundation/timer'; import { type MerkleTreeOperations } from '@aztec/world-state'; import { inspect } from 'util'; @@ -311,7 +308,7 @@ export class ProvingOrchestrator { private deferredProving( provingState: ProvingState | undefined, request: (signal: AbortSignal) => Promise, - callback: (result: T, durationMs: number) => void | Promise, + callback: (result: T) => void | Promise, ) { if (!provingState?.verifyState()) { logger.debug(`Not enqueuing job, state no longer valid`); @@ -330,10 +327,7 @@ export class ProvingOrchestrator { return; } - const timer = new Timer(); const result = await request(controller.signal); - const duration = timer.ms(); - if (!provingState?.verifyState()) { logger.debug(`State no longer valid, discarding result`); return; @@ -345,7 +339,7 @@ export class ProvingOrchestrator { return; } - await callback(result, duration); + await callback(result); } catch (err) { if (err instanceof AbortedError) { // operation was cancelled, probably because the block was cancelled @@ -367,39 +361,6 @@ export class ProvingOrchestrator { setImmediate(safeJob); } - private emitCircuitSimulationStats( - circuitName: CircuitSimulationStats['circuitName'] | null, - inputSize: number, - outputSize: number, - duration: number, - ) { - const stats: CircuitSimulationStats | undefined = circuitName - ? { - eventName: 'circuit-simulation', - circuitName, - duration, - inputSize, - outputSize, - } - : undefined; - logger.debug(`Simulated ${circuitName} circuit duration=${duration}ms`, stats); - } - - private getPublicKernelCircuitName(request: PublicKernelRequest) { - switch (request.type) { - case PublicKernelType.SETUP: - return 'public-kernel-setup'; - case PublicKernelType.APP_LOGIC: - return 'public-kernel-app-logic'; - case PublicKernelType.TEARDOWN: - return 'public-kernel-teardown'; - case PublicKernelType.TAIL: - return 'public-kernel-tail'; - default: - return null; - } - } - // Updates the merkle trees for a transaction. The first enqueued job for a transaction private async prepareBaseRollupInputs( provingState: ProvingState | undefined, @@ -475,14 +436,7 @@ export class ProvingOrchestrator { this.deferredProving( provingState, signal => this.prover.getBaseRollupProof(tx.baseRollupInputs, signal), - (result, duration) => { - this.emitCircuitSimulationStats( - 'base-rollup', - tx.baseRollupInputs.toBuffer().length, - result.inputs.toBuffer().length, - duration, - ); - + result => { validatePartialState(result.inputs.end, tx.treeSnapshots); const currentLevel = provingState.numMergeLevels + 1n; this.storeAndExecuteNextMergeLevel(provingState, currentLevel, index, [result.inputs, result.proof]); @@ -506,13 +460,7 @@ export class ProvingOrchestrator { this.deferredProving( provingState, signal => this.prover.getMergeRollupProof(inputs, signal), - (result, duration) => { - this.emitCircuitSimulationStats( - 'merge-rollup', - inputs.toBuffer().length, - result.inputs.toBuffer().length, - duration, - ); + result => { this.storeAndExecuteNextMergeLevel(provingState, level, index, [result.inputs, result.proof]); }, ); @@ -542,14 +490,7 @@ export class ProvingOrchestrator { this.deferredProving( provingState, signal => this.prover.getRootRollupProof(inputs, signal), - (result, duration) => { - this.emitCircuitSimulationStats( - 'root-rollup', - inputs.toBuffer().length, - result.inputs.toBuffer().length, - duration, - ); - + result => { provingState.rootRollupPublicInputs = result.inputs; provingState.finalProof = result.proof; @@ -567,13 +508,7 @@ export class ProvingOrchestrator { this.deferredProving( provingState, signal => this.prover.getBaseParityProof(inputs, signal), - (rootInput, duration) => { - this.emitCircuitSimulationStats( - 'base-parity', - inputs.toBuffer().length, - rootInput.publicInputs.toBuffer().length, - duration, - ); + rootInput => { provingState.setRootParityInputs(rootInput, index); if (provingState.areRootParityInputsReady()) { const rootParityInputs = new RootParityInputs( @@ -594,13 +529,7 @@ export class ProvingOrchestrator { this.deferredProving( provingState, signal => this.prover.getRootParityProof(inputs, signal), - async (rootInput, duration) => { - this.emitCircuitSimulationStats( - 'root-parity', - inputs.toBuffer().length, - rootInput.publicInputs.toBuffer().length, - duration, - ); + async rootInput => { provingState!.finalRootParityInput = rootInput; await this.checkAndEnqueueRootRollup(provingState); }, @@ -714,14 +643,7 @@ export class ProvingOrchestrator { return this.prover.getPublicKernelProof(request, signal); } }, - (result, duration) => { - this.emitCircuitSimulationStats( - this.getPublicKernelCircuitName(request), - request.inputs.toBuffer().length, - 0, - duration, - ); - + result => { const nextKernelRequest = txProvingState.getNextPublicKernelFromKernelProof(functionIndex, result.proof); // What's the status of the next kernel? if (nextKernelRequest.code === TX_PROVING_CODE.NOT_READY) { diff --git a/yarn-project/prover-client/src/prover/bb_prover.ts b/yarn-project/prover-client/src/prover/bb_prover.ts index f75d6fb16850..96ce88a86276 100644 --- a/yarn-project/prover-client/src/prover/bb_prover.ts +++ b/yarn-project/prover-client/src/prover/bb_prover.ts @@ -31,6 +31,7 @@ import { import { randomBytes } from '@aztec/foundation/crypto'; import { createDebugLogger } from '@aztec/foundation/log'; import { type Tuple } from '@aztec/foundation/serialize'; +import { Timer } from '@aztec/foundation/timer'; import { ServerCircuitArtifacts, type ServerProtocolArtifact, @@ -62,6 +63,7 @@ import { generateProof, verifyProof, } from '../bb/execute.js'; +import { circuitTypeToCircuitName, emitCircuitProvingStats, emitCircuitSimulationStats } from '../util.js'; import { type CircuitProver, KernelArtifactMapping } from './interface.js'; const logger = createDebugLogger('aztec:bb-prover'); @@ -268,7 +270,15 @@ export class BBNativeRollupProver implements CircuitProver { logger.debug(`Generating witness data for ${circuitType}`); + const simTimer = new Timer(); const outputWitness = await simulator.simulateCircuit(witnessMap, artifact); + emitCircuitSimulationStats( + circuitTypeToCircuitName(circuitType), + simTimer.ms(), + witnessMap.size * Fr.SIZE_IN_BYTES, + outputWitness.size * Fr.SIZE_IN_BYTES, + logger, + ); // Now prove the circuit from the generated witness logger.debug(`Proving ${circuitType}...`); @@ -293,6 +303,16 @@ export class BBNativeRollupProver implements CircuitProver { // Read the proof and then cleanup up our temporary directory const proof = await fs.readFile(`${provingResult.proofPath!}/${PROOF_FILENAME}`); + // does not include reading the proof from disk above because duration comes from the bb wrapper + emitCircuitProvingStats( + circuitTypeToCircuitName(circuitType), + provingResult.duration, + witnessMap.size * Fr.SIZE_IN_BYTES, + outputWitness.size * Fr.SIZE_IN_BYTES, + proof.length, + logger, + ); + await fs.rm(bbWorkingDirectory, { recursive: true, force: true }); logger.info(`Generated proof for ${circuitType} in ${provingResult.duration} ms, size: ${proof.length} fields`); diff --git a/yarn-project/prover-client/src/prover/test_circuit_prover.ts b/yarn-project/prover-client/src/prover/test_circuit_prover.ts index 6d2acdfa2e8b..226a2cff0092 100644 --- a/yarn-project/prover-client/src/prover/test_circuit_prover.ts +++ b/yarn-project/prover-client/src/prover/test_circuit_prover.ts @@ -5,7 +5,6 @@ import { PublicKernelType, makePublicInputsAndProof, } from '@aztec/circuit-types'; -import { type CircuitSimulationStats } from '@aztec/circuit-types/stats'; import { type BaseOrMergeRollupPublicInputs, type BaseParityInputs, @@ -25,7 +24,7 @@ import { makeRecursiveProof, } from '@aztec/circuits.js'; import { createDebugLogger } from '@aztec/foundation/log'; -import { elapsed } from '@aztec/foundation/timer'; +import { Timer } from '@aztec/foundation/timer'; import { BaseParityArtifact, MergeRollupArtifact, @@ -49,6 +48,7 @@ import { } from '@aztec/noir-protocol-circuits-types'; import { type SimulationProvider, WASMSimulator } from '@aztec/simulator'; +import { emitCircuitSimulationStats, mapPublicKernelToCircuitName } from '../util.js'; import { type CircuitProver, KernelArtifactMapping } from './interface.js'; const VERIFICATION_KEYS: Record = { @@ -81,11 +81,11 @@ export class TestCircuitProver implements CircuitProver { * @returns The public inputs of the parity circuit. */ public async getBaseParityProof(inputs: BaseParityInputs): Promise> { + const timer = new Timer(); const witnessMap = convertBaseParityInputsToWitnessMap(inputs); // use WASM here as it is faster for small circuits const witness = await this.wasmSimulator.simulateCircuit(witnessMap, BaseParityArtifact); - const result = convertBaseParityOutputsFromWitnessMap(witness); const rootParityInput = new RootParityInput( @@ -94,6 +94,14 @@ export class TestCircuitProver implements CircuitProver { result, ); + emitCircuitSimulationStats( + 'base-parity', + timer.ms(), + inputs.toBuffer().length, + result.toBuffer().length, + this.logger, + ); + return Promise.resolve(rootParityInput); } @@ -105,6 +113,7 @@ export class TestCircuitProver implements CircuitProver { public async getRootParityProof( inputs: RootParityInputs, ): Promise> { + const timer = new Timer(); const witnessMap = convertRootParityInputsToWitnessMap(inputs); // use WASM here as it is faster for small circuits @@ -118,6 +127,14 @@ export class TestCircuitProver implements CircuitProver { result, ); + emitCircuitSimulationStats( + 'root-parity', + timer.ms(), + inputs.toBuffer().length, + result.toBuffer().length, + this.logger, + ); + return Promise.resolve(rootParityInput); } @@ -129,6 +146,7 @@ export class TestCircuitProver implements CircuitProver { public async getBaseRollupProof( input: BaseRollupInputs, ): Promise> { + const timer = new Timer(); const witnessMap = convertSimulatedBaseRollupInputsToWitnessMap(input); const simulationProvider = this.simulationProvider ?? this.wasmSimulator; @@ -136,6 +154,13 @@ export class TestCircuitProver implements CircuitProver { const result = convertSimulatedBaseRollupOutputsFromWitnessMap(witness); + emitCircuitSimulationStats( + 'base-rollup', + timer.ms(), + input.toBuffer().length, + result.toBuffer().length, + this.logger, + ); return makePublicInputsAndProof(result, makeEmptyProof()); } /** @@ -146,6 +171,7 @@ export class TestCircuitProver implements CircuitProver { public async getMergeRollupProof( input: MergeRollupInputs, ): Promise> { + const timer = new Timer(); const witnessMap = convertMergeRollupInputsToWitnessMap(input); // use WASM here as it is faster for small circuits @@ -153,6 +179,13 @@ export class TestCircuitProver implements CircuitProver { const result = convertMergeRollupOutputsFromWitnessMap(witness); + emitCircuitSimulationStats( + 'merge-rollup', + timer.ms(), + input.toBuffer().length, + result.toBuffer().length, + this.logger, + ); return makePublicInputsAndProof(result, makeEmptyProof()); } @@ -162,26 +195,28 @@ export class TestCircuitProver implements CircuitProver { * @returns The public inputs as outputs of the simulation. */ public async getRootRollupProof(input: RootRollupInputs): Promise> { + const timer = new Timer(); const witnessMap = convertRootRollupInputsToWitnessMap(input); // use WASM here as it is faster for small circuits - const [duration, witness] = await elapsed(() => this.wasmSimulator.simulateCircuit(witnessMap, RootRollupArtifact)); + const witness = await this.wasmSimulator.simulateCircuit(witnessMap, RootRollupArtifact); const result = convertRootRollupOutputsFromWitnessMap(witness); - this.logger.debug(`Simulated root rollup circuit`, { - eventName: 'circuit-simulation', - circuitName: 'root-rollup', - duration, - inputSize: input.toBuffer().length, - outputSize: result.toBuffer().length, - } satisfies CircuitSimulationStats); + emitCircuitSimulationStats( + 'root-rollup', + timer.ms(), + input.toBuffer().length, + result.toBuffer().length, + this.logger, + ); return makePublicInputsAndProof(result, makeEmptyProof()); } public async getPublicKernelProof( kernelRequest: PublicKernelNonTailRequest, ): Promise> { + const timer = new Timer(); const kernelOps = KernelArtifactMapping[kernelRequest.type]; if (kernelOps === undefined) { throw new Error(`Unable to prove for kernel type ${PublicKernelType[kernelRequest.type]}`); @@ -191,12 +226,21 @@ export class TestCircuitProver implements CircuitProver { const witness = await this.wasmSimulator.simulateCircuit(witnessMap, ServerCircuitArtifacts[kernelOps.artifact]); const result = kernelOps.convertOutputs(witness); + emitCircuitSimulationStats( + mapPublicKernelToCircuitName(kernelRequest.type), + timer.ms(), + kernelRequest.inputs.toBuffer().length, + result.toBuffer().length, + this.logger, + ); + return makePublicInputsAndProof(result, makeEmptyProof()); } public async getPublicTailProof( kernelRequest: PublicKernelTailRequest, ): Promise> { + const timer = new Timer(); const witnessMap = convertPublicTailInputsToWitnessMap(kernelRequest.inputs); // use WASM here as it is faster for small circuits const witness = await this.wasmSimulator.simulateCircuit( @@ -205,11 +249,19 @@ export class TestCircuitProver implements CircuitProver { ); const result = convertPublicTailOutputFromWitnessMap(witness); + emitCircuitSimulationStats( + 'public-kernel-tail', + timer.ms(), + kernelRequest.inputs.toBuffer().length, + result.toBuffer().length, + this.logger, + ); + return makePublicInputsAndProof(result, makeEmptyProof()); } // Not implemented for test circuits public verifyProof(_1: ServerProtocolArtifact, _2: Proof): Promise { - throw new Error('Method not implemented.'); + return Promise.reject(new Error('Method not implemented.')); } } diff --git a/yarn-project/prover-client/src/util.ts b/yarn-project/prover-client/src/util.ts new file mode 100644 index 000000000000..694e00a6e863 --- /dev/null +++ b/yarn-project/prover-client/src/util.ts @@ -0,0 +1,82 @@ +import { type PublicKernelRequest, PublicKernelType } from '@aztec/circuit-types'; +import type { CircuitName, CircuitProvingStats, CircuitSimulationStats } from '@aztec/circuit-types/stats'; +import { type Logger } from '@aztec/foundation/log'; +import { type ServerProtocolArtifact } from '@aztec/noir-protocol-circuits-types'; + +export function emitCircuitSimulationStats( + circuitName: CircuitName, + duration: number, + inputSize: number, + outputSize: number, + logger: Logger, +) { + const stats: CircuitSimulationStats = { + eventName: 'circuit-simulation', + circuitName, + inputSize, + outputSize, + duration, + }; + + logger.debug('Circuit simulation stats', stats); +} + +export function emitCircuitProvingStats( + circuitName: CircuitName, + duration: number, + inputSize: number, + outputSize: number, + proofSize: number, + logger: Logger, +) { + const stats: CircuitProvingStats = { + eventName: 'circuit-proving', + circuitName, + duration, + inputSize, + outputSize, + proofSize, + }; + + logger.debug('Circuit proving stats', stats); +} + +export function mapPublicKernelToCircuitName(kernelType: PublicKernelRequest['type']): CircuitName { + switch (kernelType) { + case PublicKernelType.SETUP: + return 'public-kernel-setup'; + case PublicKernelType.APP_LOGIC: + return 'public-kernel-app-logic'; + case PublicKernelType.TEARDOWN: + return 'public-kernel-teardown'; + case PublicKernelType.TAIL: + return 'public-kernel-tail'; + default: + throw new Error(`Unknown kernel type: ${kernelType}`); + } +} + +export function circuitTypeToCircuitName(circuitType: ServerProtocolArtifact): CircuitName { + switch (circuitType) { + case 'BaseParityArtifact': + return 'base-parity'; + case 'RootParityArtifact': + return 'root-parity'; + case 'BaseRollupArtifact': + return 'base-rollup'; + case 'MergeRollupArtifact': + return 'merge-rollup'; + case 'RootRollupArtifact': + return 'root-rollup'; + case 'PublicKernelSetupArtifact': + return 'public-kernel-setup'; + case 'PublicKernelAppLogicArtifact': + return 'public-kernel-app-logic'; + case 'PublicKernelTeardownArtifact': + return 'public-kernel-teardown'; + case 'PublicKernelTailArtifact': + return 'public-kernel-tail'; + default: + throw new Error(`Unknown circuit type: ${circuitType}`); + } +}