feat: prover node metrics (#8086)

Tracks block proving duration

yarn-project/archiver/src/archiver/instrumentation.ts

@@ -37,7 +37,7 @@ export class ArchiverInstrumentation {
   }
 
   public processNewBlocks(syncTimePerBlock: number, blocks: L2Block[]) {
-    this.syncDuration.record(syncTimePerBlock);
+    this.syncDuration.record(Math.ceil(syncTimePerBlock));
     this.blockHeight.record(Math.max(...blocks.map(b => b.number)));
     for (const block of blocks) {
       this.blockSize.record(block.body.txEffects.length);

yarn-project/prover-node/src/job/block-proving-job.ts

@@ -11,11 +11,14 @@ import {
   type TxProvider,
 } from '@aztec/circuit-types';
 import { createDebugLogger } from '@aztec/foundation/log';
+import { Timer } from '@aztec/foundation/timer';
 import { type L1Publisher } from '@aztec/sequencer-client';
 import { type PublicProcessor, type PublicProcessorFactory } from '@aztec/simulator';
 
 import * as crypto from 'node:crypto';
 
+import { type ProverNodeMetrics } from '../metrics.js';
+
 /**
  * Job that grabs a range of blocks from the unfinalised chain from L1, gets their txs given their hashes,
  * re-executes their public calls, generates a rollup proof, and submits it to L1. This job will update the
@@ -33,6 +36,7 @@ export class BlockProvingJob {
     private l2BlockSource: L2BlockSource,
     private l1ToL2MessageSource: L1ToL2MessageSource,
     private txProvider: TxProvider,
+    private metrics: ProverNodeMetrics,
     private cleanUp: (job: BlockProvingJob) => Promise<void> = () => Promise.resolve(),
   ) {
     this.uuid = crypto.randomUUID();
@@ -53,6 +57,7 @@ export class BlockProvingJob {
 
     this.log.info(`Starting block proving job`, { fromBlock, toBlock, uuid: this.uuid });
     this.state = 'processing';
+    const timer = new Timer();
     try {
       let historicalHeader = (await this.l2BlockSource.getBlock(fromBlock - 1))?.header;
       for (let blockNumber = fromBlock; blockNumber <= toBlock; blockNumber++) {
@@ -114,6 +119,7 @@ export class BlockProvingJob {
       this.log.info(`Submitted proof for block range`, { fromBlock, toBlock, uuid: this.uuid });
 
       this.state = 'completed';
+      this.metrics.recordProvingJob(timer);
     } catch (err) {
       this.log.error(`Error running block prover job`, err, { uuid: this.uuid });
       this.state = 'failed';

yarn-project/prover-node/src/metrics.ts

@@ -0,0 +1,23 @@
+import { type Timer } from '@aztec/foundation/timer';
+import { type Histogram, Metrics, type TelemetryClient, ValueType, millisecondBuckets } from '@aztec/telemetry-client';
+
+export class ProverNodeMetrics {
+  provingJobDuration: Histogram;
+
+  constructor(client: TelemetryClient, name = 'ProverNode') {
+    const meter = client.getMeter(name);
+    this.provingJobDuration = meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, {
+      description: 'Duration of proving job',
+      unit: 'ms',
+      valueType: ValueType.INT,
+      advice: {
+        explicitBucketBoundaries: millisecondBuckets(2), // 60 buckets spanning an interval of ~100ms to ~1hour
+      },
+    });
+  }
+
+  public recordProvingJob(timerOrMs: Timer | number) {
+    const ms = Math.ceil(typeof timerOrMs === 'number' ? timerOrMs : timerOrMs.ms());
+    this.provingJobDuration.record(ms);
+  }
+}

yarn-project/prover-node/src/prover-node.test.ts

@@ -7,7 +7,7 @@ import {
 } from '@aztec/circuit-types';
 import { type L1Publisher } from '@aztec/sequencer-client';
 import { type PublicProcessorFactory, type SimulationProvider } from '@aztec/simulator';
-import { type TelemetryClient } from '@aztec/telemetry-client';
+import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
 import { type ContractDataSource } from '@aztec/types/contracts';
 import { WorldStateRunningState, type WorldStateSynchronizer } from '@aztec/world-state';
 
@@ -25,7 +25,6 @@ describe('prover-node', () => {
   let worldState: MockProxy<WorldStateSynchronizer>;
   let txProvider: MockProxy<TxProvider>;
   let simulator: MockProxy<SimulationProvider>;
-  let telemetryClient: MockProxy<TelemetryClient>;
 
   let proverNode: TestProverNode;
 
@@ -45,7 +44,7 @@ describe('prover-node', () => {
     worldState = mock<WorldStateSynchronizer>();
     txProvider = mock<TxProvider>();
     simulator = mock<SimulationProvider>();
-    telemetryClient = mock<TelemetryClient>();
+    const telemetryClient = new NoopTelemetryClient();
 
     // World state returns a new mock db every time it is asked to fork
     worldState.syncImmediateAndFork.mockImplementation(() => Promise.resolve(mock<MerkleTreeOperations>()));

yarn-project/prover-node/src/prover-node.ts

@@ -14,6 +14,7 @@ import { type ContractDataSource } from '@aztec/types/contracts';
 import { type WorldStateSynchronizer } from '@aztec/world-state';
 
 import { BlockProvingJob, type BlockProvingJobState } from './job/block-proving-job.js';
+import { ProverNodeMetrics } from './metrics.js';
 
 /**
  * An Aztec Prover Node is a standalone process that monitors the unfinalised chain on L1 for unproven blocks,
@@ -26,6 +27,7 @@ export class ProverNode {
   private latestBlockWeAreProving: number | undefined;
   private jobs: Map<string, BlockProvingJob> = new Map();
   private options: { pollingIntervalMs: number; disableAutomaticProving: boolean; maxPendingJobs: number };
+  private metrics: ProverNodeMetrics;
 
   constructor(
     private prover: ProverClient,
@@ -45,6 +47,8 @@ export class ProverNode {
       maxPendingJobs: 100,
       ...options,
     };
+
+    this.metrics = new ProverNodeMetrics(telemetryClient, 'ProverNode');
   }
 
   /**
@@ -200,6 +204,7 @@ export class ProverNode {
       this.l2BlockSource,
       this.l1ToL2MessageSource,
       this.txProvider,
+      this.metrics,
       cleanUp,
     );
   }

yarn-project/telemetry-client/src/metrics.ts

@@ -63,6 +63,8 @@ export const PROVING_ORCHESTRATOR_BASE_ROLLUP_INPUTS_DURATION =
 export const PROVING_QUEUE_JOB_SIZE = 'aztec.proving_queue.job_size';
 export const PROVING_QUEUE_SIZE = 'aztec.proving_queue.size';
 
+export const PROVER_NODE_JOB_DURATION = 'aztec.prover_node.job_duration';
+
 export const WORLD_STATE_FORK_DURATION = 'aztec.world_state.fork.duration';
 export const WORLD_STATE_SYNC_DURATION = 'aztec.world_state.sync.duration';
 export const WORLD_STATE_MERKLE_TREE_SIZE = 'aztec.world_state.merkle_tree_size';