feat: run benchmarks for ACIR proving

.github/workflows/ci.yml

@@ -78,8 +78,25 @@ jobs:
         timeout-minutes: 25
         run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --no-output +${{ matrix.test }}
 
+  acir-bench:
+    runs-on: ${{ inputs.username || github.actor }}-bench-x86
+    needs: setup-bench
+    steps:
+      - {
+          uses: actions/checkout@v4,
+          with: { ref: "${{ github.event.pull_request.head.sha }}" },
+        }
+      - uses: ./.github/ci-setup-action
+        with:
+          dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
+          concurrency_key: acir-bench-${{ inputs.username || github.actor }}-bench-x86
+      - name: ACIR Proving Bench
+        working-directory: ./noir/
+        timeout-minutes: 15
+        run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --no-output +barretenberg-acir-benches-bb
+
   # bench-summary:
-  #   needs: e2e
+  #   needs: [e2e, acir-bench]
   #   runs-on: ${{ inputs.username || github.actor }}-x86
   #   steps:
   #     - uses: actions/checkout@v4
@@ -265,12 +282,14 @@ jobs:
         timeout-minutes: 15
         run: earthly-ci --no-output +bench-ultra-honk --bench_mode=cache
 
+
   merge-check:
     runs-on: ubuntu-latest
     needs:
       - e2e
       - bb-native-tests
       - bb-bench
+      - acir-bench
       - yarn-project-formatting
       - yarn-project-test
       - prover-client-test

barretenberg/Earthfile

@@ -0,0 +1,7 @@
+VERSION 0.8
+
+acir-tests:
+    FROM ../build-images+build
+    WORKDIR /usr/src/barretenberg
+    COPY ./acir_tests .
+    SAVE ARTIFACT ./*

barretenberg/acir_tests/bench_acir_tests.sh

@@ -2,7 +2,11 @@
 
 TEST_NAMES=("$@")
 THREADS=(1 4 16 32 64)
-BENCHMARKS=$(mktemp)
+BENCHMARKS=$LOG_FILE
+
+if [[ -z "${LOG_FILE}" ]]; then
+    BENCHMARKS=$(mktemp)
+fi
 
 if [ "${#TEST_NAMES[@]}" -eq 0 ]; then
     TEST_NAMES=(sha256 ecdsa_secp256k1 ecdsa_secp256r1 schnorr double_verify_proof)
@@ -64,4 +68,6 @@ function genthreadheaders(t, len,   res) {
 }
 '
 
-rm $BENCHMARKS
+if [[ -z "${LOG_FILE}" ]]; then
+    rm $BENCHMARKS
+fi

noir/Earthfile

@@ -81,6 +81,46 @@ build:
   BUILD +nargo
   BUILD +packages
 
+build-acir-tests:
+  FROM +nargo
+  ENV PATH="/usr/src/noir-repo/target/release:${PATH}"
+  WORKDIR /usr/src/noir-repo/test_programs
+  COPY . .
+  RUN ./rebuild.sh
+  SAVE ARTIFACT /usr/src/noir-repo/test_programs/acir_artifacts/*
+
+# This is ripped from end-to-end's Earthfile, need to pull from there directly.
+UPLOAD_LOGS:
+  FUNCTION
+  ARG PULL_REQUEST
+  ARG BRANCH
+  ARG COMMIT_HASH
+  ARG LOG_FILE=./log
+  LOCALLY
+  LET COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}"
+  FROM ../+scripts
+  COPY $LOG_FILE /usr/var/log
+  ENV PULL_REQUEST=$PULL_REQUEST
+  ENV BRANCH=$BRANCH
+  ENV COMMIT_HASH=$COMMIT_HASH
+  RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY /usr/src/scripts/logs/upload_logs_to_s3.sh /usr/var/log
+
+barretenberg-acir-benches-bb:
+  FROM ../build-images/+build
+  WORKDIR /usr/src/barretenberg
+
+  COPY ../barretenberg/cpp/+preset-clang-assert/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
+  COPY +build-acir-tests/ /usr/src/acir_artifacts
+  COPY ../barretenberg/+acir-tests/ /usr/src/barretenberg/acir_tests
+
+  WORKDIR /usr/src/barretenberg/acir_tests
+
+  ENV TEST_SRC /usr/src/acir_artifacts
+  ENV LOG_FILE=/log
+  RUN touch $LOG_FILE
+  RUN ./bench_acir_tests.sh sha256
+  DO +UPLOAD_LOGS --e2e_mode=$e2e_mode --PULL_REQUEST=$PULL_REQUEST --BRANCH=$BRANCH --COMMIT_HASH=$COMMIT_HASH --LOG_FILE=/log
+
 # TOOD
 # test-packages
 # FROM aztecprotocol/noir AS noir

yarn-project/circuit-types/src/stats/metrics.ts

@@ -2,6 +2,7 @@ import { type StatsEventName } from './stats.js';
 
 /** How a metric is grouped in benchmarks: by block size, by length of chain processed, or by circuit name. */
 export type MetricGroupBy =
+  | 'threads'
   | 'block-size'
   | 'chain-length'
   | 'circuit-name'
@@ -24,6 +25,12 @@ export interface Metric {
 
 /** Metric definitions to track from benchmarks. */
 export const Metrics = [
+  {
+    name: 'proof_construction_time_sha256',
+    groupBy: 'threads',
+    description: 'Time needed to generate a proof of an ACIR program.',
+    events: ['proof_construction_time'],
+  },
   {
     name: 'l1_rollup_calldata_size_in_bytes',
     groupBy: 'block-size',

yarn-project/circuit-types/src/stats/stats.ts

@@ -1,3 +1,15 @@
+/** Stats associated with an ACIR proof generation.*/
+export type ProofConstructed = {
+  /** Name of the event for metrics purposes */
+  eventName: 'proof_construction_time';
+  /** Name of the program being proven */
+  acir_test: string;
+  /** Number of threads used for proving */
+  threads: number;
+  /** Time spent proving */
+  value: number;
+};
+
 /** Stats associated with an L2 block. */
 export type L2BlockStats = {
   /** Number of txs in the L2 block. */
@@ -235,6 +247,7 @@ export type TxAddedToPoolStats = {
 
 /** Stats emitted in structured logs with an `eventName` for tracking. */
 export type Stats =
+  | ProofConstructed
   | L1PublishStats
   | NodeSyncedChainHistoryStats
   | CircuitSimulationStats

yarn-project/end-to-end/Earthfile

@@ -28,6 +28,7 @@ UPLOAD_LOGS:
   ARG PULL_REQUEST
   ARG BRANCH
   ARG COMMIT_HASH
+  ARG LOG_FILE=./log
   LOCALLY
   LET COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}"
   FROM ../../+scripts

yarn-project/scripts/src/benchmarks/aggregate.ts

@@ -9,6 +9,7 @@
 // And then run this script from the yarn-project/scripts folder
 // LOG_FOLDER=../end-to-end/log yarn bench-aggregate
 import {
+  type ProofConstructed,
   BENCHMARK_BLOCK_SIZES,
   BENCHMARK_HISTORY_BLOCK_SIZE,
   BENCHMARK_HISTORY_CHAIN_LENGTHS,
@@ -66,6 +67,13 @@ function append(
   results[metric]![bucket].push(numeric);
 }
 
+/** Processes an entry with event name 'acir-proof-generated' and updates results */
+function processAcirProofGenerated(entry: ProofConstructed, results: BenchmarkCollectedResults) {
+  if (entry.acir_test == 'sha256') {
+    append(results, `proof_construction_time_sha256`, entry.threads, entry.value);
+  }
+}
+
 /** Processes an entry with event name 'rollup-published-to-l1' and updates results */
 function processRollupPublished(entry: L1PublishStats, results: BenchmarkCollectedResults) {
   const bucket = entry.txCount;
@@ -218,6 +226,8 @@ function processTreeInsertion(entry: TreeInsertionStats, results: BenchmarkColle
 /** Processes a parsed entry from a log-file and updates results */
 function processEntry(entry: Stats, results: BenchmarkCollectedResults, fileName: string) {
   switch (entry.eventName) {
+    case 'proof_construction_time':
+      return processAcirProofGenerated(entry, results);
     case 'rollup-published-to-l1':
       return processRollupPublished(entry, results);
     case 'l2-block-handled':

yarn-project/scripts/src/benchmarks/markdown.ts

@@ -183,6 +183,7 @@ export function getMarkdown(prNumber: number) {
   const benchmark = JSON.parse(fs.readFileSync(inputFile, 'utf-8'));
   const baseBenchmark = getBaseBenchmark();
 
+  const metricsByThreads = Metrics.filter(m => m.groupBy === 'threads').map(m => m.name);
   const metricsByBlockSize = Metrics.filter(m => m.groupBy === 'block-size').map(m => m.name);
   const metricsByChainLength = Metrics.filter(m => m.groupBy === 'chain-length').map(m => m.name);
   const metricsByCircuitName = Metrics.filter(m => m.groupBy === 'circuit-name').map(m => m.name);
@@ -217,6 +218,11 @@ All benchmarks are run on txs on the \`Benchmarking\` contract on the repository
 ${prSourceDataText}
 ${baseCommitText}
 
+### Proof generation
+
+Each column represents the number of threads used in proof generation.
+${getTableContent(pick(benchmark, metricsByThreads), baseBenchmark, 'threads')}
+
 ### L2 block published to L1
 
 Each column represents the number of txs on an L2 block published to L1.