From 27d3d76b6b07310251b3c1d97beb51a8d2e910c9 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 1 May 2024 23:04:40 +0000 Subject: [PATCH] feat: run benchmarks for ACIR proving --- .github/workflows/ci.yml | 21 +++++++++- barretenberg/Earthfile | 7 ++++ barretenberg/acir_tests/bench_acir_tests.sh | 10 ++++- noir/Earthfile | 40 +++++++++++++++++++ .../circuit-types/src/stats/metrics.ts | 7 ++++ yarn-project/circuit-types/src/stats/stats.ts | 13 ++++++ yarn-project/end-to-end/Earthfile | 1 + .../scripts/src/benchmarks/aggregate.ts | 10 +++++ .../scripts/src/benchmarks/markdown.ts | 6 +++ 9 files changed, 112 insertions(+), 3 deletions(-) create mode 100644 barretenberg/Earthfile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad8870e6ec1..a5f2001e987 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,8 +78,25 @@ jobs: timeout-minutes: 25 run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --no-output +${{ matrix.test }} + acir-bench: + runs-on: ${{ inputs.username || github.actor }}-bench-x86 + needs: setup-bench + steps: + - { + uses: actions/checkout@v4, + with: { ref: "${{ github.event.pull_request.head.sha }}" }, + } + - uses: ./.github/ci-setup-action + with: + dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + concurrency_key: acir-bench-${{ inputs.username || github.actor }}-bench-x86 + - name: ACIR Proving Bench + working-directory: ./noir/ + timeout-minutes: 15 + run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --no-output +barretenberg-acir-benches-bb + # bench-summary: - # needs: e2e + # needs: [e2e, acir-bench] # runs-on: ${{ inputs.username || github.actor }}-x86 # steps: # - uses: actions/checkout@v4 @@ -265,12 +282,14 @@ jobs: timeout-minutes: 15 run: earthly-ci --no-output +bench-ultra-honk --bench_mode=cache + merge-check: runs-on: ubuntu-latest needs: - e2e - bb-native-tests - bb-bench + - acir-bench - yarn-project-formatting - yarn-project-test - prover-client-test diff --git a/barretenberg/Earthfile b/barretenberg/Earthfile new file mode 100644 index 00000000000..b6636cb671f --- /dev/null +++ b/barretenberg/Earthfile @@ -0,0 +1,7 @@ +VERSION 0.8 + +acir-tests: + FROM ../build-images+build + WORKDIR /usr/src/barretenberg + COPY ./acir_tests . + SAVE ARTIFACT ./* diff --git a/barretenberg/acir_tests/bench_acir_tests.sh b/barretenberg/acir_tests/bench_acir_tests.sh index a41b261d1cc..05d763cea89 100755 --- a/barretenberg/acir_tests/bench_acir_tests.sh +++ b/barretenberg/acir_tests/bench_acir_tests.sh @@ -2,7 +2,11 @@ TEST_NAMES=("$@") THREADS=(1 4 16 32 64) -BENCHMARKS=$(mktemp) +BENCHMARKS=$LOG_FILE + +if [[ -z "${LOG_FILE}" ]]; then + BENCHMARKS=$(mktemp) +fi if [ "${#TEST_NAMES[@]}" -eq 0 ]; then TEST_NAMES=(sha256 ecdsa_secp256k1 ecdsa_secp256r1 schnorr double_verify_proof) @@ -64,4 +68,6 @@ function genthreadheaders(t, len, res) { } ' -rm $BENCHMARKS \ No newline at end of file +if [[ -z "${LOG_FILE}" ]]; then + rm $BENCHMARKS +fi diff --git a/noir/Earthfile b/noir/Earthfile index 18faae05e2e..63b3d73ce6e 100644 --- a/noir/Earthfile +++ b/noir/Earthfile @@ -81,6 +81,46 @@ build: BUILD +nargo BUILD +packages +build-acir-tests: + FROM +nargo + ENV PATH="/usr/src/noir-repo/target/release:${PATH}" + WORKDIR /usr/src/noir-repo/test_programs + COPY . . + RUN ./rebuild.sh + SAVE ARTIFACT /usr/src/noir-repo/test_programs/acir_artifacts/* + +# This is ripped from end-to-end's Earthfile, need to pull from there directly. +UPLOAD_LOGS: + FUNCTION + ARG PULL_REQUEST + ARG BRANCH + ARG COMMIT_HASH + ARG LOG_FILE=./log + LOCALLY + LET COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" + FROM ../+scripts + COPY $LOG_FILE /usr/var/log + ENV PULL_REQUEST=$PULL_REQUEST + ENV BRANCH=$BRANCH + ENV COMMIT_HASH=$COMMIT_HASH + RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY /usr/src/scripts/logs/upload_logs_to_s3.sh /usr/var/log + +barretenberg-acir-benches-bb: + FROM ../build-images/+build + WORKDIR /usr/src/barretenberg + + COPY ../barretenberg/cpp/+preset-clang-assert/bin/bb /usr/src/barretenberg/cpp/build/bin/bb + COPY +build-acir-tests/ /usr/src/acir_artifacts + COPY ../barretenberg/+acir-tests/ /usr/src/barretenberg/acir_tests + + WORKDIR /usr/src/barretenberg/acir_tests + + ENV TEST_SRC /usr/src/acir_artifacts + ENV LOG_FILE=/log + RUN touch $LOG_FILE + RUN ./bench_acir_tests.sh sha256 + DO +UPLOAD_LOGS --e2e_mode=$e2e_mode --PULL_REQUEST=$PULL_REQUEST --BRANCH=$BRANCH --COMMIT_HASH=$COMMIT_HASH --LOG_FILE=/log + # TOOD # test-packages # FROM aztecprotocol/noir AS noir diff --git a/yarn-project/circuit-types/src/stats/metrics.ts b/yarn-project/circuit-types/src/stats/metrics.ts index 5b7217e42f5..f642c2f3b83 100644 --- a/yarn-project/circuit-types/src/stats/metrics.ts +++ b/yarn-project/circuit-types/src/stats/metrics.ts @@ -2,6 +2,7 @@ import { type StatsEventName } from './stats.js'; /** How a metric is grouped in benchmarks: by block size, by length of chain processed, or by circuit name. */ export type MetricGroupBy = + | 'threads' | 'block-size' | 'chain-length' | 'circuit-name' @@ -24,6 +25,12 @@ export interface Metric { /** Metric definitions to track from benchmarks. */ export const Metrics = [ + { + name: 'proof_construction_time_sha256', + groupBy: 'threads', + description: 'Time needed to generate a proof of an ACIR program.', + events: ['proof_construction_time'], + }, { name: 'l1_rollup_calldata_size_in_bytes', groupBy: 'block-size', diff --git a/yarn-project/circuit-types/src/stats/stats.ts b/yarn-project/circuit-types/src/stats/stats.ts index 6c28b6087eb..8873c535ff6 100644 --- a/yarn-project/circuit-types/src/stats/stats.ts +++ b/yarn-project/circuit-types/src/stats/stats.ts @@ -1,3 +1,15 @@ +/** Stats associated with an ACIR proof generation.*/ +export type ProofConstructed = { + /** Name of the event for metrics purposes */ + eventName: 'proof_construction_time'; + /** Name of the program being proven */ + acir_test: string; + /** Number of threads used for proving */ + threads: number; + /** Time spent proving */ + value: number; +}; + /** Stats associated with an L2 block. */ export type L2BlockStats = { /** Number of txs in the L2 block. */ @@ -235,6 +247,7 @@ export type TxAddedToPoolStats = { /** Stats emitted in structured logs with an `eventName` for tracking. */ export type Stats = + | ProofConstructed | L1PublishStats | NodeSyncedChainHistoryStats | CircuitSimulationStats diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index 8ff7a7f9aa1..baa8913be6f 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -28,6 +28,7 @@ UPLOAD_LOGS: ARG PULL_REQUEST ARG BRANCH ARG COMMIT_HASH + ARG LOG_FILE=./log LOCALLY LET COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" FROM ../../+scripts diff --git a/yarn-project/scripts/src/benchmarks/aggregate.ts b/yarn-project/scripts/src/benchmarks/aggregate.ts index f0bf75811f9..bc8b1ea5154 100644 --- a/yarn-project/scripts/src/benchmarks/aggregate.ts +++ b/yarn-project/scripts/src/benchmarks/aggregate.ts @@ -9,6 +9,7 @@ // And then run this script from the yarn-project/scripts folder // LOG_FOLDER=../end-to-end/log yarn bench-aggregate import { + type ProofConstructed, BENCHMARK_BLOCK_SIZES, BENCHMARK_HISTORY_BLOCK_SIZE, BENCHMARK_HISTORY_CHAIN_LENGTHS, @@ -66,6 +67,13 @@ function append( results[metric]![bucket].push(numeric); } +/** Processes an entry with event name 'acir-proof-generated' and updates results */ +function processAcirProofGenerated(entry: ProofConstructed, results: BenchmarkCollectedResults) { + if (entry.acir_test == 'sha256') { + append(results, `proof_construction_time_sha256`, entry.threads, entry.value); + } +} + /** Processes an entry with event name 'rollup-published-to-l1' and updates results */ function processRollupPublished(entry: L1PublishStats, results: BenchmarkCollectedResults) { const bucket = entry.txCount; @@ -218,6 +226,8 @@ function processTreeInsertion(entry: TreeInsertionStats, results: BenchmarkColle /** Processes a parsed entry from a log-file and updates results */ function processEntry(entry: Stats, results: BenchmarkCollectedResults, fileName: string) { switch (entry.eventName) { + case 'proof_construction_time': + return processAcirProofGenerated(entry, results); case 'rollup-published-to-l1': return processRollupPublished(entry, results); case 'l2-block-handled': diff --git a/yarn-project/scripts/src/benchmarks/markdown.ts b/yarn-project/scripts/src/benchmarks/markdown.ts index 772238679a9..6558145f0c5 100644 --- a/yarn-project/scripts/src/benchmarks/markdown.ts +++ b/yarn-project/scripts/src/benchmarks/markdown.ts @@ -183,6 +183,7 @@ export function getMarkdown(prNumber: number) { const benchmark = JSON.parse(fs.readFileSync(inputFile, 'utf-8')); const baseBenchmark = getBaseBenchmark(); + const metricsByThreads = Metrics.filter(m => m.groupBy === 'threads').map(m => m.name); const metricsByBlockSize = Metrics.filter(m => m.groupBy === 'block-size').map(m => m.name); const metricsByChainLength = Metrics.filter(m => m.groupBy === 'chain-length').map(m => m.name); const metricsByCircuitName = Metrics.filter(m => m.groupBy === 'circuit-name').map(m => m.name); @@ -217,6 +218,11 @@ All benchmarks are run on txs on the \`Benchmarking\` contract on the repository ${prSourceDataText} ${baseCommitText} +### Proof generation + +Each column represents the number of threads used in proof generation. +${getTableContent(pick(benchmark, metricsByThreads), baseBenchmark, 'threads')} + ### L2 block published to L1 Each column represents the number of txs on an L2 block published to L1.