diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 184848f54f4..6b4db3a4d53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,8 +153,39 @@ jobs: --no-output \ +${{ matrix.test }} + acir-bench: + runs-on: ubuntu-latest + needs: setup + steps: + - uses: actions/checkout@v4 + with: { ref: "${{ env.GIT_COMMIT }}" } + - uses: ./.github/ci-setup-action + - name: Setup and Test + uses: ./.github/ensure-tester-with-images + timeout-minutes: 40 + with: + runner_type: 16core-tester-x86 + builder_type: builder-x86 + # these are copied to the tester and expected by the earthly command below + # if they fail to copy, it will try to build them on the tester and fail + builder_images_to_copy: aztecprotocol/barretenberg-acir-benches:${{ env.GIT_COMMIT }} + # command to produce the images in case they don't exist + builder_command: cd noir && ../scripts/earthly-ci +export-bench-acir-bb + run: | + set -eux + cd ./noir/ + export FORCE_COLOR=1 + export EARTHLY_BUILD_ARGS="${{ env.EARTHLY_BUILD_ARGS }}" + ../scripts/earthly-ci -P \ + --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ + --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ + --no-output \ + +bench-publish-acir-bb + bench-summary: - needs: bench-e2e + needs: + - acir-bench + - bench-e2e runs-on: ${{ inputs.username || github.actor }}-x86 steps: - uses: actions/checkout@v4 @@ -181,24 +212,6 @@ jobs: working-directory: ./yarn-project/scripts run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} +bench-comment - noir-format: - needs: setup - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - - uses: ./.github/ci-setup-action - with: - concurrency_key: noir-format-x86 - - name: "Format Noir" - working-directory: ./noir/ - timeout-minutes: 40 - run: earthly-ci --no-output ./+format - - name: "Format noir-projects" - working-directory: ./noir-projects/ - timeout-minutes: 40 - run: earthly-ci --no-output ./+format - # barretenberg (prover) native and AVM (public VM) tests # only ran on x86 for resource reasons (memory intensive) bb-native-tests: @@ -232,6 +245,24 @@ jobs: timeout-minutes: 40 run: earthly-ci --no-output ./+test + noir-format: + needs: setup + runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + steps: + - uses: actions/checkout@v4 + with: { ref: "${{ env.GIT_COMMIT }}" } + - uses: ./.github/ci-setup-action + with: + concurrency_key: noir-format-x86 + - name: "Format Noir" + working-directory: ./noir/ + timeout-minutes: 40 + run: earthly-ci --no-output ./+format + - name: "Format noir-projects" + working-directory: ./noir-projects/ + timeout-minutes: 40 + run: earthly-ci --no-output ./+format + noir-test: needs: setup runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 @@ -468,6 +499,7 @@ jobs: - setup - e2e - bb-native-tests + - acir-bench - bb-bench - yarn-project-formatting - yarn-project-test diff --git a/Earthfile b/Earthfile index 155a0df3082..2c3da388fc7 100644 --- a/Earthfile +++ b/Earthfile @@ -60,4 +60,4 @@ base-log-uploader: unzip awscliv2.zip && \ ./aws/install --bin-dir /usr/local/bin --install-dir /usr/local/aws-cli --update && \ rm -rf aws awscliv2.zip - COPY +scripts/scripts /usr/src/scripts \ No newline at end of file + COPY +scripts/scripts /usr/src/scripts diff --git a/barretenberg/acir_tests/bench_acir_tests.sh b/barretenberg/acir_tests/bench_acir_tests.sh index a41b261d1cc..8cdc19a7c8f 100755 --- a/barretenberg/acir_tests/bench_acir_tests.sh +++ b/barretenberg/acir_tests/bench_acir_tests.sh @@ -1,8 +1,14 @@ #!/usr/bin/env bash +cd "$(dirname "$0")" + TEST_NAMES=("$@") THREADS=(1 4 16 32 64) -BENCHMARKS=$(mktemp) +BENCHMARKS=$LOG_FILE + +if [[ -z "${LOG_FILE}" ]]; then + BENCHMARKS=$(mktemp) +fi if [ "${#TEST_NAMES[@]}" -eq 0 ]; then TEST_NAMES=(sha256 ecdsa_secp256k1 ecdsa_secp256r1 schnorr double_verify_proof) @@ -64,4 +70,6 @@ function genthreadheaders(t, len, res) { } ' -rm $BENCHMARKS \ No newline at end of file +if [[ -z "${LOG_FILE}" ]]; then + rm $BENCHMARKS +fi diff --git a/barretenberg/cpp/src/barretenberg/common/benchmark.hpp b/barretenberg/cpp/src/barretenberg/common/benchmark.hpp index 98715635c92..ad966dc3833 100644 --- a/barretenberg/cpp/src/barretenberg/common/benchmark.hpp +++ b/barretenberg/cpp/src/barretenberg/common/benchmark.hpp @@ -94,7 +94,7 @@ template void write_benchmark(const std::string& } std::ostringstream oss; oss << "{\"timestamp\": \"" << getCurrentTimestamp() << "\", " - << "\"name\": \"" << name << "\", " + << "\"eventName\": \"" << name << "\", " << "\"type\": \"" << TypeTraits::type << "\", " << "\"value\": " << value << ", " << "\"threads\": " << env_hardware_concurrency(); diff --git a/noir/Earthfile b/noir/Earthfile index 9337b11452a..88d87214d66 100644 --- a/noir/Earthfile +++ b/noir/Earthfile @@ -216,6 +216,43 @@ build-acir-tests: RUN ./rebuild.sh true SAVE ARTIFACT /usr/src/noir-repo/test_programs/acir_artifacts/* +barretenberg-acir-benches-bb: + FROM ubuntu:noble + RUN apt update && \ + apt install -y curl git jq unzip + COPY ../barretenberg/cpp/+preset-clang-assert/bin/bb /usr/src/barretenberg/cpp/build/bin/bb + COPY ../barretenberg/+acir-tests/ /usr/src/barretenberg/acir_tests + COPY +build-acir-tests/ /usr/src/acir_artifacts + + ENV TEST_SRC=/usr/src/acir_artifacts + + WORKDIR /usr/src/barretenberg/acir_tests + +export-bench-acir-bb: + ARG EARTHLY_GIT_HASH + FROM +barretenberg-acir-benches-bb + SAVE IMAGE aztecprotocol/barretenberg-acir-benches:$EARTHLY_GIT_HASH + +bench-publish-acir-bb: + ARG PULL_REQUEST + ARG BRANCH + ARG COMMIT_HASH + LOCALLY + + # Let docker compose know about the pushed tags above + ENV AZTEC_DOCKER_TAG=$(git rev-parse HEAD) + # Optimize to not cause serial behavior if image already exists + IF ! docker image ls --format '{{.Repository}}:{{.Tag}}' | grep "aztecprotocol/barretenberg-acir-benches:$AZTEC_DOCKER_TAG" + WAIT + BUILD +export-bench-acir-bb + END + END + + RUN mkdir -p ./log + RUN docker run -v "$(pwd)/log":/log -e LOG_FILE=/log/bench-acir.jsonl --rm aztecprotocol/barretenberg-acir-benches:$AZTEC_DOCKER_TAG ./bench_acir_tests.sh sha256 + + DO ../+UPLOAD_LOGS --PULL_REQUEST=$PULL_REQUEST --BRANCH=$BRANCH --COMMIT_HASH=$COMMIT_HASH + barretenberg-acir-tests-bb: FROM ../build-images/+build diff --git a/yarn-project/circuit-types/src/stats/metrics.ts b/yarn-project/circuit-types/src/stats/metrics.ts index 41fbc543064..97e97abedfe 100644 --- a/yarn-project/circuit-types/src/stats/metrics.ts +++ b/yarn-project/circuit-types/src/stats/metrics.ts @@ -2,6 +2,7 @@ import { type StatsEventName } from './stats.js'; /** How a metric is grouped in benchmarks: by block size, by length of chain processed, or by circuit name. */ export type MetricGroupBy = + | 'threads' | 'block-size' | 'chain-length' | 'protocol-circuit-name' @@ -25,6 +26,12 @@ export interface Metric { /** Metric definitions to track from benchmarks. */ export const Metrics = [ + { + name: 'proof_construction_time_sha256', + groupBy: 'threads', + description: 'Time needed to generate a proof of an ACIR program.', + events: ['proof_construction_time'], + }, { name: 'l1_rollup_calldata_size_in_bytes', groupBy: 'block-size', diff --git a/yarn-project/circuit-types/src/stats/stats.ts b/yarn-project/circuit-types/src/stats/stats.ts index 09843ab2b03..8d956321455 100644 --- a/yarn-project/circuit-types/src/stats/stats.ts +++ b/yarn-project/circuit-types/src/stats/stats.ts @@ -1,3 +1,15 @@ +/** Stats associated with an ACIR proof generation.*/ +export type ProofConstructed = { + /** Name of the event for metrics purposes */ + eventName: 'proof_construction_time'; + /** Name of the program being proven */ + acir_test: string; + /** Number of threads used for proving */ + threads: number; + /** Time spent proving */ + value: number; +}; + /** Stats associated with an L2 block. */ export type L2BlockStats = { /** Number of txs in the L2 block. */ @@ -255,6 +267,7 @@ export type TxAddedToPoolStats = { /** Stats emitted in structured logs with an `eventName` for tracking. */ export type Stats = + | ProofConstructed | L1PublishStats | NodeSyncedChainHistoryStats | CircuitSimulationStats diff --git a/yarn-project/scripts/Earthfile b/yarn-project/scripts/Earthfile index 98f856d6fac..5f26109d9ad 100644 --- a/yarn-project/scripts/Earthfile +++ b/yarn-project/scripts/Earthfile @@ -65,5 +65,4 @@ bench-comment: RUN --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN \ [ -f $BENCH_FOLDER/benchmark.json ] \ && (cd /usr/src/yarn-project/scripts && AZTEC_BOT_COMMENTER_GITHUB_TOKEN=$AZTEC_BOT_COMMENTER_GITHUB_TOKEN yarn bench-comment) \ - || echo "No benchmark file found in $BENCH_FOLDER" - + || echo "No benchmark file found in $BENCH_FOLDER" \ No newline at end of file diff --git a/yarn-project/scripts/src/benchmarks/aggregate.ts b/yarn-project/scripts/src/benchmarks/aggregate.ts index 4d2a35f559f..caa95c22010 100644 --- a/yarn-project/scripts/src/benchmarks/aggregate.ts +++ b/yarn-project/scripts/src/benchmarks/aggregate.ts @@ -24,6 +24,7 @@ import { type MetricName, type NodeSyncedChainHistoryStats, type NoteProcessorCaughtUpStats, + type ProofConstructed, type Stats, type TreeInsertionStats, type TxAddedToPoolStats, @@ -66,6 +67,13 @@ function append( results[metric]![bucket].push(numeric); } +/** Processes an entry with event name 'acir-proof-generated' and updates results */ +function processAcirProofGenerated(entry: ProofConstructed, results: BenchmarkCollectedResults) { + if (entry.acir_test === 'sha256') { + append(results, `proof_construction_time_sha256`, entry.threads, entry.value); + } +} + /** Processes an entry with event name 'rollup-published-to-l1' and updates results */ function processRollupPublished(entry: L1PublishStats, results: BenchmarkCollectedResults) { const bucket = entry.txCount; @@ -240,6 +248,8 @@ function processTreeInsertion(entry: TreeInsertionStats, results: BenchmarkColle /** Processes a parsed entry from a log-file and updates results */ function processEntry(entry: Stats, results: BenchmarkCollectedResults, fileName: string) { switch (entry.eventName) { + case 'proof_construction_time': + return processAcirProofGenerated(entry, results); case 'rollup-published-to-l1': return processRollupPublished(entry, results); case 'l2-block-handled': diff --git a/yarn-project/scripts/src/benchmarks/markdown.ts b/yarn-project/scripts/src/benchmarks/markdown.ts index 5730067f7c5..537cdc5f487 100644 --- a/yarn-project/scripts/src/benchmarks/markdown.ts +++ b/yarn-project/scripts/src/benchmarks/markdown.ts @@ -183,6 +183,7 @@ export function getMarkdown(prNumber: number) { const benchmark = JSON.parse(fs.readFileSync(inputFile, 'utf-8')); const baseBenchmark = getBaseBenchmark(); + const metricsByThreads = Metrics.filter(m => m.groupBy === 'threads').map(m => m.name); const metricsByBlockSize = Metrics.filter(m => m.groupBy === 'block-size').map(m => m.name); const metricsByChainLength = Metrics.filter(m => m.groupBy === 'chain-length').map(m => m.name); const kernelCircuitMetrics = Metrics.filter(m => m.groupBy === 'protocol-circuit-name').map(m => m.name); @@ -218,6 +219,11 @@ All benchmarks are run on txs on the \`Benchmarking\` contract on the repository ${prSourceDataText} ${baseCommitText} +### Proof generation + +Each column represents the number of threads used in proof generation. +${getTableContent(pick(benchmark, metricsByThreads), baseBenchmark, 'threads')} + ### L2 block published to L1 Each column represents the number of txs on an L2 block published to L1.