diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3657b19b68..9fabf53a06d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,7 +118,8 @@ jobs: build: needs: [build-images, changes] - if: (needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' + # NOTE: we don't skip this job here instead using if's below so that dependencies are not broken + # we still want build to be at the bottom of the dependency chain runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 outputs: e2e_list: ${{ steps.e2e_list.outputs.list }} @@ -127,11 +128,13 @@ jobs: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } - uses: ./.github/ci-setup-action + if: (needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' with: concurrency_key: build-x86 # prepare images locally, tagged by commit hash - name: "Build E2E Image" timeout-minutes: 40 + if: (needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' run: | earthly-ci ./yarn-project+export-e2e-test-images # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end @@ -160,7 +163,7 @@ jobs: # all the non-bench end-to-end integration tests for aztec e2e: needs: [build, changes] - if: needs.changes.outputs.non-barretenberg-cpp == 'true' || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'e2e') + if: (needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'e2e') runs-on: ubuntu-20.04 strategy: fail-fast: false @@ -190,7 +193,7 @@ jobs: # all the benchmarking end-to-end integration tests for aztec (not required to merge) bench-e2e: needs: [build, changes] - if: needs.changes.outputs.non-barretenberg-cpp == 'true' || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'bench') + if: (needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'bench') runs-on: ubuntu-20.04 strategy: fail-fast: false @@ -221,7 +224,7 @@ jobs: acir-bench: runs-on: ubuntu-20.04 - needs: [build-images, changes] + needs: [build, changes] # Note: not fully accurate, but to work with bench-summary needs to be the same as bench-e2e if: needs.changes.outputs.non-barretenberg-cpp == 'true' steps: @@ -258,7 +261,6 @@ jobs: ref: "${{ github.event.pull_request.head.sha }}" - uses: ./.github/ci-setup-action with: - dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" concurrency_key: build-x86 - name: "Build and upload bench aggregate file" working-directory: ./yarn-project/scripts @@ -286,7 +288,7 @@ jobs: AZTEC_BOT_GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} bb-gcc: - needs: [build-images, changes] + needs: [build, changes] runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 if: needs.changes.outputs.barretenberg-cpp == 'true' steps: @@ -306,7 +308,7 @@ jobs: # barretenberg (prover) native, AVM (public VM) and Merkle tree (world state) tests # only ran on x86 for resource reasons (memory intensive) bb-native-tests: - needs: [build-images, changes] + needs: [build, changes] runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 if: needs.changes.outputs.barretenberg-cpp == 'true' steps: @@ -324,7 +326,7 @@ jobs: run: earthly-ci --exec-stats --no-output +test --hardware_concurrency=64 bb-js-test: - needs: [build-images, changes] + needs: [build, changes] runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 if: needs.changes.outputs.barretenberg == 'true' steps: @@ -339,7 +341,7 @@ jobs: run: earthly-ci --no-output ./+test noir-build-acir-tests: - needs: [build-images, changes] + needs: [build, changes] runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true' steps: @@ -413,7 +415,7 @@ jobs: run: earthly-ci --no-output ./+barretenberg-acir-tests-bb.js noir-format: - needs: [build-images, changes] + needs: [build, changes] runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 if: needs.changes.outputs.noir == 'true' || needs.changes.outputs.noir-projects == 'true' steps: @@ -433,7 +435,7 @@ jobs: earthly-ci --no-output ./+format noir-test: - needs: [build-images, changes] + needs: [build, changes] runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 if: needs.changes.outputs.noir == 'true' steps: @@ -446,7 +448,7 @@ jobs: run: earthly-ci --no-output ./noir+test noir-examples: - needs: [build-images, changes] + needs: [build, changes] runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true' steps: @@ -459,7 +461,7 @@ jobs: run: earthly-ci --no-output ./noir+examples noir-packages-test: - needs: [build-images, changes] + needs: [build, changes] runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true' steps: @@ -473,7 +475,7 @@ jobs: run: earthly-ci --no-output ./noir+packages-test noir-projects: - needs: [build-images, changes, build] + needs: [build, changes, build] runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true' || needs.changes.outputs.noir-projects == 'true' || needs.changes.outputs.txe == 'true' steps: @@ -488,7 +490,7 @@ jobs: earthly-ci --no-output ./noir-projects/+test avm-format: - needs: [build-images, changes] + needs: [build, changes] runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 if: needs.changes.outputs.avm-transpiler == 'true' || needs.changes.outputs.noir == 'true' steps: @@ -542,7 +544,7 @@ jobs: run: earthly-ci --no-output ./yarn-project/+prover-client-test l1-contracts-test: - needs: [build-images, changes] + needs: [build, changes] runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 if: needs.changes.outputs.l1-contracts == 'true' steps: @@ -555,7 +557,7 @@ jobs: run: earthly-ci --no-output ./l1-contracts+test docs-preview: - needs: [build-images, changes] + needs: [build, changes] runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 if: needs.changes.outputs.non-barretenberg-cpp == 'true' steps: @@ -576,7 +578,7 @@ jobs: bb-bench: runs-on: ubuntu-20.04 - needs: [build-images, changes] + needs: [build, changes] if: needs.changes.outputs.barretenberg-cpp == 'true' steps: - uses: actions/checkout@v4 @@ -661,7 +663,7 @@ jobs: run: earthly-ci -P --no-output +test --box=${{ matrix.box }} --browser=${{ matrix.browser }} --mode=cache protocol-circuits-gates-report: - needs: [build-images, changes] + needs: [build, changes] if: needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 permissions: @@ -697,7 +699,7 @@ jobs: message: ${{ steps.gates_diff.outputs.markdown }} public-functions-size-report: - needs: [build-images, changes] + needs: [build, changes] if: needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 permissions: diff --git a/avm-transpiler/.rebuild_patterns b/avm-transpiler/.rebuild_patterns new file mode 100644 index 00000000000..e73892d9e89 --- /dev/null +++ b/avm-transpiler/.rebuild_patterns @@ -0,0 +1 @@ +^avm-transpiler/.*$ \ No newline at end of file diff --git a/avm-transpiler/Earthfile b/avm-transpiler/Earthfile index ce6d6597cf7..95199fbe3d2 100644 --- a/avm-transpiler/Earthfile +++ b/avm-transpiler/Earthfile @@ -4,8 +4,6 @@ source: # we rely on noir source, which this image has FROM ../noir+nargo - # move noir contents to /usr/src/noir - RUN mv /usr/src /noir && mkdir /usr/src && mv /noir /usr/src # work in avm-transpiler WORKDIR /usr/src/avm-transpiler diff --git a/avm-transpiler/scripts/bootstrap_native.sh b/avm-transpiler/scripts/bootstrap_native.sh index 3e0e2ed853a..5b61375fff9 100755 --- a/avm-transpiler/scripts/bootstrap_native.sh +++ b/avm-transpiler/scripts/bootstrap_native.sh @@ -9,7 +9,7 @@ export GIT_DIRTY=false if [ -f ".gitrepo" ]; then export GIT_COMMIT=$(awk '/commit =/ {print $3}' .gitrepo) else - export GIT_COMMIT=$(git rev-parse --verify HEAD) + export GIT_COMMIT=${COMMIT_HASH:-$(git rev-parse --verify HEAD)} fi # Build native. diff --git a/barretenberg/Earthfile b/barretenberg/Earthfile index a77cede937a..a21497733e0 100644 --- a/barretenberg/Earthfile +++ b/barretenberg/Earthfile @@ -104,6 +104,8 @@ barretenberg-acir-tests-bb.js: RUN cd ../ts && yarn ENV VERBOSE=1 ENV TEST_SRC /usr/src/acir_artifacts + # We have diminishing returns after 32 cores, and unnecessarily use resources + ENV HARDWARE_CONCURRENCY=32 # TODO(https://github.com/noir-lang/noir/issues/5106) # TODO(https://github.com/AztecProtocol/aztec-packages/issues/6672)c diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile index 4fc9b14d326..1074fe85ffb 100644 --- a/barretenberg/cpp/Earthfile +++ b/barretenberg/cpp/Earthfile @@ -1,271 +1,309 @@ VERSION 0.8 +# Note, this is only used on the main build path, caching test artifacts is less useful +# As those only run if there are changes anyway +CACHE_BUILD_BIN: + FUNCTION + ARG command + ARG build_artifacts + ARG prefix + DO ../../build-system/s3-cache-scripts/+WITH_CACHE \ + --prefix="barretenberg-$prefix" \ + --command="$command && rm -rf build/{deps,lib,src}" \ + --rebuild_patterns=".rebuild_patterns" \ + --build_artifacts="build/bin" + wasmtime: - FROM ubuntu:noble - RUN apt update && apt install -y curl xz-utils - RUN curl https://wasmtime.dev/install.sh -sSf | bash - SAVE ARTIFACT /root/.wasmtime/bin/wasmtime + FROM ubuntu:noble + RUN apt update && apt install -y curl xz-utils + RUN curl https://wasmtime.dev/install.sh -sSf | bash + SAVE ARTIFACT /root/.wasmtime/bin/wasmtime source: - FROM ../../build-images+from-registry - WORKDIR /usr/src/barretenberg - # cpp source - COPY --dir src/barretenberg src/CMakeLists.txt src - # cmake source - COPY --dir cmake CMakeLists.txt CMakePresets.json . - # for debugging rebuilds - RUN echo CONTENT HASH $(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}') | tee .content-hash + FROM ../../build-images+from-registry + WORKDIR /usr/src/barretenberg/cpp + # cpp source + COPY --dir src/barretenberg src/CMakeLists.txt src + # cmake source + COPY --dir cmake CMakeLists.txt CMakePresets.json .rebuild_patterns . preset-darwin-arm64: - FROM +source - LET OSX_SDK="MacOSX14.0.sdk" - LET OSX_SDK_URL="https://github.com/joseluisq/macosx-sdks/releases/download/14.0/${OSX_SDK}.tar.xz" - RUN curl -sSL "$OSX_SDK_URL" | tar -xJ -C /opt/osxcross/SDK && rm -rf /opt/osxcross/SDK/$OSX_SDK/System - RUN cmake --preset darwin-arm64 -Bbuild && cmake --build build --target bb && rm -rf build/{deps,lib,src} - SAVE ARTIFACT build/bin AS LOCAL build-darwin-arm64/bin + FROM +source + LET OSX_SDK="MacOSX14.0.sdk" + LET OSX_SDK_URL="https://github.com/joseluisq/macosx-sdks/releases/download/14.0/${OSX_SDK}.tar.xz" + RUN curl -sSL "$OSX_SDK_URL" | tar -xJ -C /opt/osxcross/SDK && rm -rf /opt/osxcross/SDK/$OSX_SDK/System + DO +CACHE_BUILD_BIN --prefix=preset-darwin-arm64 \ + --command="cmake --preset darwin-arm64 -Bbuild && cmake --build build --target bb" + SAVE ARTIFACT build/bin AS LOCAL build-darwin-arm64/bin preset-release: - FROM +source - RUN cmake --preset clang16 -Bbuild && cmake --build build --target bb && rm -rf build/{deps,lib,src} - SAVE ARTIFACT build/bin + FROM +source + DO +CACHE_BUILD_BIN --prefix=preset-release \ + --command="cmake --preset clang16 -Bbuild && cmake --build build --target bb" + SAVE ARTIFACT build/bin + +# This is hacked onto a dependency on the bottom of our build for lack of a better place to test the .rebuild_patterns flow +test-cache-write: + FROM ../../build-images+from-registry + WORKDIR /usr/src/barretenberg/cpp + DO +CACHE_BUILD_BIN --prefix=test-cache \ + --command="mkdir -p build/bin && echo hi > build/bin/cache" + SAVE ARTIFACT build/bin + +# As above +test-cache-read: + FROM ../../build-images+from-registry + WORKDIR /usr/src/barretenberg/cpp + # fails if cache does not exist + DO +CACHE_BUILD_BIN --prefix=test-cache \ + --command="exit 1" + SAVE ARTIFACT build/bin preset-release-world-state: FROM +source - RUN cmake --preset clang16-pic -Bbuild && cmake --build build --target world_state_napi && cp ./build/lib/world_state_napi.node ./build/bin && rm -rf build/{deps,lib,src} + DO +CACHE_BUILD_BIN --prefix=preset-release-world-state \ + --command="cmake --preset clang16-pic -Bbuild && cmake --build build --target world_state_napi && mv ./build/lib/world_state_napi.node ./build/bin" SAVE ARTIFACT build/bin preset-release-assert: - FROM +source - RUN cmake --preset clang16-assert -Bbuild && cmake --build build --target bb crypto_merkle_tree_tests && rm -rf build/{deps,lib,src} - SAVE ARTIFACT build/bin + FROM +source + DO +CACHE_BUILD_BIN --prefix=preset-release-assert \ + --command="cmake --preset clang16-assert -Bbuild && cmake --build build --target bb crypto_merkle_tree_tests" + SAVE ARTIFACT build/bin preset-debug: - FROM +source - RUN cmake --preset clang16-dbg -Bbuild && cmake --build build --target bb && rm -rf build/{deps,lib,src} - SAVE ARTIFACT build/bin + FROM +source + DO +CACHE_BUILD_BIN --prefix=preset-debug \ + --command="cmake --preset clang16-dbg -Bbuild && cmake --build build --target bb" + SAVE ARTIFACT build/bin preset-msan-check: - FROM aztecprotocol/bb-ubuntu-lunar - RUN git clone --depth 1 -b release/16.x https://github.com/llvm/llvm-project - RUN cd llvm-project - RUN mkdir llvm-project/build - - ENV CXX=clang++-16 - ENV CC=clang-16 - RUN cmake -G Ninja -S llvm-project/runtimes -B llvm-project/build \ - -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \ - -DLLVM_USE_SANITIZER=MemoryWithOrigins \ - -DCMAKE_INSTALL_PREFIX=/opt - - RUN ninja -C llvm-project/build cxx cxxabi - RUN ninja -C llvm-project/build install-cxx install-cxxabi - - ENV MSAN_CFLAGS="-std=c++20 -fsanitize=memory -fsanitize-memory-track-origins -nostdinc++ -I/opt/include -I/opt/include/c++/v1" - ENV MSAN_LFLAGS="-fsanitize=memory -fsanitize-memory-track-origins -stdlib=libc++ -L/opt/lib -lc++abi -Wl,-rpath,/opt/lib" - - WORKDIR /usr/src/barretenberg - # cpp source - COPY --dir src/barretenberg src/CMakeLists.txt src - # cmake source - COPY --dir cmake CMakeLists.txt CMakePresets.json . - - # We never fail this as test-listing targets can timeout. Just pragmatically go on to see if the binary exists. - RUN cmake --preset msan -Bbuild && cmake --build build --target client_ivc_tests || true - # install SRS needed for proving - COPY --dir ./srs_db/+build/. srs_db - RUN echo "Warning: If ./bin/client_ivc_tests is not found, there may be build failures above." - RUN cd build && ./bin/client_ivc_tests --gtest_also_run_disabled_tests + FROM aztecprotocol/bb-ubuntu-lunar + RUN git clone --depth 1 -b release/16.x https://github.com/llvm/llvm-project + RUN cd llvm-project + RUN mkdir llvm-project/build + + ENV CXX=clang++-16 + ENV CC=clang-16 + RUN cmake -G Ninja -S llvm-project/runtimes -B llvm-project/build \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \ + -DLLVM_USE_SANITIZER=MemoryWithOrigins \ + -DCMAKE_INSTALL_PREFIX=/opt + + RUN ninja -C llvm-project/build cxx cxxabi + RUN ninja -C llvm-project/build install-cxx install-cxxabi + + ENV MSAN_CFLAGS="-std=c++20 -fsanitize=memory -fsanitize-memory-track-origins -nostdinc++ -I/opt/include -I/opt/include/c++/v1" + ENV MSAN_LFLAGS="-fsanitize=memory -fsanitize-memory-track-origins -stdlib=libc++ -L/opt/lib -lc++abi -Wl,-rpath,/opt/lib" + + WORKDIR /usr/src/barretenberg + # cpp source + COPY --dir src/barretenberg src/CMakeLists.txt src + # cmake source + COPY --dir cmake CMakeLists.txt CMakePresets.json . + + # We never fail this as test-listing targets can timeout. Just pragmatically go on to see if the binary exists. + RUN cmake --preset msan -Bbuild && cmake --build build --target client_ivc_tests || true + # install SRS needed for proving + COPY --dir ./srs_db/+build/. srs_db + RUN echo "Warning: If ./bin/client_ivc_tests is not found, there may be build failures above." + RUN cd build && ./bin/client_ivc_tests --gtest_also_run_disabled_tests preset-wasm: - ARG TARGETARCH - FROM +source - IF [ $TARGETARCH = arm64 ] - # Just use threads for now - FROM +preset-wasm-threads - SAVE ARTIFACT build/bin - ELSE - RUN cmake --preset wasm -Bbuild && cmake --build build --target barretenberg.wasm && rm -rf build/{deps,lib,src} - RUN /opt/wasi-sdk/bin/llvm-strip ./build/bin/barretenberg.wasm - SAVE ARTIFACT build/bin - SAVE IMAGE --cache-hint - END + ARG TARGETARCH + FROM +source + IF [ $TARGETARCH = arm64 ] + # Just use threads for now + FROM +preset-wasm-threads + SAVE ARTIFACT build/bin + ELSE + DO +CACHE_BUILD_BIN --prefix=preset-wasm \ + --command="cmake --preset wasm -Bbuild && cmake --build build --target barretenberg.wasm && /opt/wasi-sdk/bin/llvm-strip ./build/bin/barretenberg.wasm" + SAVE ARTIFACT build/bin + SAVE IMAGE --cache-hint + END preset-sol: - FROM +source - RUN cmake --preset clang16 && cmake --build --preset clang16 --target solidity_key_gen solidity_proof_gen && rm -rf build/{deps,lib,src} - SAVE ARTIFACT build/bin + FROM +source + DO +CACHE_BUILD_BIN --prefix=preset-sol \ + --command="cmake --preset clang16 && cmake --build --preset clang16 --target solidity_key_gen solidity_proof_gen" + SAVE ARTIFACT build/bin preset-wasm-threads: - FROM +source - RUN cmake --preset wasm-threads -Bbuild && cmake --build build --target barretenberg.wasm && rm -rf build/{deps,lib,src} - RUN /opt/wasi-sdk/bin/llvm-strip ./build/bin/barretenberg.wasm - SAVE ARTIFACT build/bin + FROM +source + DO +CACHE_BUILD_BIN --prefix=preset-wasm-threads \ + --command="cmake --preset wasm-threads -Bbuild && cmake --build build --target barretenberg.wasm && /opt/wasi-sdk/bin/llvm-strip ./build/bin/barretenberg.wasm" + SAVE ARTIFACT build/bin preset-gcc: - FROM +source - RUN cmake --preset gcc -Bbuild && cmake --build build && rm -rf build/{deps,lib,src} - SAVE ARTIFACT build/bin + FROM +source + RUN cmake --preset gcc -Bbuild && cmake --build build + SAVE ARTIFACT build/bin preset-fuzzing: - FROM +source - RUN cmake --preset fuzzing -Bbuild && cmake --build build && rm -rf build/{deps,lib,src} - SAVE ARTIFACT build/bin + FROM +source + RUN cmake --preset fuzzing -Bbuild && cmake --build build + SAVE ARTIFACT build/bin preset-clang-assert: - FROM +source - RUN cmake --preset clang16-assert -Bbuild && cmake --build build --target bb && rm -rf build/{deps,lib,src} - SAVE ARTIFACT build/bin + FROM +source + DO +CACHE_BUILD_BIN --prefix=preset-clang-assert \ + --command="cmake --preset clang16-assert -Bbuild && cmake --build build --target bb" + SAVE ARTIFACT build/bin # benchmarking images -# these are either fresh builds just for benching (op-count and op-count-time) -# or build the binaries we need for benchmarking preset-op-count-bench: - FROM +source - RUN cmake --preset op-count -DCMAKE_BUILD_TYPE=RelWithAssert -Bbuild && cmake --build build --target ultra_honk_bench --target client_ivc_bench && rm -rf build/{deps,lib,src} - SAVE ARTIFACT build/bin + FROM +source + # note: we do not cache bench-only artifacts + RUN cmake --preset op-count -DCMAKE_BUILD_TYPE=RelWithAssert -Bbuild && cmake --build build --target ultra_honk_bench --target client_ivc_bench + SAVE ARTIFACT build/bin preset-op-count-time-bench: - FROM +source - RUN cmake --preset op-count-time -Bbuild && cmake --build build --target ultra_honk_bench --target client_ivc_bench && rm -rf build/{deps,lib,src} - SAVE ARTIFACT build/bin + FROM +source + # note: we do not cache bench-only artifacts + RUN cmake --preset op-count-time -Bbuild && cmake --build build --target ultra_honk_bench --target client_ivc_bench + SAVE ARTIFACT build/bin preset-release-bench: - FROM +preset-release - RUN cmake --build build --target ultra_honk_bench --target client_ivc_bench && rm -rf build/{deps,lib,src} - SAVE ARTIFACT build/bin + FROM +source + # note: we do not cache bench-only artifacts + RUN cmake --preset clang16 -Bbuild && cmake --build build --target ultra_honk_bench --target client_ivc_bench + SAVE ARTIFACT build/bin preset-wasm-bench: - FROM +preset-wasm-threads - RUN cmake --build build --target ultra_honk_bench --target client_ivc_bench && rm -rf build/{deps,lib,src} - SAVE ARTIFACT build/bin + FROM +source + # note: we do not cache bench-only artifacts + RUN cmake --preset wasm-threads -Bbuild && cmake --build build --target ultra_honk_bench --target client_ivc_bench + SAVE ARTIFACT build/bin # test images preset-release-assert-test: - FROM +preset-release-assert - # build all targets for tests - RUN cmake --build build && rm -rf build/{deps,lib,src} - SAVE ARTIFACT build/bin + FROM +source + # build all targets for tests + # note: we do not cache test-only artifacts + RUN cmake --preset clang16-assert -Bbuild && cmake --build build + SAVE ARTIFACT build/bin # Sent to the bench runner using a earthly --push +bench-binaries # then we can run earthly +bench-ultra-honk --bench_mode=cache bench-binaries: - ARG EARTHLY_GIT_HASH - ARG TARGETARCH - ARG bench_mode=build - LOCALLY - IF [ $bench_mode = cache ] - FROM aztecprotocol/bb-bench-binaries:$TARGETARCH-$EARTHLY_GIT_HASH - SAVE ARTIFACT ./* - ELSE - FROM scratch - COPY +preset-op-count-time-bench/bin/*_bench op-count-time/bin/ - COPY +preset-op-count-bench/bin/*_bench op-count/bin/ - COPY +preset-release-bench/bin/*_bench release/bin/ - COPY +preset-wasm-bench/bin/*_bench wasm/bin/ - SAVE ARTIFACT ./* - SAVE IMAGE --push aztecprotocol/bb-bench-binaries:$TARGETARCH-$EARTHLY_GIT_HASH - END + ARG EARTHLY_GIT_HASH + ARG TARGETARCH + ARG bench_mode=build + LOCALLY + IF [ $bench_mode = cache ] + FROM aztecprotocol/bb-bench-binaries:$TARGETARCH-$EARTHLY_GIT_HASH + SAVE ARTIFACT ./* + ELSE + FROM scratch + COPY +preset-op-count-time-bench/bin/*_bench op-count-time/bin/ + COPY +preset-op-count-bench/bin/*_bench op-count/bin/ + COPY +preset-release-bench/bin/*_bench release/bin/ + COPY +preset-wasm-bench/bin/*_bench wasm/bin/ + SAVE ARTIFACT ./* + SAVE IMAGE --push aztecprotocol/bb-bench-binaries:$TARGETARCH-$EARTHLY_GIT_HASH + END # Runs on the bench image, sent from the builder runner bench-ultra-honk: - BUILD +wasmtime # prefetch - FROM ubuntu:noble - COPY --dir +bench-binaries/* . - # install SRS needed for proving - COPY --dir ./srs_db/+build/. srs_db - RUN cd release && ./bin/ultra_honk_bench --benchmark_out=../ultra_honk_release.json --benchmark_filter="construct_proof_ultrahonk_power_of_2/20$" - COPY +wasmtime/wasmtime /usr/bin/wasmtime - RUN cd wasm && wasmtime run --env HARDWARE_CONCURRENCY=16 -Wthreads=y -Sthreads=y --dir=".." ./bin/ultra_honk_bench --benchmark_out=../ultra_honk_wasm.json --benchmark_filter="construct_proof_ultrahonk_power_of_2/20$" - SAVE ARTIFACT ultra_honk*.json + BUILD +wasmtime # prefetch + FROM ubuntu:noble + COPY --dir +bench-binaries/* . + # install SRS needed for proving + COPY --dir ./srs_db/+build/. srs_db + RUN cd release && ./bin/ultra_honk_bench --benchmark_out=../ultra_honk_release.json --benchmark_filter="construct_proof_ultrahonk_power_of_2/20$" + COPY +wasmtime/wasmtime /usr/bin/wasmtime + RUN cd wasm && wasmtime run --env HARDWARE_CONCURRENCY=16 -Wthreads=y -Sthreads=y --dir=".." ./bin/ultra_honk_bench --benchmark_out=../ultra_honk_wasm.json --benchmark_filter="construct_proof_ultrahonk_power_of_2/20$" + SAVE ARTIFACT ultra_honk*.json bench-client-ivc: - BUILD +wasmtime # prefetch - FROM ubuntu:noble - COPY --dir +bench-binaries/* . - # install SRS needed for proving - COPY --dir ./srs_db/+build/. srs_db - RUN cd release && ./bin/client_ivc_bench --benchmark_out=../client_ivc_release.json --benchmark_filter="ClientIVCBench/Full/6$" - RUN cd op-count && ./bin/client_ivc_bench --benchmark_out=../client_ivc_op_count.json --benchmark_filter="ClientIVCBench/Full/6$" - RUN cd op-count-time && ./bin/client_ivc_bench --benchmark_out=../client_ivc_op_count_time.json --benchmark_filter="ClientIVCBench/Full/6$" - COPY +wasmtime/wasmtime /usr/bin/wasmtime - RUN cd wasm && wasmtime run --env HARDWARE_CONCURRENCY=16 -Wthreads=y -Sthreads=y --dir=".." ./bin/client_ivc_bench --benchmark_out=../client_ivc_wasm.json --benchmark_filter="ClientIVCBench/Full/6$" - SAVE ARTIFACT client_ivc*.json + BUILD +wasmtime # prefetch + FROM ubuntu:noble + COPY --dir +bench-binaries/* . + # install SRS needed for proving + COPY --dir ./srs_db/+build/. srs_db + RUN cd release && ./bin/client_ivc_bench --benchmark_out=../client_ivc_release.json --benchmark_filter="ClientIVCBench/Full/6$" + RUN cd op-count && ./bin/client_ivc_bench --benchmark_out=../client_ivc_op_count.json --benchmark_filter="ClientIVCBench/Full/6$" + RUN cd op-count-time && ./bin/client_ivc_bench --benchmark_out=../client_ivc_op_count_time.json --benchmark_filter="ClientIVCBench/Full/6$" + COPY +wasmtime/wasmtime /usr/bin/wasmtime + RUN cd wasm && wasmtime run --env HARDWARE_CONCURRENCY=16 -Wthreads=y -Sthreads=y --dir=".." ./bin/client_ivc_bench --benchmark_out=../client_ivc_wasm.json --benchmark_filter="ClientIVCBench/Full/6$" + SAVE ARTIFACT client_ivc*.json bench: - FROM aztecprotocol/bb-ubuntu-lunar - # for sequential running - WAIT - COPY +bench-ultra-honk/* . - END - WAIT - COPY +bench-client-ivc/* . - END - COPY ./scripts/ci/combine_benchmarks.py ./scripts/ci/combine_benchmarks.py - RUN ./scripts/ci/combine_benchmarks.py \ - native client_ivc_release.json \ - native ultra_honk_release.json \ - wasm client_ivc_wasm.json \ - wasm ultra_honk_wasm.json \ - "" client_ivc_op_count.json \ - "" client_ivc_op_count_time.json \ - > bench.json - SAVE ARTIFACT bench.json + FROM aztecprotocol/bb-ubuntu-lunar + # for sequential running + WAIT + COPY +bench-ultra-honk/* . + END + WAIT + COPY +bench-client-ivc/* . + END + COPY ./scripts/ci/combine_benchmarks.py ./scripts/ci/combine_benchmarks.py + RUN ./scripts/ci/combine_benchmarks.py \ + native client_ivc_release.json \ + native ultra_honk_release.json \ + wasm client_ivc_wasm.json \ + wasm ultra_honk_wasm.json \ + "" client_ivc_op_count.json \ + "" client_ivc_op_count_time.json \ + > bench.json + SAVE ARTIFACT bench.json # Sent to the bench runner using a earthly --push +test-binaries # then we can run earthly +test --test_mode=cache test-binaries: - ARG EARTHLY_GIT_HASH - ARG TARGETARCH - ARG test_mode=build - LOCALLY - IF [ $test_mode = cache ] - FROM aztecprotocol/bb-test-binaries:$TARGETARCH-$EARTHLY_GIT_HASH - SAVE ARTIFACT build - ELSE - FROM scratch - COPY +preset-release-assert-test/bin/*_tests build/bin/ - SAVE ARTIFACT build - SAVE IMAGE --push aztecprotocol/bb-test-binaries:$TARGETARCH-$EARTHLY_GIT_HASH - END + ARG EARTHLY_GIT_HASH + ARG TARGETARCH + ARG test_mode=build + LOCALLY + IF [ $test_mode = cache ] + FROM aztecprotocol/bb-test-binaries:$TARGETARCH-$EARTHLY_GIT_HASH + SAVE ARTIFACT build + ELSE + FROM scratch + COPY +preset-release-assert-test/bin/*_tests build/bin/ + SAVE ARTIFACT build + SAVE IMAGE --push aztecprotocol/bb-test-binaries:$TARGETARCH-$EARTHLY_GIT_HASH + END test-clang-format: - FROM +source - COPY .clang-format . - COPY format.sh . - RUN ./format.sh check + FROM +source + COPY .clang-format . + COPY format.sh . + RUN ./format.sh check test: - ARG hardware_concurrency="" - # prefetch - BUILD +preset-release-assert-test - BUILD +test-clang-format - BUILD ./srs_db/+build # prefetch - FROM +preset-release-assert-test - COPY --dir ./srs_db/+build/. srs_db - # limit hardware concurrency, if provided - IF [ "$HARDWARE_CONCURRENCY" != "" ] - ENV HARDWARE_CONCURRENCY=$hardware_concurrency - END - RUN cd build && GTEST_COLOR=1 ctest -j$(nproc) --output-on-failure + ARG hardware_concurrency="" + # prefetch + BUILD +preset-release-assert-test + BUILD +test-clang-format + BUILD ./srs_db/+build # prefetch + FROM +preset-release-assert-test + COPY --dir ./srs_db/+build/. srs_db + # limit hardware concurrency, if provided + IF [ "$HARDWARE_CONCURRENCY" != "" ] + ENV HARDWARE_CONCURRENCY=$hardware_concurrency + END + RUN cd build && GTEST_COLOR=1 ctest -j$(nproc) --output-on-failure vm-full-test: - ARG hardware_concurrency="" - # prefetch - BUILD +test-binaries - BUILD +preset-release-assert-test - BUILD ./srs_db/+build # prefetch - FROM +source - COPY --dir +test-binaries/build build - FROM +preset-release-assert-test - COPY --dir ./srs_db/+build/. srs_db - # limit hardware concurrency, if provided - IF [ "$HARDWARE_CONCURRENCY" != "" ] - ENV HARDWARE_CONCURRENCY=$hardware_concurrency - END - RUN cd build && GTEST_COLOR=1 AVM_ENABLE_FULL_PROVING=1 ctest -j4 --test-dir src/barretenberg/vm --output-on-failure + ARG hardware_concurrency="" + # prefetch + BUILD +test-binaries + BUILD +preset-release-assert-test + BUILD ./srs_db/+build # prefetch + FROM +source + COPY --dir +test-binaries/build build + FROM +preset-release-assert-test + COPY --dir ./srs_db/+build/. srs_db + # limit hardware concurrency, if provided + IF [ "$HARDWARE_CONCURRENCY" != "" ] + ENV HARDWARE_CONCURRENCY=$hardware_concurrency + END + RUN cd build && GTEST_COLOR=1 AVM_ENABLE_FULL_PROVING=1 ctest -j4 --test-dir src/barretenberg/vm --output-on-failure build: - BUILD +preset-wasm - BUILD +preset-wasm-threads - BUILD +preset-release - BUILD +preset-release-world-state + BUILD +preset-wasm + BUILD +preset-wasm-threads + BUILD +preset-release + BUILD +preset-release-world-state diff --git a/barretenberg/cpp/scripts/ci/upload_benchmarks_to_s3.sh b/barretenberg/cpp/scripts/ci/upload_benchmarks_to_s3.sh deleted file mode 100755 index 6a924ba33df..00000000000 --- a/barretenberg/cpp/scripts/ci/upload_benchmarks_to_s3.sh +++ /dev/null @@ -1,21 +0,0 @@ -# Uploads to S3 a recent barretenberg benchmark run. -#!/usr/bin/env bash -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -retry ecr_login -extract_repo barretenberg-bench /usr/src extracted-repo - -BUCKET_NAME="aztec-ci-artifacts" -COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" - -if [ "${BRANCH:-}" = "master" ]; then - TARGET_FOLDER="barretenberg-bench-v1/master/$COMMIT_HASH/" -elif [ -n "${PULL_REQUEST:-}" ]; then - TARGET_FOLDER="barretenberg-bench-v1/pulls/${PULL_REQUEST##*/}" -else - echo Skipping upload since no target folder was defined - exit -fi -echo "Uploading to s3://$BUCKET_NAME/$TARGET_FOLDER" -aws s3 cp extracted-repo/src/barretenberg/cpp/build/ultra_honk_rounds_bench.json "s3://$BUCKET_NAME/$TARGET_FOLDER/ultra_honk_rounds_bench.json" diff --git a/barretenberg/ts/.rebuild_patterns b/barretenberg/ts/.rebuild_patterns new file mode 100644 index 00000000000..562e0c9acb2 --- /dev/null +++ b/barretenberg/ts/.rebuild_patterns @@ -0,0 +1 @@ +^barretenberg/ts/.*$ \ No newline at end of file diff --git a/barretenberg/ts/Earthfile b/barretenberg/ts/Earthfile index b18e91f9039..fa0419fed85 100644 --- a/barretenberg/ts/Earthfile +++ b/barretenberg/ts/Earthfile @@ -1,49 +1,76 @@ VERSION 0.8 -FROM ../../build-images+from-registry -WORKDIR /usr/src/barretenberg/ts-build +CACHE: + FUNCTION + ARG command + ARG build_artifacts + ARG prefix + DO ../../build-system/s3-cache-scripts/+WITH_CACHE \ + --prefix="bb.js-$prefix" \ + --command="$command" \ + --rebuild_patterns="../cpp/.rebuild_patterns .rebuild_patterns" \ + --build_artifacts="$build_artifacts" -# minimum files to download yarn packages -# keep timestamps for incremental builds -COPY --dir .yarn package.json yarn.lock .yarnrc.yml . -RUN echo CONTENT HASH $(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}') | tee .content-hash -RUN yarn --immutable +deps: + FROM ../../build-images+from-registry + WORKDIR /usr/src/barretenberg/ts -# other source files -COPY --dir src *.json *.js *.cjs . + # minimum files to download yarn packages + # keep timestamps for incremental builds + COPY --dir .yarn package.json yarn.lock .yarnrc.yml . + RUN yarn --immutable -# copy over wasm builds from cpp folder -COPY ../cpp/+preset-wasm-threads/bin/barretenberg.wasm src/barretenberg_wasm/barretenberg-threads.wasm -COPY ../cpp/+preset-wasm/bin/barretenberg.wasm src/barretenberg_wasm/barretenberg.wasm -COPY ../cpp/+preset-wasm-threads/bin/barretenberg.wasm dest/node/barretenberg_wasm/barretenberg-threads.wasm -COPY ../cpp/+preset-wasm-threads/bin/barretenberg.wasm dest/node-cjs/barretenberg_wasm/barretenberg-threads.wasm + # other source files + COPY --dir src *.json *.js *.cjs . + + # copy over wasm builds from cpp folder + COPY ../cpp/+preset-wasm-threads/bin/barretenberg.wasm src/barretenberg_wasm/barretenberg-threads.wasm + COPY ../cpp/+preset-wasm/bin/barretenberg.wasm src/barretenberg_wasm/barretenberg.wasm + COPY ../cpp/+preset-wasm-threads/bin/barretenberg.wasm dest/node/barretenberg_wasm/barretenberg-threads.wasm + COPY ../cpp/+preset-wasm-threads/bin/barretenberg.wasm dest/node-cjs/barretenberg_wasm/barretenberg-threads.wasm esm: - RUN yarn build:esm - SAVE ARTIFACT /usr/src/barretenberg/ts-build build + FROM +deps + DO +CACHE \ + --prefix="browser" \ + --command="yarn build:esm" \ + --build_artifacts="." + SAVE ARTIFACT /usr/src/barretenberg/ts build cjs: + FROM +deps COPY scripts/cjs_postprocess.sh scripts/ - RUN yarn build:cjs - SAVE ARTIFACT /usr/src/barretenberg/ts-build build + DO +CACHE \ + --prefix="cjs" \ + --command="yarn build:cjs" \ + --build_artifacts="." + SAVE ARTIFACT /usr/src/barretenberg/ts build browser: + FROM +deps RUN yarn build:browser - SAVE ARTIFACT /usr/src/barretenberg/ts-build build + DO +CACHE \ + --prefix="browser" \ + --command="yarn build:browser" \ + --build_artifacts="." + SAVE ARTIFACT /usr/src/barretenberg/ts build test-prettier-format: + FROM +deps RUN yarn formatting build: + FROM +deps # collect all our build types - COPY +esm/build /usr/src/barretenberg/ts-build - COPY +cjs/build /usr/src/barretenberg/ts-build - COPY +browser/build /usr/src/barretenberg/ts-build + COPY +esm/build /usr/src/barretenberg/ts + COPY +cjs/build /usr/src/barretenberg/ts + COPY +browser/build /usr/src/barretenberg/ts # We want to create a pure package, as would be published to npm, for consuming projects. - RUN yarn pack && tar zxf package.tgz && rm package.tgz && mv package ../ts + RUN yarn pack && tar zxf package.tgz && rm package.tgz SAVE ARTIFACT /usr/src/barretenberg/ts build test: + FROM +deps BUILD +test-prettier-format # We frequently get tests timing out in GA, increase the timeout here RUN yarn test --testTimeout=300000 diff --git a/build-system/s3-cache-scripts/Earthfile b/build-system/s3-cache-scripts/Earthfile new file mode 100644 index 00000000000..71abcd1dd9e --- /dev/null +++ b/build-system/s3-cache-scripts/Earthfile @@ -0,0 +1,56 @@ +VERSION 0.8 +# Sets up Earthly boilerplate for using our s3 cache. + +setup-cache-helper: + FROM scratch + COPY *.sh . + SAVE ARTIFACT ./* + +SETUP_CACHE: + FUNCTION + COPY +setup-cache-helper/* /usr/src/build-system/s3-cache-scripts/ + +# needed intermediary to call out to LOCALLY and go back to normal context +compute-content-hash: + ARG local_directory_equivalent + ARG rebuild_patterns + LOCALLY + RUN [ ! -z "$rebuild_patterns" ] || (echo 'Did not pass rebuild patterns to cache command!' && exit 1) + LET content_hash_script="$(pwd)/compute-content-hash.sh" + LET content_hash=$(cd "../../$local_directory_equivalent" && AZTEC_CACHE_REBUILD_PATTERNS="$rebuild_patterns" "$content_hash_script") + FROM ubuntu:noble + RUN echo "$content_hash" > .content-hash + SAVE ARTIFACT .content-hash + +WRITE_CONTENT_HASH: + FUNCTION + ARG rebuild_patterns + LET local_dir=$(pwd | sed 's|^/usr/src/||') + COPY (+compute-content-hash/.content-hash --local_directory_equivalent="$local_dir" --rebuild_patterns="$rebuild_patterns") . + +# 1) starts our cache server if not up +# 2) copies necessary context +# 3) first tries our rebuild pattern and context against our cache tool to see if we can use our cache +# 4) if not, we build with $command +# 5) then we try to upload with our cache tool for next time +# note, if the cache tool is unreachable at $AZTEC_CACHE_TOOL_IP we silently fail cache-download-pattern.sh +# and silently pass cache-upload-pattern.sh meaning the command always runs +WITH_CACHE: + FUNCTION + ARG command + ARG prefix + ARG rebuild_patterns + ARG build_artifacts + # print a clean version as the one below will be obfuscated + RUN echo "--> RUN WITH CACHE $command (start)" + DO +SETUP_CACHE + DO +WRITE_CONTENT_HASH --rebuild_patterns="$rebuild_patterns" + # configure AWS access and S3 cache params (note these can be blank if not using S3 build cache) + RUN --secret AWS_ACCESS_KEY_ID \ + --secret AWS_SECRET_ACCESS_KEY \ + --secret S3_BUILD_CACHE_UPLOAD \ + --secret S3_BUILD_CACHE_DOWNLOAD \ + --secret S3_BUILD_CACHE_MINIO_URL \ + # Note, this should be edited in tandem with the ARGs above, but for clean logs we wrap this in a script + /usr/src/build-system/s3-cache-scripts/earthly-s3-cache.sh + RUN echo "--> RUN WITH CACHE $command (end)" diff --git a/build-system/s3-cache-scripts/README.md b/build-system/s3-cache-scripts/README.md new file mode 100644 index 00000000000..68e84ff3022 --- /dev/null +++ b/build-system/s3-cache-scripts/README.md @@ -0,0 +1,15 @@ +Scripts that implement a simple scheme to upload and download from S3 for use with caching. Supports .rebuild_patterns files inside the monorepo for detecting changes. +Assumes a git committed state. If that is not the case, you should not use the cache (as the Earthly usage does not). + +Rationale: +- We need a unified cache tool that can support distributed caching. This is needed to replace our old docker image-based caching. It is easier to share S3 access and overall easier to use S3 tarballs rather than docker images. + +We put a list of .rebuild_patterns files (that have regex matching files in git) into the AZTEC_CACHE_REBUILD_PATTERNS env variable and a prefix for our artifact. +It does not need to be a unique name per build as our content hash will be appended. +- AZTEC_CACHE_REBUILD_PATTERNS=barretenberg/cpp/.rebuild_patterns ./cache-download-pattern.sh barretenberg + +If we want to upload to the local or distributed cache (again depending if we started the server with S3_READ/S3_WRITE): +- AZTEC_CACHE_REBUILD_PATTERNS=barretenberg/cpp/.rebuild_patterns ./cache-download-pattern.sh barretenberg + +Installation: +- This is just some shell scripts, but you do need AWS credentials set up and aws commandline installed otherwise the scripts **do nothing**. \ No newline at end of file diff --git a/build-system/s3-cache-scripts/cache-download.sh b/build-system/s3-cache-scripts/cache-download.sh new file mode 100755 index 00000000000..3fbbf43503e --- /dev/null +++ b/build-system/s3-cache-scripts/cache-download.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -eu + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Get the tar.gz file name from the argument +TAR_FILE="$1" + +function on_exit() { + # Cleanup the temporary tar.gz file + rm -f "$TAR_FILE" +} +# Run on any exit +trap on_exit EXIT + +# Attempt to download the cache file +aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp "s3://aztec-ci-artifacts/build-cache/$TAR_FILE" "$TAR_FILE" --quiet + +# Extract the cache file +tar -xzf "$TAR_FILE" + +echo "Cache download and extraction complete." diff --git a/build-system/s3-cache-scripts/cache-upload.sh b/build-system/s3-cache-scripts/cache-upload.sh new file mode 100755 index 00000000000..d574d246e90 --- /dev/null +++ b/build-system/s3-cache-scripts/cache-upload.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -eu + +if [ "$#" -lt 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Name, intended to have .tar.gz ending +NAME="$1" + +shift 1 + +TAR_DIR=$(mktemp -d) +TAR_FILE="$TAR_DIR/${NAME}" + +function on_exit() { + # Cleanup the temporary folder + rm -rf "$TAR_DIR" +} +trap on_exit EXIT + +# Create the tar.gz file +# Rest of args are our binary paths +tar -czf "$TAR_FILE" $@ + +aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp "$TAR_FILE" "s3://aztec-ci-artifacts/build-cache/$NAME" \ No newline at end of file diff --git a/build-system/s3-cache-scripts/compute-content-hash.sh b/build-system/s3-cache-scripts/compute-content-hash.sh new file mode 100755 index 00000000000..b3d9d3c1c43 --- /dev/null +++ b/build-system/s3-cache-scripts/compute-content-hash.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Ensure AZTEC_CACHE_REBUILD_PATTERNS is set +if [ -z "${AZTEC_CACHE_REBUILD_PATTERNS:-}" ]; then + echo "Error: AZTEC_CACHE_REBUILD_PATTERNS environment variable is not set." + exit 1 +fi + +# Read rebuild patterns as multi-line string +REBUILD_PATTERNS="$AZTEC_CACHE_REBUILD_PATTERNS" + +# Concatenate patterns with '|' and double escape backslashes for AWK +AWK_PATTERN=$(cat $REBUILD_PATTERNS | sed 's/\\/\\\\/g' | tr '\n' '|' | sed 's/|$//') + +# use git repo root because that is where our patterns are focused +cd $(git rev-parse --show-toplevel) +# Use git ls-tree and AWK to filter files matching the rebuild patterns and extract their hashes +FILE_HASHES=$(git ls-tree -r HEAD | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}') + +# Check if FILE_HASHES is empty +if [ -z "$FILE_HASHES" ]; then + echo "No files matched the rebuild patterns $REBUILD_PATTERNS." + echo "Awk pattern expanded: $AWK_PATTERN." + exit 1 +fi + +# Sort the hashes and compute the content hash +CONTENT_HASH=$(echo "$FILE_HASHES" | sort | git hash-object --stdin) +echo "$CONTENT_HASH" \ No newline at end of file diff --git a/build-system/s3-cache-scripts/earthly-s3-cache.sh b/build-system/s3-cache-scripts/earthly-s3-cache.sh new file mode 100755 index 00000000000..571eb210450 --- /dev/null +++ b/build-system/s3-cache-scripts/earthly-s3-cache.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Note, this should be edited in tandem with the Earthfile in this folder that uses it, it is coupled heavily with earthly's needs +# and uses these ARGs directly +# - $build_artifacts +# - $prefix +# - $command +# The rest of the env variables are injected as secrets (e.g. aws creds and s3 modes) +set -eu + +# definitions +FILE="$prefix-$(cat .content-hash).tar.gz" +function s3_download() { + if [ "${S3_BUILD_CACHE_DOWNLOAD:-true}" = "false" ] || [ "${AWS_ACCESS_KEY_ID}" == "" ] ; then + return 1 # require a rebuild + fi + /usr/src/build-system/s3-cache-scripts/cache-download.sh "$FILE" +} +function s3_upload() { + if [ "${S3_BUILD_CACHE_UPLOAD:-true}" = "false" ] || [ "${AWS_ACCESS_KEY_ID}" == "" ] ; then + return 0 # exit silently + fi + /usr/src/build-system/s3-cache-scripts/cache-upload.sh "$FILE" $build_artifacts +} +function minio_download() { + if [ -z "$S3_BUILD_CACHE_MINIO_URL" ] ; then + return 1 # require rebuild + fi + # minio is S3-compatible + S3_BUILD_CACHE_AWS_PARAMS="--endpoint-url $S3_BUILD_CACHE_MINIO_URL" AWS_SECRET_ACCESS_KEY=minioadmin AWS_ACCESS_KEY_ID=minioadmin \ + /usr/src/build-system/s3-cache-scripts/cache-download.sh "$FILE" +} +function minio_upload() { + if [ -z "$S3_BUILD_CACHE_MINIO_URL" ] ; then + return 0 # exit silently + fi + # minio is S3-compatible + S3_BUILD_CACHE_AWS_PARAMS="--endpoint-url $S3_BUILD_CACHE_MINIO_URL" AWS_SECRET_ACCESS_KEY=minioadmin AWS_ACCESS_KEY_ID=minioadmin \ + /usr/src/build-system/s3-cache-scripts/cache-upload.sh "$FILE" $build_artifacts +} + +# commands +if minio_download ; then + # got them from local file server cache, great + exit +fi +if s3_download ; then + # got them from S3, replicate to minio for faster cache next time (if configured, won't be in CI) + minio_upload + exit +fi + +if ! bash -c "$command" ; then + exit 1 # we have failed to build, don't continue +fi + +minio_upload || echo "Minio upload failed!" +s3_upload || echo "S3 upload failed!" diff --git a/build-system/s3-cache-scripts/test.sh b/build-system/s3-cache-scripts/test.sh new file mode 100755 index 00000000000..765de5ed00b --- /dev/null +++ b/build-system/s3-cache-scripts/test.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# test only minio caching +set -eu +function delete_test_cache() { + AWS_SECRET_ACCESS_KEY=minioadmin AWS_ACCESS_KEY_ID=minioadmin \ + aws --endpoint http://localhost:12000 \ + s3 rm s3://aztec-ci-artifacts --recursive --exclude "*" --include "build-cache/barretenberg-test-cache-*.tar.gz" 2>&1 || true +} +function minio_cache_only() { + "$(git rev-parse --show-toplevel)/scripts/earthly-local" --no-cache $@ +} + +cd $(dirname $0) + +# if we have minio already running make sure the cache is deleted +# otherwise it will be run by next earthly local command +nc -z 127.0.0.1 12000 2>/dev/null >/dev/null && delete_test_cache +# Our assertions (thanks to -e flag): +# expect file to not exist at first +if minio_cache_only ../../barretenberg/cpp/+test-cache-read 2>/dev/null ; then + echo "Cache read without write should fail!" + exit 1 +fi +minio_cache_only ../../barretenberg/cpp/+test-cache-write 2>/dev/null +[ -d ~/.minio/data/*/*/barretenberg-test-cache*.tar.gz ] # minio cache files should be written +minio_cache_only ../../barretenberg/cpp/+test-cache-read 2>/dev/null # we should be able to read now +echo "Success!" diff --git a/noir-projects/Earthfile b/noir-projects/Earthfile index 658f5987e48..6f2a5381675 100644 --- a/noir-projects/Earthfile +++ b/noir-projects/Earthfile @@ -5,8 +5,6 @@ source: # Install nargo COPY ../noir/+nargo/nargo /usr/bin/nargo - # Install bb - COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb # Install world state napi COPY ../barretenberg/cpp/+preset-release-world-state/bin/world_state_napi.node /usr/src/barretenberg/cpp/build/bin/world_state_napi.node @@ -16,9 +14,6 @@ source: COPY package.json yarn.lock mega_honk_circuits.json . COPY --dir aztec-nr noir-contracts noir-protocol-circuits mock-protocol-circuits scripts . - # for debugging rebuilds - RUN echo CONTENT HASH $(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}') | tee .content-hash - build-contracts: FROM +source diff --git a/noir/Earthfile b/noir/Earthfile index 2ad31a5ba23..f650341603b 100644 --- a/noir/Earthfile +++ b/noir/Earthfile @@ -1,10 +1,12 @@ VERSION 0.8 nargo-src: - FROM ../build-images/+from-registry - WORKDIR /usr/src - # Relevant source (TODO finer-grained 'tooling') + WORKDIR /usr/src/noir + COPY noir-repo/.github/scripts/*.sh noir-repo/.github/scripts/ + # First install wasm bindgen + RUN ./noir-repo/.github/scripts/wasm-bindgen-install.sh + # Relevant source COPY --dir \ noir-repo/acvm-repo \ noir-repo/compiler \ @@ -24,8 +26,8 @@ nargo-src: # we don't want Noir to rebuild everytime the parent repo changes # just only when it changes # the commit hash gets injected into version strings - ENV COMMIT_HASH=$(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}') - RUN echo $COMMIT_HASH > .content-hash + DO ../build-system/s3-cache-scripts/+WRITE_CONTENT_HASH --rebuild_patterns=".rebuild_patterns_native" + ENV COMMIT_HASH=$(cat .content-hash) # # borrow Nix's approach to build everything in 1970 ENV SOURCE_TIMESTAMP=1 @@ -33,13 +35,18 @@ nargo-src: nargo: FROM +nargo-src - RUN ./noir-repo/.github/scripts/wasm-bindgen-install.sh COPY ./scripts/bootstrap_native.sh ./scripts/bootstrap_native.sh + # Note: build is decoupled from barretenberg + LET rebuild_patterns=".rebuild_patterns_native" # build noir nargo & acvm, and make sure the big build and deps folders don't hit cache - RUN ./scripts/bootstrap_native.sh && rm -rf /usr/src/noir-repo/target/release/{build,deps} - SAVE ARTIFACT /usr/src/noir-repo/target/release/nargo nargo - SAVE ARTIFACT /usr/src/noir-repo/target/release/acvm acvm - SAVE IMAGE aztecprotocol/nargo + DO ../build-system/s3-cache-scripts/+WITH_CACHE \ + --prefix="noir-nargo" \ + --rebuild_patterns="$rebuild_patterns" \ + --command="./scripts/bootstrap_native.sh && rm -rf noir-repo/target/release/{build,deps}" \ + --build_artifacts="noir-repo/target/release/nargo noir-repo/target/release/acvm" + + SAVE ARTIFACT noir-repo/target/release/nargo nargo + SAVE ARTIFACT noir-repo/target/release/acvm acvm test: FROM +nargo-src @@ -48,12 +55,11 @@ test: # Some of the debugger tests are a little flaky wrt to timeouts so we allow a couple of retries. ENV NEXTEST_RETRIES=2 - RUN ./scripts/test_native.sh examples: FROM +nargo - ENV PATH="/usr/src/noir-repo/target/release:${PATH}" + ENV PATH="/usr/src/noir/noir-repo/target/release:${PATH}" COPY --dir noir-repo/examples noir-repo COPY ../barretenberg/cpp/+preset-clang-assert/bin/bb /usr/src/barretenberg/cpp/build/bin/bb @@ -71,7 +77,7 @@ examples: format: FROM +nargo - ENV PATH=$PATH:/usr/src/noir-repo/target/release + ENV PATH="/usr/src/noir/noir-repo/target/release:${PATH}" COPY ./noir-repo/test_programs ./noir-repo/test_programs COPY ./noir-repo/noir_stdlib ./noir-repo/noir_stdlib @@ -135,18 +141,22 @@ packages-deps: noir-repo packages: - FROM +packages-deps - - COPY ./scripts/bootstrap_packages.sh ./scripts/bootstrap_packages.sh - RUN ./scripts/bootstrap_packages.sh - SAVE ARTIFACT packages - SAVE IMAGE --cache-hint + FROM +packages-deps + COPY ./scripts/bootstrap_packages.sh ./scripts/bootstrap_packages.sh + ENV REBUILD_PATTERNS="../barretenberg/cpp/.rebuild_patterns .rebuild_patterns_packages" + # build noir nargo & acvm, and make sure the big build and deps folders don't hit cache + DO ../build-system/s3-cache-scripts/+WITH_CACHE \ + --prefix="noir-packages" \ + --rebuild_patterns="$REBUILD_PATTERNS" \ + --command="./scripts/bootstrap_packages.sh" \ + --build_artifacts="packages" + SAVE ARTIFACT packages packages-test-build: FROM +packages-deps - COPY +nargo/nargo /usr/src/noir/noir-repo/target/release/nargo - COPY +nargo/acvm /usr/src/noir/noir-repo/target/release/acvm + COPY +nargo/nargo noir-repo/target/release/nargo + COPY +nargo/acvm noir-repo/target/release/acvm ENV NARGO_BACKEND_PATH=/usr/src/barretenberg/ts/dest/node/main.js ENV PATH=$PATH:/usr/src/noir/noir-repo/target/release @@ -155,8 +165,6 @@ packages-test-build: RUN yarn --immutable WORKDIR /usr/src/noir/noir-repo - COPY --dir noir-repo/.github/scripts/wasm-bindgen-install.sh ./.github/scripts/wasm-bindgen-install.sh - RUN ./.github/scripts/wasm-bindgen-install.sh ENV SOURCE_DATE_EPOCH=$(date +%s) ENV GIT_DIRTY=false @@ -188,9 +196,11 @@ packages-test-node: packages-test-browser: FROM ../build-images/+base-slim-node - COPY --dir +packages-test-build/usr/src /usr WORKDIR /usr/src/noir/noir-repo + # cache playwright install + COPY noir-repo/.github/scripts/*.sh .github/scripts/ RUN ./.github/scripts/playwright-install.sh + COPY --dir +packages-test-build/usr/src /usr RUN yarn workspaces foreach \ --verbose \ --include integration-tests \ @@ -207,14 +217,14 @@ build-acir-tests: RUN rm -rf .earthly-staging && mkdir -p .earthly-staging RUN cp --parents $(git ls-files "noir-repo/test_programs/*.toml" "noir-repo/test_programs/*.nr" "noir-repo/test_programs/rebuild.sh") .earthly-staging FROM ../build-images/+from-registry - COPY +nargo/ /usr/src/noir-repo/target/release - ENV PATH="/usr/src/noir-repo/target/release:${PATH}" - WORKDIR /usr/src/noir-repo/test_programs - COPY .earthly-staging/noir-repo/test_programs /usr/src/noir-repo/test_programs/ - RUN /usr/src/noir-repo/target/release/nargo --version + COPY +nargo/ /usr/src/noir/noir-repo/target/release + ENV PATH="/usr/src/noir/noir-repo/target/release:${PATH}" + WORKDIR /usr/src/noir/noir-repo/test_programs + COPY .earthly-staging/noir-repo/test_programs /usr/src/noir/noir-repo/test_programs/ + RUN /usr/src/noir/noir-repo/target/release/nargo --version # TODO(#6225): We have trouble with concurrency and pass 'true' to build in serial, see #6225 for details RUN ./rebuild.sh - SAVE ARTIFACT /usr/src/noir-repo/test_programs/acir_artifacts/* + SAVE ARTIFACT /usr/src/noir/noir-repo/test_programs/acir_artifacts/* barretenberg-acir-benches-bb: FROM ubuntu:noble diff --git a/scripts/ci/maybe_exit_spot.sh b/scripts/ci/maybe_exit_spot.sh index 0077a00c4e3..944f5b027a0 100644 --- a/scripts/ci/maybe_exit_spot.sh +++ b/scripts/ci/maybe_exit_spot.sh @@ -1,7 +1,7 @@ #!/bin/bash set -eux -MAX_WAIT_TIME=300 # Maximum wait time in seconds +MAX_WAIT_TIME=600 # Maximum wait time in seconds, 10 minutes WAIT_INTERVAL=10 # Interval between checks in seconds elapsed_time=0 diff --git a/scripts/earthly-ci b/scripts/earthly-ci index 91beeaa1a4a..d78305e67a3 100755 --- a/scripts/earthly-ci +++ b/scripts/earthly-ci @@ -33,7 +33,31 @@ function wipe_non_cache_docker_state { } EARTHLY_RUN_STATS_JSON="earthly-run-stats.json" -EARTHLY_ARGS="--logstream-debug-manifest-file $EARTHLY_RUN_STATS_JSON --secret AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} --secret AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${AZTEC_BOT_GITHUB_TOKEN:-}" +# Run earthly with our necesary secrets initialized +# AWS credentials can be blank, however we will not use the S3 cache at all if so. +if [ "$(git rev-parse --abbrev-ref HEAD)" = master ] || [ "$(git rev-parse --abbrev-ref HEAD)" = *build-cache* ] ; then + # We upload to S3 on master. Meant for CI on master. + S3_BUILD_CACHE_UPLOAD=true +else + # Don't upload to S3 on pull request branches (for now, at least) + S3_BUILD_CACHE_UPLOAD=false +fi + +EARTHLY_ARGS="" + +function run_earthly() { + # We abuse secrets with regular config that we don't want to alter the cache (see https://docs.earthly.dev/docs/guides/secrets) + # we do not run with minio in CI + earthly --logstream-debug-manifest-file $EARTHLY_RUN_STATS_JSON \ + --secret AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} \ + --secret AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} \ + --secret S3_BUILD_CACHE_MINIO_URL="" \ + --secret S3_BUILD_CACHE_UPLOAD="$S3_BUILD_CACHE_UPLOAD" \ + --secret S3_BUILD_CACHE_DOWNLOAD="true" \ + --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${AZTEC_BOT_GITHUB_TOKEN:-} \ + $EARTHLY_ARGS \ + $@ +} function print_earthly_command_timings() { jq --version >/dev/null || return @@ -57,7 +81,7 @@ function print_earthly_command_timings() { # Handle earthly commands and retries while [ $ATTEMPT_COUNT -lt $MAX_ATTEMPTS ]; do - if earthly $EARTHLY_ARGS $@ 2>&1 | tee $OUTPUT_FILE >&2 ; then + if run_earthly $@ 2>&1 | tee $OUTPUT_FILE >&2 ; then print_earthly_command_timings || true exit 0 # Success, exit the script else diff --git a/scripts/earthly-local b/scripts/earthly-local new file mode 100755 index 00000000000..8813ac1ae04 --- /dev/null +++ b/scripts/earthly-local @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Run earthly with our necesary secrets initialized +# AWS credentials can be blank HOWEVER this will disable S3 caching. +export EARTHLY_ALLOW_PRIVILEGED=true + +set -eu + +function start_minio() { + if nc -z 127.0.0.1 12000 2>/dev/null >/dev/null ; then + # Already started + return + fi + mkdir -p ~/.minio/data + docker run -d -p 12000:9000 -p 12001:12001 -v ~/.minio/data:/data \ + quay.io/minio/minio server /data --console-address ":12001" + # make our cache bucket + aws --endpoint-url http://localhost:12000 s3 mb s3://aztec-ci-artifacts 2>/dev/null || true +} + +S3_BUILD_CACHE_UPLOAD=${S3_BUILD_CACHE_UPLOAD:-false} +S3_BUILD_CACHE_MINIO_URL="http://$(hostname -I | awk '{print $1}'):12000" + +# local file server for a quicker cache layer +start_minio + +if ! git diff-index --quiet HEAD --; then + echo "Warning: You have unstaged changes. Disabling S3 caching and local minio caching for earthly to not accidentally pollute cache (which uses git data)." >&2 + S3_BUILD_CACHE_UPLOAD=false + S3_BUILD_CACHE_DOWNLOAD=false + S3_BUILD_CACHE_MINIO_URL="" +elif [ ! -z "${AWS_ACCESS_KEY_ID:-}" ] ; then + S3_BUILD_CACHE_DOWNLOAD=true +else + S3_BUILD_CACHE_UPLOAD=false + S3_BUILD_CACHE_DOWNLOAD=false +fi + +earthly --secret AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} \ + --secret AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} \ + --secret S3_BUILD_CACHE_MINIO_URL="$S3_BUILD_CACHE_MINIO_URL" \ + --secret S3_BUILD_CACHE_UPLOAD="$S3_BUILD_CACHE_UPLOAD" \ + --secret S3_BUILD_CACHE_DOWNLOAD="$S3_BUILD_CACHE_DOWNLOAD" \ + --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${AZTEC_BOT_GITHUB_TOKEN:-} $@ \ No newline at end of file diff --git a/scripts/setup-earthly-local.sh b/scripts/setup-earthly-local.sh new file mode 100755 index 00000000000..59fafd19b4c --- /dev/null +++ b/scripts/setup-earthly-local.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +echo 'alias earthly="$(git rev-parse --show-toplevel)/scripts/earthly-local"' >> ~/.zshrc \ No newline at end of file