feat: CI/local S3 build cache (#8802)

What this PR achieves: Optional S3 caching in Earthly files with a 2nd layer of file caching (except in C). A future step would be to use this cache locally for fast bootstrapping. Currently implemented for barretenberg and noir. If AWS credentials are available with this script, S3 will be used as a download source. Otherwise, just the local minio cache will be used. If both are used, minio will act like a pull-through cache for S3 and provide a two-tiered caching solution good for when earthly is being really forgetful and you are using artifacts that have not changed in a while. USAGE NOTE: After this PR, you will want to run on scripts/setup-earthly-local.sh or equivalent if not using zsh. This will create an alias for earthly as earthly-local in the repo, which sets up scripts and starts the (currently always on) file server. USAGE NOTE: Because we use git to reliably make content hashes that aren't prone to changing due to temporary files, we require all changes to be committed before we content hashes will be used. The cache will not be used at all if there are staged changes. Taking feedback on this. I recommend doing something like `git commit -am 'sync' && earthly ...` if changing files in a loop (possibly with --amend). Remember to `git add` new wanted files or you may get old cache. USAGE NOTE: If you do not want AWS secrets in the env for every program you can figure out your own wrapper alias, perhaps reading from files EARTHFILE DEV NOTE: There is a little bit of a dance to grab rebuild patterns and run with the git context on the local machine. To enable this, WORKDIR needs to match the repo layout, some adjustments thusly Bundled: - Just make build the root of the dependency tree in github. This should fix a lot of contention issues Followups: - Cache protocol circuits and verification keys
AztecProtocol · Sep 27, 2024 · 06be26e · 06be26e
1 parent 7411acc
commit 06be26e
Show file tree

Hide file tree

Showing 22 changed files with 670 additions and 310 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -118,7 +118,8 @@ jobs:
 
   build:
     needs: [build-images, changes]
-    if: (needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master'
+    # NOTE: we don't skip this job here instead using if's below so that dependencies are not broken
+    # we still want build to be at the bottom of the dependency chain
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     outputs:
       e2e_list: ${{ steps.e2e_list.outputs.list }}
@@ -127,11 +128,13 @@ jobs:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
+        if: (needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master'
         with:
           concurrency_key: build-x86
       # prepare images locally, tagged by commit hash
       - name: "Build E2E Image"
         timeout-minutes: 40
+        if: (needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master'
         run: |
           earthly-ci ./yarn-project+export-e2e-test-images
       # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end
@@ -160,7 +163,7 @@ jobs:
   # all the non-bench end-to-end integration tests for aztec
   e2e:
     needs: [build, changes]
-    if: needs.changes.outputs.non-barretenberg-cpp == 'true' || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'e2e')
+    if: (needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'e2e')
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
@@ -190,7 +193,7 @@ jobs:
   # all the benchmarking end-to-end integration tests for aztec (not required to merge)
   bench-e2e:
     needs: [build, changes]
-    if: needs.changes.outputs.non-barretenberg-cpp == 'true' || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'bench')
+    if: (needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'bench')
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
@@ -221,7 +224,7 @@ jobs:
 
   acir-bench:
     runs-on: ubuntu-20.04
-    needs: [build-images, changes]
+    needs: [build, changes]
     # Note: not fully accurate, but to work with bench-summary needs to be the same as bench-e2e
     if: needs.changes.outputs.non-barretenberg-cpp == 'true'
     steps:
@@ -258,7 +261,6 @@ jobs:
           ref: "${{ github.event.pull_request.head.sha }}"
       - uses: ./.github/ci-setup-action
         with:
-          dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
           concurrency_key: build-x86
       - name: "Build and upload bench aggregate file"
         working-directory: ./yarn-project/scripts
@@ -286,7 +288,7 @@ jobs:
           AZTEC_BOT_GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
 
   bb-gcc:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.barretenberg-cpp == 'true'
     steps:
@@ -306,7 +308,7 @@ jobs:
   # barretenberg (prover) native, AVM (public VM) and Merkle tree (world state) tests
   # only ran on x86 for resource reasons (memory intensive)
   bb-native-tests:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.barretenberg-cpp == 'true'
     steps:
@@ -324,7 +326,7 @@ jobs:
         run: earthly-ci --exec-stats --no-output +test --hardware_concurrency=64
 
   bb-js-test:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.barretenberg == 'true'
     steps:
@@ -339,7 +341,7 @@ jobs:
         run: earthly-ci --no-output ./+test
 
   noir-build-acir-tests:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true'
     steps:
@@ -413,7 +415,7 @@ jobs:
         run: earthly-ci --no-output ./+barretenberg-acir-tests-bb.js
 
   noir-format:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.noir == 'true' || needs.changes.outputs.noir-projects == 'true'
     steps:
@@ -433,7 +435,7 @@ jobs:
           earthly-ci --no-output ./+format
 
   noir-test:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.noir == 'true'
     steps:
@@ -446,7 +448,7 @@ jobs:
         run: earthly-ci --no-output ./noir+test
 
   noir-examples:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true'
     steps:
@@ -459,7 +461,7 @@ jobs:
         run: earthly-ci --no-output ./noir+examples
 
   noir-packages-test:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true'
     steps:
@@ -473,7 +475,7 @@ jobs:
         run: earthly-ci --no-output ./noir+packages-test
 
   noir-projects:
-    needs: [build-images, changes, build]
+    needs: [build, changes, build]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true' || needs.changes.outputs.noir-projects == 'true' || needs.changes.outputs.txe == 'true'
     steps:
@@ -488,7 +490,7 @@ jobs:
           earthly-ci --no-output ./noir-projects/+test
 
   avm-format:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.avm-transpiler == 'true' || needs.changes.outputs.noir == 'true'
     steps:
@@ -542,7 +544,7 @@ jobs:
         run: earthly-ci --no-output ./yarn-project/+prover-client-test
 
   l1-contracts-test:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.l1-contracts == 'true'
     steps:
@@ -555,7 +557,7 @@ jobs:
         run: earthly-ci --no-output ./l1-contracts+test
 
   docs-preview:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.non-barretenberg-cpp == 'true'
     steps:
@@ -576,7 +578,7 @@ jobs:
 
   bb-bench:
     runs-on: ubuntu-20.04
-    needs: [build-images, changes]
+    needs: [build, changes]
     if: needs.changes.outputs.barretenberg-cpp == 'true'
     steps:
       - uses: actions/checkout@v4
@@ -661,7 +663,7 @@ jobs:
         run: earthly-ci -P --no-output +test --box=${{ matrix.box }} --browser=${{ matrix.browser }} --mode=cache
 
   protocol-circuits-gates-report:
-    needs: [build-images, changes]
+    needs: [build, changes]
     if: needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true'
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     permissions:
@@ -697,7 +699,7 @@ jobs:
           message: ${{ steps.gates_diff.outputs.markdown }}
 
   public-functions-size-report:
-    needs: [build-images, changes]
+    needs: [build, changes]
     if: needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true'
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     permissions:

diff --git a/avm-transpiler/.rebuild_patterns b/avm-transpiler/.rebuild_patterns
@@ -0,0 +1 @@
+^avm-transpiler/.*$
diff --git a/avm-transpiler/Earthfile b/avm-transpiler/Earthfile
@@ -4,8 +4,6 @@ source:
     # we rely on noir source, which this image has
     FROM ../noir+nargo
 
-    # move noir contents to /usr/src/noir
-    RUN mv /usr/src /noir && mkdir /usr/src && mv /noir /usr/src
     # work in avm-transpiler
     WORKDIR /usr/src/avm-transpiler
 

diff --git a/avm-transpiler/scripts/bootstrap_native.sh b/avm-transpiler/scripts/bootstrap_native.sh
@@ -9,7 +9,7 @@ export GIT_DIRTY=false
 if [ -f ".gitrepo" ]; then
   export GIT_COMMIT=$(awk '/commit =/ {print $3}' .gitrepo)
 else
-  export GIT_COMMIT=$(git rev-parse --verify HEAD)
+  export GIT_COMMIT=${COMMIT_HASH:-$(git rev-parse --verify HEAD)}
 fi
 
 # Build native.

diff --git a/barretenberg/Earthfile b/barretenberg/Earthfile
@@ -104,6 +104,8 @@ barretenberg-acir-tests-bb.js:
     RUN cd ../ts && yarn
     ENV VERBOSE=1
     ENV TEST_SRC /usr/src/acir_artifacts
+    # We have diminishing returns after 32 cores, and unnecessarily use resources
+    ENV HARDWARE_CONCURRENCY=32
 
     # TODO(https://github.com/noir-lang/noir/issues/5106)
     # TODO(https://github.com/AztecProtocol/aztec-packages/issues/6672)c