feat: CI/local S3 build cache

.github/workflows/ci.yml

@@ -118,7 +118,8 @@ jobs:
 
   build:
     needs: [build-images, changes]
-    if: (needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master'
+    # NOTE: we don't skip this job here instead using if's below so that dependencies are not broken
+    # we still want build to be at the bottom of the dependency chain
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     outputs:
       e2e_list: ${{ steps.e2e_list.outputs.list }}
@@ -127,11 +128,13 @@ jobs:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
+        if: (needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master'
         with:
           concurrency_key: build-x86
       # prepare images locally, tagged by commit hash
       - name: "Build E2E Image"
         timeout-minutes: 40
+        if: (needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master'
         run: |
           earthly-ci ./yarn-project+export-e2e-test-images
       # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end
@@ -160,7 +163,7 @@ jobs:
   # all the non-bench end-to-end integration tests for aztec
   e2e:
     needs: [build, changes]
-    if: needs.changes.outputs.non-barretenberg-cpp == 'true' || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'e2e')
+    if: (needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'e2e')
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
@@ -190,7 +193,7 @@ jobs:
   # all the benchmarking end-to-end integration tests for aztec (not required to merge)
   bench-e2e:
     needs: [build, changes]
-    if: needs.changes.outputs.non-barretenberg-cpp == 'true' || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'bench')
+    if: (needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'bench')
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
@@ -221,7 +224,7 @@ jobs:
 
   acir-bench:
     runs-on: ubuntu-20.04
-    needs: [build-images, changes]
+    needs: [build, changes]
     # Note: not fully accurate, but to work with bench-summary needs to be the same as bench-e2e
     if: needs.changes.outputs.non-barretenberg-cpp == 'true'
     steps:
@@ -258,7 +261,6 @@ jobs:
           ref: "${{ github.event.pull_request.head.sha }}"
       - uses: ./.github/ci-setup-action
         with:
-          dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
           concurrency_key: build-x86
       - name: "Build and upload bench aggregate file"
         working-directory: ./yarn-project/scripts
@@ -286,7 +288,7 @@ jobs:
           AZTEC_BOT_GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
 
   bb-gcc:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.barretenberg-cpp == 'true'
     steps:
@@ -306,7 +308,7 @@ jobs:
   # barretenberg (prover) native, AVM (public VM) and Merkle tree (world state) tests
   # only ran on x86 for resource reasons (memory intensive)
   bb-native-tests:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.barretenberg-cpp == 'true'
     steps:
@@ -324,7 +326,7 @@ jobs:
         run: earthly-ci --exec-stats --no-output +test --hardware_concurrency=64
 
   bb-js-test:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.barretenberg == 'true'
     steps:
@@ -339,7 +341,7 @@ jobs:
         run: earthly-ci --no-output ./+test
 
   noir-build-acir-tests:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true'
     steps:
@@ -413,7 +415,7 @@ jobs:
         run: earthly-ci --no-output ./+barretenberg-acir-tests-bb.js
 
   noir-format:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.noir == 'true' || needs.changes.outputs.noir-projects == 'true'
     steps:
@@ -433,7 +435,7 @@ jobs:
           earthly-ci --no-output ./+format
 
   noir-test:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.noir == 'true'
     steps:
@@ -446,7 +448,7 @@ jobs:
         run: earthly-ci --no-output ./noir+test
 
   noir-examples:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true'
     steps:
@@ -459,7 +461,7 @@ jobs:
         run: earthly-ci --no-output ./noir+examples
 
   noir-packages-test:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true'
     steps:
@@ -473,7 +475,7 @@ jobs:
         run: earthly-ci --no-output ./noir+packages-test
 
   noir-projects:
-    needs: [build-images, changes, build]
+    needs: [build, changes, build]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true' || needs.changes.outputs.noir-projects == 'true' || needs.changes.outputs.txe == 'true'
     steps:
@@ -488,7 +490,7 @@ jobs:
           earthly-ci --no-output ./noir-projects/+test
 
   avm-format:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.avm-transpiler == 'true' || needs.changes.outputs.noir == 'true'
     steps:
@@ -542,7 +544,7 @@ jobs:
         run: earthly-ci --no-output ./yarn-project/+prover-client-test
 
   l1-contracts-test:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.l1-contracts == 'true'
     steps:
@@ -555,7 +557,7 @@ jobs:
         run: earthly-ci --no-output ./l1-contracts+test
 
   docs-preview:
-    needs: [build-images, changes]
+    needs: [build, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     if: needs.changes.outputs.non-barretenberg-cpp == 'true'
     steps:
@@ -576,7 +578,7 @@ jobs:
 
   bb-bench:
     runs-on: ubuntu-20.04
-    needs: [build-images, changes]
+    needs: [build, changes]
     if: needs.changes.outputs.barretenberg-cpp == 'true'
     steps:
       - uses: actions/checkout@v4
@@ -661,7 +663,7 @@ jobs:
         run: earthly-ci -P --no-output +test --box=${{ matrix.box }} --browser=${{ matrix.browser }} --mode=cache
 
   protocol-circuits-gates-report:
-    needs: [build-images, changes]
+    needs: [build, changes]
     if: needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true'
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     permissions:
@@ -697,7 +699,7 @@ jobs:
           message: ${{ steps.gates_diff.outputs.markdown }}
 
   public-functions-size-report:
-    needs: [build-images, changes]
+    needs: [build, changes]
     if: needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true'
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     permissions:

avm-transpiler/.rebuild_patterns

@@ -0,0 +1 @@
+^avm-transpiler/.*$

avm-transpiler/Earthfile

@@ -4,8 +4,6 @@ source:
     # we rely on noir source, which this image has
     FROM ../noir+nargo
 
-    # move noir contents to /usr/src/noir
-    RUN mv /usr/src /noir && mkdir /usr/src && mv /noir /usr/src
     # work in avm-transpiler
     WORKDIR /usr/src/avm-transpiler
 

avm-transpiler/scripts/bootstrap_native.sh

@@ -9,7 +9,7 @@ export GIT_DIRTY=false
 if [ -f ".gitrepo" ]; then
   export GIT_COMMIT=$(awk '/commit =/ {print $3}' .gitrepo)
 else
-  export GIT_COMMIT=$(git rev-parse --verify HEAD)
+  export GIT_COMMIT=${COMMIT_HASH:-$(git rev-parse --verify HEAD)}
 fi
 
 # Build native.

barretenberg/Earthfile

@@ -104,6 +104,8 @@ barretenberg-acir-tests-bb.js:
     RUN cd ../ts && yarn
     ENV VERBOSE=1
     ENV TEST_SRC /usr/src/acir_artifacts
+    # We have diminishing returns after 32 cores, and unnecessarily use resources
+    ENV HARDWARE_CONCURRENCY=32
 
     # TODO(https://github.com/noir-lang/noir/issues/5106)
     # TODO(https://github.com/AztecProtocol/aztec-packages/issues/6672)c