chore(ci): fix restarts with fresh spot, acir test fixes, non-mandato…

…ry benches (#6226)

Co-authored-by: Santiago Palladino <santiago@aztecprotocol.com>

.github/workflows/ci.yml

@@ -52,11 +52,14 @@ jobs:
         run: earthly-ci ./yarn-project+export-e2e-test-images
       # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end
       # (Note ARM uses just 2 tests as a smoketest)
-      - name: Create list of end-to-end jobs
+      - name: Create list of non-bench end-to-end jobs
         id: e2e_list
-        run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' |  sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT
+        run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' | grep -v '+bench' |  sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT
+      - name: Create list of bench end-to-end jobs
+        id: bench_list
+        run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep '+bench' |  sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT
 
-  # all the end-to-end integration tests for aztec
+  # all the non-bench end-to-end integration tests for aztec
   e2e:
     needs: build
     runs-on: ${{ inputs.username || github.actor }}-x86
@@ -81,6 +84,31 @@ jobs:
           --no-output \
           +${{ matrix.test }} --skip_build=true
 
+  # all the benchmarking end-to-end integration tests for aztec (not required to merge)
+  bench-e2e:
+    needs: build
+    runs-on: ${{ inputs.username || github.actor }}-x86
+    strategy:
+      fail-fast: false
+      matrix:
+        test: ${{ fromJson( needs.build.outputs.bench_list )}}
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+      - uses: ./.github/ci-setup-action
+        with:
+          dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
+          # must be globally unique for build x runner
+          concurrency_key: e2e-${{ inputs.username || github.actor }}-x86-${{ matrix.test }}
+      - name: Test
+        working-directory: ./yarn-project/end-to-end/
+        timeout-minutes: 25
+        run: earthly-ci -P \
+          --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
+          --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
+          --no-output \
+          +${{ matrix.test }} --skip_build=true
+
   # bench-summary:
   #   needs: e2e
   #   runs-on: ${{ inputs.username || github.actor }}-x86
@@ -112,7 +140,7 @@ jobs:
   # barretenberg (prover) native and AVM (public VM) tests
   # only ran on x86 for resource reasons (memory intensive)
   bb-native-tests:
-    needs: build
+    needs: setup
     runs-on: ${{ inputs.username || github.actor }}-x86
     steps:
       - {

.github/workflows/setup-runner.yml

@@ -147,4 +147,6 @@ jobs:
           fi
 
       - name: Run Earthly Bootstrap
-        run: earthly bootstrap
+        run: |
+          earthly bootstrap
+          touch /run/.earthly-bootstrap # Used in `earthly-ci` wrapper to check that earthly has been bootstrapped ok

barretenberg/cpp/Earthfile

@@ -180,6 +180,9 @@ test-clang-format:
 
 test:
     ARG hardware_concurrency=""
+    # prefetch
+    BUILD +test-binaries
+    BUILD +preset-release-assert-test
     BUILD +test-clang-format
     BUILD ./srs_db/+build # prefetch
     FROM +source

noir/.gitignore

@@ -1,2 +1,3 @@
 **/package.tgz
-packages
+packages
+.earthly-staging