Merge branch 'master' into jz/docs-tut-crowdfund

AztecProtocol · Apr 18, 2024 · c1c1ae3 · c1c1ae3
2 parents 2a6d451 + cb1e25b
commit c1c1ae3
Show file tree

Hide file tree

Showing 846 changed files with 15,853 additions and 21,215 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
diff --git a/.github/ci-setup-action/action.yml b/.github/ci-setup-action/action.yml
@@ -64,5 +64,5 @@ runs:
           echo "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" > "/run/${{ inputs.concurrency_key }}.lock"
           echo "/run/${{ inputs.concurrency_key }}.lock acquired."
         post: |
-          rm "/run/${{ inputs.concurrency_key }}.lock"
+          rm "/run/${{ inputs.concurrency_key }}.lock" || true
           echo "/run/${{ inputs.concurrency_key }}.lock removed."
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -40,13 +40,11 @@ jobs:
       - uses: ./.github/ci-setup-action
         with:
           dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
-          concurrency_token: "${{ secrets.AZTEC_GITHUB_TOKEN }}"
-          # must be globally unique for build x runner
           concurrency_key: build-${{ github.actor }}-x86
       # prepare images locally, tagged by commit hash
       - name: "Build E2E Image"
         timeout-minutes: 40
-        run: earthly ./yarn-project+export-end-to-end
+        run: earthly-ci ./yarn-project+export-end-to-end
       # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end
       # (Note ARM uses just 2 tests as a smoketest)
       - name: Create list of end-to-end jobs
@@ -66,18 +64,17 @@ jobs:
       - uses: ./.github/ci-setup-action
         with:
           dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
-          concurrency_token: "${{ secrets.AZTEC_GITHUB_TOKEN }}"
           # must be globally unique for build x runner
           concurrency_key: e2e-${{ github.actor }}-x86-${{ matrix.test }}
       - name: Test
         working-directory: ./yarn-project/end-to-end/
         timeout-minutes: 25
-        run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache
+        run: earthly-ci -P --no-output +${{ matrix.test }} --e2e_mode=cache
       # TODO
       # - name: Upload logs
       #   run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log
 
-  # barretenberg (prover) native tests
+  # barretenberg (prover) native and AVM (public VM) tests
   # only ran on x86 for resource reasons (memory intensive)
   bb-native-tests:
     needs: setup
@@ -90,13 +87,13 @@ jobs:
       - uses: ./.github/ci-setup-action
         with:
           dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
-          concurrency_token: "${{ secrets.AZTEC_GITHUB_TOKEN }}"
           # must be globally unique for build x runner
           concurrency_key: bb-native-tests-${{ github.actor }}-x86
       - name: "Native Prover Tests"
         working-directory: ./barretenberg/cpp/
         timeout-minutes: 25
-        run: earthly --no-output +test
+        # limit our parallelism to half our cores
+        run: earthly-ci --no-output +test --hardware_concurrency=64
 
   # push benchmarking binaries to dockerhub registry
   bb-bench-binaries:
@@ -107,14 +104,12 @@ jobs:
       - uses: ./.github/ci-setup-action
         with:
           dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
-          concurrency_token: "${{ secrets.AZTEC_GITHUB_TOKEN }}"
-          # must be globally unique for build x runner
           concurrency_key: bb-bench-binaries-${{ github.actor }}-x86
       - name: Build and Push Binaries
         if: ${{ github.event.inputs.just_start_spot != 'true' }}
         timeout-minutes: 15
         working-directory: ./barretenberg/cpp/
-        run: earthly --push +bench-binaries
+        run: earthly-ci --push +bench-binaries
 
   setup-bench:
     uses: ./.github/workflows/setup-runner.yml
@@ -137,35 +132,35 @@ jobs:
       - uses: ./.github/ci-setup-action
         with:
           dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
-          concurrency_token: "${{ secrets.AZTEC_GITHUB_TOKEN }}"
-          # must be globally unique for build x runner
-          # technically not needed as we only make one GA runner, but a backup
           concurrency_key: bb-bench-${{ github.actor }}-bench-x86
       # Use bench_mode=cache to read the pushed build above
       - name: Client IVC Bench
         working-directory: ./barretenberg/cpp/
         timeout-minutes: 15
-        run: earthly --no-output +bench-client-ivc --bench_mode=cache
+        run: earthly-ci --no-output +bench-client-ivc --bench_mode=cache
 
       - name: Ultrahonk Bench
         working-directory: ./barretenberg/cpp/
         timeout-minutes: 15
-        run: earthly --no-output +bench-ultra-honk --bench_mode=cache
-
-  # # Post actions, deploy and summarize logs
-  # aztec-bench-summary:
-  #   runs-on: ${{ github.actor }}
-  #  # IMPORTANT security flaw if we don't need 'check-run-condition'
-  #   needs: e2e-x86
-  #   concurrency:
-  #     group: aztec-bench-summary-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86
-  #     cancel-in-progress: true
-  #   steps:
-  #     - name: Checkout
-  #       uses: actions/checkout@v4
-  #       with:
-  #         ref: ${{ github.event.pull_request.head.sha }}
+        run: earthly-ci --no-output +bench-ultra-honk --bench_mode=cache
 
-  #     - name: "Assemble benchmark summary from uploaded logs"
-  #         command: ./scripts/ci/assemble_e2e_benchmark_earthly.sh
+  merge-check:
+    runs-on: ${{ github.actor }}-x86
+    needs: [e2e, bb-native-tests, bb-bench]
+    steps:
+      - run: echo Pull request merging now allowed.
 
+  notify:
+    needs: [e2e, bb-native-tests, bb-bench]
+    runs-on: ubuntu-latest
+    if: ${{ github.ref == 'refs/heads/master' && failure() }}
+    steps:
+      - name: Send notification to aztec3-ci channel if workflow failed on master
+        uses: slackapi/slack-github-action@v1.25.0
+        with:
+          payload: |
+            {
+              "url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WORKFLOW_TRIGGER_URL }}
diff --git a/.github/workflows/setup-runner.yml b/.github/workflows/setup-runner.yml
@@ -3,7 +3,7 @@
 # just for the CI job. These are specced per user and run the entire CI.
 # TODO These have a persistent EBS volume that forms a fast-online docker image cache (used by Earthly), meaning
 # TODO build steps that ran in previous invocations are quickly ran from cache.
-name: Reusable Spot Instance and Setup Workflow
+name: Setup Runner and CI
 on:
   workflow_call:
     inputs:
@@ -52,9 +52,12 @@ on:
 jobs:
   start-builder:
     runs-on: ubuntu-latest
+    # we want to avoid race conditions when making spot across multiple PRs as we only use one runner
+    concurrency:
+      group: start-builder-${{ inputs.runner_label }}
     steps:
       - name: Start EC2 runner
-        uses: AztecProtocol/ec2-action-builder@v0.6
+        uses: AztecProtocol/ec2-action-builder@v0.13
         with:
           github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }}
           aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -69,6 +72,7 @@ jobs:
           ec2_instance_type: ${{ inputs.ec2_instance_type }}
           ec2_ami_id: ${{ inputs.ec2_ami_id }}
           ec2_instance_ttl: ${{ inputs.ec2_instance_ttl }}
+          ec2_key_name: "build-instance"
           ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]'
 
   setup:
@@ -107,7 +111,7 @@ jobs:
 
       - name: Run Docker Prune
         # helps to not overuse space
-        run: docker system prune -f
+        run: docker system prune -f || true
 
       - name: Run Earthly Bootstrap
         run: earthly bootstrap
diff --git a/.github/workflows/start-spot.yml b/.github/workflows/start-spot.yml
@@ -0,0 +1,28 @@
+# Useful if the spot runners are in a bad state
+name: Start Personal Spot
+on:
+  workflow_dispatch: {}
+jobs:
+  stop-build-x86:
+    uses: ./.github/workflows/setup-runner.yml
+    with:
+      runner_label: ${{ github.actor }}-x86
+      ebs_cache_size_gb: 256
+      runner_concurrency: 50
+      subaction: start
+      ec2_instance_type: m6a.32xlarge
+      ec2_ami_id: ami-04d8422a9ba4de80f
+      ec2_instance_ttl: 40 # refreshed by jobs
+    secrets: inherit
+
+  stop-bench:
+    uses: ./.github/workflows/setup-runner.yml
+    with:
+      runner_label: ${{ github.actor }}-bench-x86
+      ebs_cache_size_gb: 64
+      runner_concurrency: 1
+      subaction: start
+      ec2_instance_type: m6a.4xlarge
+      ec2_ami_id: ami-04d8422a9ba4de80f
+      ec2_instance_ttl: 15 # refreshed by jobs
+    secrets: inherit
diff --git a/.noir-sync-commit b/.noir-sync-commit
@@ -1 +1 @@
-1d96937a8e94a91c0c17c97102498d067fca76c3
+6cc105ee441e093b4fccdd5fcc3db922eb28a3fb
diff --git a/.release-please-manifest.json b/.release-please-manifest.json
@@ -1,7 +1,7 @@
 {
-  ".": "0.34.0",
-  "yarn-project/cli": "0.34.0",
-  "yarn-project/aztec": "0.34.0",
-  "barretenberg": "0.34.0",
-  "barretenberg/ts": "0.34.0"
+  ".": "0.35.1",
+  "yarn-project/cli": "0.35.1",
+  "yarn-project/aztec": "0.35.1",
+  "barretenberg": "0.35.1",
+  "barretenberg/ts": "0.35.1"
 }
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -125,10 +125,6 @@
   "[javascript]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode"
   },
-  // Now when a new file is pasted in /yellow-paper/docs/readme.md, the image file is created at /yellow-paper/docs/images/readme/image.png.
-  "markdown.copyFiles.destination": {
-    "/yellow-paper/**/*": "images/${documentBaseName}/"
-  },
   ///////////////////////////////////////
   // C++/Circuits settings
   ///////////////////////////////////////
@@ -169,5 +165,6 @@
     "**/target/**": true,
     "**/l1-contracts/lib/**": true,
     "**/barretenberg/cpp/build*/**": true
-  }
+  },
+  "cmake.sourceDirectory": "/mnt/user-data/adam/aztec-packages/barretenberg/cpp"
 }