feat: add proving retries #6795
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [master] | |
| pull_request: {} | |
| workflow_dispatch: | |
| inputs: | |
| username: | |
| description: "Defaults to GitHub Actor" | |
| required: false | |
| runner_action: | |
| description: "The action to take with the self-hosted runner (start, stop, restart)." | |
| required: false | |
| concurrency: | |
| # force parallelism in master | |
| group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} | |
| cancel-in-progress: true | |
| jobs: | |
| setup: | |
| uses: ./.github/workflows/setup-runner.yml | |
| with: | |
| runner_label: ${{ inputs.username || github.actor }}-x86 | |
| ebs_cache_size_gb: 256 | |
| runner_concurrency: 20 | |
| subaction: ${{ inputs.runner_action || 'start' }} | |
| # This instance list has a spot placement score of 9/10. Note we used to just use m6a.32xlarge, which had a score of 1! | |
| # https://us-east-2.console.aws.amazon.com/ec2/home?region=us-east-2#SpotPlacementScore: | |
| ec2_instance_type: i4i.32xlarge m6a.32xlarge m6i.32xlarge m6id.32xlarge m6idn.32xlarge m6in.32xlarge m7a.32xlarge r6a.32xlarge r6i.32xlarge r6id.32xlarge | |
| ec2_ami_id: ami-04d8422a9ba4de80f | |
| ec2_spot_instance_strategy: BestEffort | |
| ec2_instance_ttl: 40 # refreshed by jobs | |
| secrets: inherit | |
| build: | |
| needs: setup | |
| runs-on: ${{ inputs.username || github.actor }}-x86 | |
| outputs: | |
| e2e_list: ${{ steps.e2e_list.outputs.list }} | |
| steps: | |
| - { | |
| uses: actions/checkout@v4, | |
| with: { ref: "${{ github.event.pull_request.head.sha }}" }, | |
| } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" | |
| concurrency_key: build-${{ inputs.username || github.actor }}-x86 | |
| # prepare images locally, tagged by commit hash | |
| - name: "Build E2E Image" | |
| timeout-minutes: 40 | |
| run: earthly-ci ./yarn-project+export-e2e-test-images | |
| # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end | |
| # (Note ARM uses just 2 tests as a smoketest) | |
| - name: Create list of end-to-end jobs | |
| id: e2e_list | |
| run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT | |
| # all the end-to-end integration tests for aztec | |
| e2e: | |
| needs: build | |
| runs-on: ${{ inputs.username || github.actor }}-x86 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test: ${{ fromJson( needs.build.outputs.e2e_list )}} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: { ref: "${{ github.event.pull_request.head.sha }}" } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" | |
| # must be globally unique for build x runner | |
| concurrency_key: e2e-${{ inputs.username || github.actor }}-x86-${{ matrix.test }} | |
| - name: Test | |
| working-directory: ./yarn-project/end-to-end/ | |
| timeout-minutes: 25 | |
| run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --no-output +${{ matrix.test }} | |
| # bench-summary: | |
| # needs: e2e | |
| # runs-on: ${{ inputs.username || github.actor }}-x86 | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # with: | |
| # fetch-depth: 100 # Downloading base benchmark from master requires access to history | |
| # ref: "${{ github.event.pull_request.head.sha }}" | |
| # - uses: ./.github/ci-setup-action | |
| # with: | |
| # dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" | |
| # concurrency_key: build-${{ inputs.username || github.actor }}-x86 | |
| # - name: "Build and upload bench aggregate file" | |
| # working-directory: ./yarn-project/scripts | |
| # run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} +bench-aggregate | |
| # - name: "Download base benchmark" | |
| # if: ${{ github.event_name == 'pull_request' }} | |
| # run: scripts/logs/download_base_benchmark_from_s3.sh | |
| # env: | |
| # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| # BENCH_FOLDER: ./yarn-project/scripts/bench | |
| # PULL_REQUEST: "${{ github.event.pull_request.number }}" | |
| # - name: "Generate summary comment if pull request" | |
| # if: ${{ github.event_name == 'pull_request' }} | |
| # working-directory: ./yarn-project/scripts | |
| # run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} +bench-comment | |
| # barretenberg (prover) native and AVM (public VM) tests | |
| # only ran on x86 for resource reasons (memory intensive) | |
| bb-native-tests: | |
| needs: setup | |
| runs-on: ${{ inputs.username || github.actor }}-x86 | |
| steps: | |
| - { | |
| uses: actions/checkout@v4, | |
| with: { ref: "${{ github.event.pull_request.head.sha }}" }, | |
| } | |
| # Only allow one memory-hunger prover test to use this runner | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" | |
| # must be globally unique for build x runner | |
| concurrency_key: bb-native-tests-${{ inputs.username || github.actor }}-x86 | |
| - name: "Native Prover Tests" | |
| working-directory: ./barretenberg/cpp/ | |
| timeout-minutes: 25 | |
| # limit our parallelism to half our cores | |
| run: earthly-ci --no-output +test --hardware_concurrency=64 | |
| noir-projects: | |
| needs: setup | |
| runs-on: ${{ inputs.username || github.actor }}-x86 | |
| steps: | |
| - { | |
| uses: actions/checkout@v4, | |
| with: { ref: "${{ github.event.pull_request.head.sha }}" }, | |
| } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" | |
| concurrency_key: noir-projects-${{ inputs.username || github.actor }}-x86 | |
| - name: "Noir Projects" | |
| timeout-minutes: 25 | |
| run: earthly-ci --no-output ./noir-projects/+test | |
| yarn-project-formatting: | |
| needs: setup | |
| runs-on: ${{ github.actor }}-x86 | |
| steps: | |
| - { | |
| uses: actions/checkout@v4, | |
| with: { ref: "${{ github.event.pull_request.head.sha }}" }, | |
| } | |
| # Only allow one memory-hunger prover test to use this runner | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" | |
| concurrency_key: yarn-project-formatting-${{ github.actor }}-x86 | |
| - name: "Yarn Project Tests" | |
| timeout-minutes: 25 | |
| run: earthly-ci --no-output ./yarn-project/+format-check | |
| yarn-project-test: | |
| needs: noir-projects | |
| runs-on: ${{ github.actor }}-x86 | |
| steps: | |
| - { | |
| uses: actions/checkout@v4, | |
| with: { ref: "${{ github.event.pull_request.head.sha }}" }, | |
| } | |
| # Only allow one memory-hunger prover test to use this runner | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" | |
| concurrency_key: yarn-project-test-${{ github.actor }}-x86 | |
| - name: "Yarn Project Tests" | |
| timeout-minutes: 25 | |
| run: earthly-ci --no-output ./yarn-project/+test | |
| prover-client-test: | |
| needs: noir-projects | |
| runs-on: ${{ github.actor }}-x86 | |
| steps: | |
| - { | |
| uses: actions/checkout@v4, | |
| with: { ref: "${{ github.event.pull_request.head.sha }}" }, | |
| } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" | |
| concurrency_key: prover-client-test-${{ github.actor }}-x86 | |
| - name: "Prover Client Tests" | |
| timeout-minutes: 25 | |
| run: earthly-ci --no-output ./yarn-project/+prover-client-test | |
| docs-preview: | |
| needs: setup | |
| runs-on: ${{ inputs.username || github.actor }}-x86 | |
| if: github.event.number | |
| steps: | |
| - { | |
| uses: actions/checkout@v4, | |
| with: { ref: "${{ github.event.pull_request.head.sha }}" }, | |
| } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" | |
| concurrency_key: docs-preview-${{ inputs.username || github.actor }}-x86 | |
| - name: "Docs Preview" | |
| timeout-minutes: 25 | |
| run: earthly --no-output ./docs/+deploy-preview --PR=${{ github.event.number }} --AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} --NETLIFY_AUTH_TOKEN=${{ secrets.NETLIFY_AUTH_TOKEN }} --NETLIFY_SITE_ID=${{ secrets.NETLIFY_SITE_ID }} | |
| # push benchmarking binaries to dockerhub registry | |
| bb-bench-binaries: | |
| needs: setup | |
| runs-on: ${{ inputs.username || github.actor }}-x86 | |
| steps: | |
| - { | |
| uses: actions/checkout@v4, | |
| with: { ref: "${{ github.event.pull_request.head.sha }}" }, | |
| } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" | |
| concurrency_key: bb-bench-binaries-${{ inputs.username || github.actor }}-x86 | |
| - name: Build and Push Binaries | |
| timeout-minutes: 15 | |
| working-directory: ./barretenberg/cpp/ | |
| run: earthly-ci --push +bench-binaries | |
| setup-bench: | |
| uses: ./.github/workflows/setup-runner.yml | |
| needs: bb-bench-binaries | |
| with: | |
| runner_label: ${{ inputs.username || github.actor }}-bench-x86 | |
| ebs_cache_size_gb: 64 | |
| runner_concurrency: 1 | |
| subaction: ${{ inputs.runner_action || 'start' }} | |
| ec2_instance_type: m6a.4xlarge | |
| ec2_ami_id: ami-04d8422a9ba4de80f | |
| ec2_instance_ttl: 15 # refreshed by jobs | |
| secrets: inherit | |
| bb-bench: | |
| runs-on: ${{ inputs.username || github.actor }}-bench-x86 | |
| needs: setup-bench | |
| steps: | |
| - { | |
| uses: actions/checkout@v4, | |
| with: { ref: "${{ github.event.pull_request.head.sha }}" }, | |
| } | |
| - uses: ./.github/ci-setup-action | |
| with: | |
| dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" | |
| concurrency_key: bb-bench-${{ inputs.username || github.actor }}-bench-x86 | |
| # Use bench_mode=cache to read the pushed build above | |
| - name: Client IVC Bench | |
| working-directory: ./barretenberg/cpp/ | |
| timeout-minutes: 15 | |
| run: earthly-ci --no-output +bench-client-ivc --bench_mode=cache | |
| - name: Ultrahonk Bench | |
| working-directory: ./barretenberg/cpp/ | |
| timeout-minutes: 15 | |
| run: earthly-ci --no-output +bench-ultra-honk --bench_mode=cache | |
| merge-check: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - e2e | |
| - bb-native-tests | |
| - bb-bench | |
| - yarn-project-formatting | |
| - yarn-project-test | |
| - prover-client-test | |
| if: always() | |
| steps: | |
| - run: | | |
| echo "e2e status: ${{ needs.e2e.result }}" | |
| echo "bb-native-tests status: ${{ needs.bb-native-tests.result }}" | |
| echo "bb-bench status: ${{ needs.bb-bench.result }}" | |
| echo "yarn-project-formatting status: ${{ needs.yarn-project-formatting.result }}" | |
| echo "yarn-project-test status: ${{ needs.yarn-project-test.result }}" | |
| if [[ "${{ needs.e2e.result }}" != 'success' || "${{ needs.bb-native-tests.result }}" != 'success' || "${{ needs.bb-bench.result }}" != 'success' || "${{ needs.yarn-project-formatting.result }}" != 'success' || "${{ needs.yarn-project-test.result }}" != 'success' ]]; then | |
| echo "Pull request merging not allowed due to failures." | |
| exit 1 | |
| fi | |
| echo "Pull request merging now allowed." | |
| notify: | |
| needs: | |
| [ | |
| e2e, | |
| bb-native-tests, | |
| bb-bench, | |
| yarn-project-formatting, | |
| yarn-project-test, | |
| prover-client-test, | |
| ] | |
| runs-on: ubuntu-latest | |
| if: ${{ github.ref == 'refs/heads/master' && failure() }} | |
| steps: | |
| - name: Send notification to aztec3-ci channel if workflow failed on master | |
| uses: slackapi/slack-github-action@v1.25.0 | |
| with: | |
| payload: | | |
| { | |
| "url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WORKFLOW_TRIGGER_URL }} |