-
Notifications
You must be signed in to change notification settings - Fork 234
166 lines (157 loc) · 6.12 KB
/
ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
name: CI
on:
push:
branches: [master]
pull_request: {}
workflow_dispatch:
inputs:
runner_action:
description: "The action to take with the self-hosted runner (start, stop, restart)."
required: false
just_start_spot:
description: "Should we just run spots?"
type: boolean
required: false
concurrency:
# force parallelism in master
group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}
cancel-in-progress: true
jobs:
setup:
uses: ./.github/workflows/setup-runner.yml
with:
runner_label: ${{ github.actor }}-x86
ebs_cache_size_gb: 256
runner_concurrency: 20
subaction: ${{ github.event.inputs.runner_action || 'start' }}
ec2_instance_type: m6a.32xlarge
ec2_ami_id: ami-04d8422a9ba4de80f
ec2_instance_ttl: 40 # refreshed by jobs
secrets: inherit
build:
needs: setup
runs-on: ${{ github.actor }}-x86
if: ${{ github.event.inputs.just_start_spot != 'true' }}
outputs:
e2e_list: ${{ steps.e2e_list.outputs.list }}
steps:
- {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}}
- uses: ./.github/ci-setup-action
with:
dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
concurrency_key: build-${{ github.actor }}-x86
# prepare images locally, tagged by commit hash
- name: "Build E2E Image"
timeout-minutes: 40
run: earthly-ci ./yarn-project+export-end-to-end
# We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end
# (Note ARM uses just 2 tests as a smoketest)
- name: Create list of end-to-end jobs
id: e2e_list
run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT
# all the end-to-end integration tests for aztec
e2e:
needs: build
runs-on: ${{ github.actor }}-x86
strategy:
fail-fast: false
matrix:
test: ${{ fromJson( needs.build.outputs.e2e_list )}}
steps:
- {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}}
- uses: ./.github/ci-setup-action
with:
dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
# must be globally unique for build x runner
concurrency_key: e2e-${{ github.actor }}-x86-${{ matrix.test }}
- name: Test
working-directory: ./yarn-project/end-to-end/
timeout-minutes: 25
run: earthly-ci -P --no-output +${{ matrix.test }} --e2e_mode=cache
# TODO
# - name: Upload logs
# run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log
# barretenberg (prover) native and AVM (public VM) tests
# only ran on x86 for resource reasons (memory intensive)
bb-native-tests:
needs: setup
runs-on: ${{ github.actor }}-x86
strategy:
fail-fast: false
steps:
- {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}}
# Only allow one memory-hunger prover test to use this runner
- uses: ./.github/ci-setup-action
with:
dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
# must be globally unique for build x runner
concurrency_key: bb-native-tests-${{ github.actor }}-x86
- name: "Native Prover Tests"
working-directory: ./barretenberg/cpp/
timeout-minutes: 25
# limit our parallelism to half our cores
run: earthly-ci --no-output +test --hardware_concurrency=64
# push benchmarking binaries to dockerhub registry
bb-bench-binaries:
needs: setup
runs-on: ${{ github.actor }}-x86
steps:
- {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}}
- uses: ./.github/ci-setup-action
with:
dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
concurrency_key: bb-bench-binaries-${{ github.actor }}-x86
- name: Build and Push Binaries
if: ${{ github.event.inputs.just_start_spot != 'true' }}
timeout-minutes: 15
working-directory: ./barretenberg/cpp/
run: earthly-ci --push +bench-binaries
setup-bench:
uses: ./.github/workflows/setup-runner.yml
needs: bb-bench-binaries
with:
runner_label: ${{ github.actor }}-bench-x86
ebs_cache_size_gb: 64
runner_concurrency: 1
subaction: ${{ github.event.inputs.runner_action || 'start' }}
ec2_instance_type: m6a.4xlarge
ec2_ami_id: ami-04d8422a9ba4de80f
ec2_instance_ttl: 15 # refreshed by jobs
secrets: inherit
bb-bench:
runs-on: ${{ github.actor }}-bench-x86
needs: setup-bench
steps:
- {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}}
- uses: ./.github/ci-setup-action
with:
dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
concurrency_key: bb-bench-${{ github.actor }}-bench-x86
# Use bench_mode=cache to read the pushed build above
- name: Client IVC Bench
working-directory: ./barretenberg/cpp/
timeout-minutes: 15
run: earthly-ci --no-output +bench-client-ivc --bench_mode=cache
- name: Ultrahonk Bench
working-directory: ./barretenberg/cpp/
timeout-minutes: 15
run: earthly-ci --no-output +bench-ultra-honk --bench_mode=cache
merge-check:
runs-on: ${{ github.actor }}-x86
needs: [e2e, bb-native-tests, bb-bench]
steps:
- run: echo Pull request merging now allowed.
notify:
needs: [e2e, bb-native-tests, bb-bench]
runs-on: ubuntu-latest
if: ${{ github.ref == 'refs/heads/master' && failure() }}
steps:
- name: Send notification to aztec3-ci channel if workflow failed on master
uses: slackapi/slack-github-action@v1.25.0
with:
payload: |
{
"url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WORKFLOW_TRIGGER_URL }}