-
Notifications
You must be signed in to change notification settings - Fork 0
202 lines (202 loc) · 8.23 KB
/
flask.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
name: Build, test, and deploy backend
on:
push:
branches:
- master
paths:
- .github/workflows/flask.yml
- flask/**/*
- Dockerfile
- docker-compose*.yaml
pull_request:
branches:
- '*'
paths:
- .github/workflows/flask.yml
- flask/**/*
- Dockerfile
- docker-compose*.yaml
jobs:
build:
runs-on: ubuntu-latest
# Cancel in-progress build jobs for the current workflow so we're not wasting
# time on outdated builds. This isn't scoped to the entire workflow as cancelling
# an in-progress deploy could have unforeseen consequences.
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-only-cancel-in-progress-jobs-or-runs-for-the-current-workflow
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
COMPOSE_FILE: docker-compose.test.yaml
steps:
- uses: actions/checkout@v3
- uses: docker/setup-buildx-action@v2
- uses: actions/cache@v3
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
- name: Infer metadata (development image)
uses: docker/metadata-action@v4
id: meta-dev
with:
images: ghcr.io/${{ github.repository }}
tags: |
dev
- uses: docker/login-action@v2
if: github.event_name == 'push'
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build development image
uses: docker/build-push-action@v3
with:
tags: ${{ steps.meta-dev.outputs.tags }}
labels: ${{ steps.meta-dev.outputs.labels }}
context: flask
pull: true
load: true
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache,mode=max
- run: cp sample.env .env
- name: Pytest
run: docker-compose run --rm app pytest --cov=app --cov-report term-missing --cov-report xml
- uses: codecov/codecov-action@v3
with:
fail_ci_if_error: true
files: ./flask/coverage.xml
flags: pytest
- name: Black
run: docker-compose run --rm app black --check
- name: Stop services
run: docker-compose down
- name: Push development image
uses: docker/build-push-action@v3
if: github.event_name == 'push'
with:
push: true
tags: ${{ steps.meta-dev.outputs.tags }}
labels: ${{ steps.meta-dev.outputs.labels }}
context: flask
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache,mode=max
- name: Infer metadata (production image)
uses: docker/metadata-action@v4
if: github.event_name == 'push'
id: meta-prod
with:
images: ghcr.io/${{ github.repository }}
flavor: latest=${{ github.ref == 'refs/heads/master' }}
tags: |
type=ref,event=tag
type=sha,format=long
- name: Build production image
uses: docker/build-push-action@v3
if: github.event_name == 'push'
with:
push: true
tags: ${{ steps.meta-prod.outputs.tags }}
labels: ${{ steps.meta-prod.outputs.labels }}
context: flask
file: "{context}/../Dockerfile"
build-args: GIT_SHA=${{ github.sha }}
pull: true
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache,mode=max
deploy:
needs: build
if: github.event_name == 'push'
strategy:
matrix:
include:
- runner: cheo-ri
environment: CHEO-RI_backend
- runner: ccm
environment: CCM_backend
runs-on: [self-hosted, "${{ matrix.runner }}"]
environment: ${{ matrix.environment }}
concurrency: ${{ matrix.environment }}
steps:
- uses: actions/checkout@v3
- uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Configure SSH
id: configure
# The self-hosted runner is not ephemeral, so we load the secret key into
# an agent instead in memory. Keep track of it for further workflow steps
# and so we can clean it up and not leave orphaned processes hanging around.
#
# https://docs.docker.com/engine/context/working-with-contexts/
# This avoids passing an -H parameter to every Docker CLI call.
run: |
SSH_AGENT_EVAL=$(ssh-agent -s)
eval "$SSH_AGENT_EVAL"
ssh-add - <<< "${{ secrets.DEPLOY_PRIVATE_KEY }}"
echo "::set-output name=ssh-agent-eval::$SSH_AGENT_EVAL"
echo "::set-output name=ssh-agent-pid::$SSH_AGENT_PID"
mkdir -p ~/.ssh
echo "${{ secrets.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
chmod -R g-rwx,o-rwx ~/.ssh
docker context create deploy-target --docker host=ssh://${{ secrets.DEPLOY_SSH_HOST }}
docker context use deploy-target
# The secret mapping to environment variables is different, but the script is the same
- name: Deploy (CHEO-RI)
if: matrix.runner == 'cheo-ri'
# Even though this is deploying to a remote Docker Engine,
# Compose uses the registry credentials of the client
env:
COMPOSE_FILE: docker-compose.cheo.yaml
ST_VERSION: latest
ST_SECRET_KEY: ${{ secrets.ST_SECRET_KEY }}
MYSQL_CONNECTION_STRING: ${{ secrets.MYSQL_CONNECTION_STRING }}
MINIO_ENDPOINT: ${{ secrets.MINIO_ENDPOINT }}
MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }}
MINIO_SECRET_KEY: ${{ secrets.MINIO_SECRET_KEY }}
MSTEAMS_WEBHOOK_URL: ${{ secrets.MSTEAMS_WEBHOOK }}
SENDGRID_API_KEY: ${{ secrets.SENDGRID_API_KEY }}
SENDGRID_EMAIL_TEMPLATE_ID: ${{ secrets.SENDGRID_EMAIL_TEMPLATE_ID }}
SENDGRID_TO_EMAIL: ${{ secrets.SENDGRID_TO_EMAIL }}
SENDGRID_FROM_EMAIL: ${{ secrets.SENDGRID_FROM_EMAIL }}
SLURM_ENDPOINT: ${{ secrets.SLURM_ENDPOINT }}
SLURM_USER: ${{ secrets.SLURM_USER }}
SLURM_JWT: ${{ secrets.SLURM_JWT }}
run: |
eval "${{ steps.configure.outputs.ssh-agent-eval }}"
export GUNICORN_WORKERS=$( ssh ${{ secrets.DEPLOY_SSH_HOST }} 'echo $(( $(nproc) * 2 + 1 ))' )
docker-compose pull
docker-compose up -d --remove-orphans
- name: Deploy (CCM)
if: matrix.runner == 'ccm'
# Even though this is deploying to a remote Docker Engine,
# Compose uses the registry credentials of the client
env:
COMPOSE_FILE: docker-compose.ccm.yaml
ST_VERSION: latest
MSTEAMS_WEBHOOK_URL: ${{ secrets.MSTEAMS_WEBHOOK }}
PROJECT_ROOT: /opt/stager
HIRAKI_ST_SECRET_KEY: ${{ secrets.HIRAKI_ST_SECRET_KEY }}
HIRAKI_MYSQL_CONNECTION_STRING: ${{ secrets.HIRAKI_MYSQL_CONNECTION_STRING }}
HIRAKI_MINIO_ENDPOINT: ${{ secrets.HIRAKI_MINIO_ENDPOINT }}
HIRAKI_MINIO_ACCESS_KEY: ${{ secrets.HIRAKI_MINIO_ACCESS_KEY }}
HIRAKI_MINIO_SECRET_KEY: ${{ secrets.HIRAKI_MINIO_SECRET_KEY }}
MUISE_ST_SECRET_KEY: ${{ secrets.MUISE_ST_SECRET_KEY }}
MUISE_MYSQL_CONNECTION_STRING: ${{ secrets.MUISE_MYSQL_CONNECTION_STRING }}
MUISE_MINIO_ENDPOINT: ${{ secrets.MUISE_MINIO_ENDPOINT }}
MUISE_MINIO_ACCESS_KEY: ${{ secrets.MUISE_MINIO_ACCESS_KEY }}
MUISE_MINIO_SECRET_KEY: ${{ secrets.MUISE_MINIO_SECRET_KEY }}
run: |
eval "${{ steps.configure.outputs.ssh-agent-eval }}"
export GUNICORN_WORKERS=$( ssh ${{ secrets.DEPLOY_SSH_HOST }} 'echo $(( $(nproc) + 1 ))' )
docker-compose pull
docker-compose up -d --remove-orphans
- name: Clean up
if: always()
run: |
docker context rm -f deploy-target
eval "${{ steps.configure.outputs.ssh-agent-eval }}"
SSH_AGENT_PID="${{ steps.configure.outputs.ssh-agent-pid }}" ssh-agent -k