forked from apache/beam
-
Notifications
You must be signed in to change notification settings - Fork 0
411 lines (395 loc) · 16.2 KB
/
build_wheels.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# To learn more about GitHub Actions in Apache Beam check the CI.md
name: Build python source distribution and wheels
on:
schedule:
- cron: '10 2 * * *'
push:
branches: ['master', 'release-*']
tags: 'v*'
pull_request:
branches: ['master', 'release-*']
tags: 'v*'
paths: ['sdks/python/**', 'model/**', 'release/**']
workflow_dispatch:
# This allows a subsequently queued workflow run to interrupt previous runs
concurrency:
group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login}}'
cancel-in-progress: true
env:
GCP_PATH: "gs://${{ secrets.GCP_PYTHON_WHEELS_BUCKET }}/${GITHUB_REF##*/}/${GITHUB_SHA}-${GITHUB_RUN_ID}/"
jobs:
check_env_variables:
timeout-minutes: 5
name: "Check environment variables"
runs-on: ubuntu-latest
env:
EVENT_NAME: ${{ github.event_name }}
PY_VERSIONS_FULL: "cp38-* cp39-* cp310-* cp311-*"
outputs:
gcp-variables-set: ${{ steps.check_gcp_variables.outputs.gcp-variables-set }}
py-versions-full: ${{ steps.set-py-versions.outputs.py-versions-full }}
py-versions-test: ${{ steps.set-py-versions.outputs.py-versions-test }}
steps:
- uses: actions/checkout@v4
- name: "Check are GCP variables set"
run: "./scripts/ci/ci_check_are_gcp_variables_set.sh"
id: check_gcp_variables
env:
GCP_SA_EMAIL: ${{ secrets.GCP_SA_EMAIL }}
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
GCP_PYTHON_WHEELS_BUCKET: ${{ secrets.GCP_PYTHON_WHEELS_BUCKET }}
GCP_PROJECT_ID: "not-needed-here"
GCP_REGION: "not-needed-here"
GCP_TESTING_BUCKET: "not-needed-here"
- name: Set Python Versions for different environments
id: set-py-versions
run: |
set -xeu
if [ $EVENT_NAME == "pull_request" ]; then
# run highest supported version on pull request.
echo "py-versions-test=${PY_VERSIONS_FULL##* }" >> $GITHUB_OUTPUT
else
# run full version for push and cron jobs.
echo "py-versions-test=$PY_VERSIONS_FULL" >> $GITHUB_OUTPUT
fi
# Output full set of versions so that we can test all languages on pull requests for certain platforms.
echo "py-versions-full=$PY_VERSIONS_FULL" >> $GITHUB_OUTPUT
build_source:
runs-on: ubuntu-latest
name: Build python source distribution
outputs:
is_rc: ${{ steps.is_rc.outputs.is_rc }}
rc_num: ${{ steps.get_rc_version.outputs.RC_NUM }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install python
uses: actions/setup-python@v5
with:
python-version: 3.8
- name: Get tag
id: get_tag
run: |
echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT
- name: Check whether an -RC tag was applied to the commit.
id: is_rc
run: |
echo ${{ steps.get_tag.outputs.TAG }} > temp
OUTPUT=$( if grep -e '-RC.' -q temp; then echo 1; else echo 0; fi)
echo "is_rc=$OUTPUT" >> $GITHUB_OUTPUT
- name: Get RELEASE_VERSION and RC_NUM
if: steps.is_rc.outputs.is_rc == 1
id: get_rc_version
run: |
RC_NUM=$(sed -n "s/^.*-RC\([0-9]*\)/\1/p" temp)
RELEASE_VERSION=$(sed -n "s/^v\(.*\)-RC[0-9]/\1/p" temp)
echo "RC_NUM=$RC_NUM" >> $GITHUB_OUTPUT
echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_OUTPUT
- name: Build source
working-directory: ./sdks/python
run: pip install -U build && python -m build --sdist
- name: Add checksums
working-directory: ./sdks/python/dist
run: |
file=$(ls | grep .tar.gz | head -n 1)
sha512sum $file > ${file}.sha512
- name: Unzip source
working-directory: ./sdks/python
run: tar -xzvf dist/$(ls dist | grep .tar.gz | head -n 1)
- name: Rename source directory
working-directory: ./sdks/python
run: mv $(ls | grep apache-beam) apache-beam-source
- name: Upload source as artifact
# Pinned to v3 because of https://github.com/actions/download-artifact/issues/249
uses: actions/upload-artifact@v3
with:
name: source
path: sdks/python/apache-beam-source
- name: Upload compressed sources as artifacts
# Pinned to v3 because of https://github.com/actions/download-artifact/issues/249
uses: actions/upload-artifact@v3
with:
name: source_zip
path: sdks/python/dist
- name: Clear dist
if: steps.is_rc.outputs.is_rc == 1
working-directory: ./sdks/python
run: |
rm -r ./dist
rm -rd apache-beam-source
- name: Rewrite SDK version to include RC number
if: steps.is_rc.outputs.is_rc == 1
working-directory: ./sdks/python
run: |
RELEASE_VERSION=${{ steps.get_rc_version.outputs.RELEASE_VERSION }}
RC_NUM=${{ steps.get_rc_version.outputs.RC_NUM }}
sed -i -e "s/${RELEASE_VERSION}/${RELEASE_VERSION}rc${RC_NUM}/g" apache_beam/version.py
- name: Build RC source
if: steps.is_rc.outputs.is_rc == 1
working-directory: ./sdks/python
run: pip install -U build && python -m build --sdist
- name: Add RC checksums
if: steps.is_rc.outputs.is_rc == 1
working-directory: ./sdks/python/dist
run: |
file=$(ls | grep .tar.gz | head -n 1)
sha512sum $file > ${file}.sha512
- name: Unzip RC source
if: steps.is_rc.outputs.is_rc == 1
working-directory: ./sdks/python
run: tar -xzvf dist/$(ls dist | grep .tar.gz | head -n 1)
- name: Rename RC source directory
if: steps.is_rc.outputs.is_rc == 1
working-directory: ./sdks/python
run: mv $(ls | grep apache-beam) apache-beam-source-rc
- name: Upload RC source as artifact
if: steps.is_rc.outputs.is_rc == 1
# Pinned to v3 because of https://github.com/actions/download-artifact/issues/249
uses: actions/upload-artifact@v3
with:
name: source_rc${{ steps.get_rc_version.outputs.RC_NUM }}
path: sdks/python/apache-beam-source-rc
- name: Upload compressed RC sources as artifacts
if: steps.is_rc.outputs.is_rc == 1
# Pinned to v3 because of https://github.com/actions/download-artifact/issues/249
uses: actions/upload-artifact@v3
with:
name: source_zip_rc${{ steps.get_rc_version.outputs.RC_NUM }}
path: sdks/python/dist
prepare_gcs:
name: Prepare GCS
needs:
- build_source
- check_env_variables
runs-on: ubuntu-latest
if: needs.check_env_variables.outputs.gcp-variables-set == 'true' && github.event_name != 'pull_request'
steps:
- name: Authenticate on GCP
uses: google-github-actions/setup-gcloud@v0
with:
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
service_account_key: ${{ secrets.GCP_SA_KEY }}
- name: Remove existing files on GCS bucket
run: gsutil rm -r ${{ env.GCP_PATH }} || true
upload_source_to_gcs:
name: Upload python source distribution to GCS bucket
needs:
- prepare_gcs
- check_env_variables
runs-on: ubuntu-latest
if: needs.check_env_variables.outputs.gcp-variables-set == 'true'
steps:
- name: Download compressed sources from artifacts
# Pinned to v3 because of https://github.com/actions/download-artifact/issues/249
uses: actions/download-artifact@v3
with:
name: source_zip
path: source/
- name: Authenticate on GCP
uses: google-github-actions/setup-gcloud@v0
with:
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
service_account_key: ${{ secrets.GCP_SA_KEY }}
- name: Copy sources to GCS bucket
run: gsutil cp -r -a public-read source/* ${{ env.GCP_PATH }}
build_wheels:
name: Build python wheels on ${{matrix.arch}} for ${{ matrix.os_python.os }}
needs:
- check_env_variables
- build_source
env:
CIBW_ARCHS_LINUX: ${{matrix.arch}}
runs-on: ${{ matrix.os_python.os }}
strategy:
matrix:
os_python: [
{"os": "ubuntu-latest", "python": "${{ needs.check_env_variables.outputs.py-versions-full }}" },
{"os": "macos-latest", "python": "${{ needs.check_env_variables.outputs.py-versions-test }}" },
{"os": "windows-latest", "python": "${{ needs.check_env_variables.outputs.py-versions-test }}" },
]
arch: [auto]
include:
- os_python: {"os": "ubuntu-latest", "python": "${{ needs.check_env_variables.outputs.py-versions-test }}" }
arch: aarch64
steps:
- name: Download python source distribution from artifacts
# Pinned to v3 because of https://github.com/actions/download-artifact/issues/249
uses: actions/download-artifact@v3
with:
name: source
path: apache-beam-source
- name: Download Python SDK RC source distribution from artifacts
if: ${{ needs.build_source.outputs.is_rc == 1 }}
# Pinned to v3 because of https://github.com/actions/download-artifact/issues/249
uses: actions/download-artifact@v3
with:
name: source_rc${{ needs.build_source.outputs.rc_num }}
path: apache-beam-source-rc
- name: Install Python
uses: actions/setup-python@v5
with:
python-version: 3.8
- uses: docker/setup-qemu-action@v1
if: ${{matrix.arch == 'aarch64'}}
name: Set up QEMU
- name: Install cibuildwheel
# note: sync cibuildwheel version with gradle task sdks:python:bdistPy* steps
run: pip install cibuildwheel==2.9.0
- name: Build wheel
working-directory: apache-beam-source
env:
CIBW_BUILD: ${{ matrix.os_python.python }}
# TODO: https://github.com/apache/beam/issues/23048
CIBW_SKIP: "*-musllinux_*"
CIBW_ENVIRONMENT: "SETUPTOOLS_USE_DISTUTILS=stdlib"
CIBW_BEFORE_BUILD: pip install cython==0.29.36 numpy --config-settings=setup-args="-Dallow-noblas=true" && pip install --upgrade setuptools
run: cibuildwheel --print-build-identifiers && cibuildwheel --output-dir wheelhouse
shell: bash
- name: install sha512sum on MacOS
if: startsWith(matrix.os_python.os, 'macos')
run: brew install coreutils
- name: Add checksums
working-directory: apache-beam-source/wheelhouse/
run: |
for file in *.whl; do
sha512sum $file > ${file}.sha512
done
shell: bash
- name: Upload wheels as artifacts
# Pinned to v3 because of https://github.com/actions/download-artifact/issues/249
uses: actions/upload-artifact@v3
with:
name: wheelhouse-${{ matrix.os_python.os }}${{ (matrix.arch == 'aarch64' && '-aarch64') || '' }}
path: apache-beam-source/wheelhouse/
- name: Build RC wheels
if: ${{ needs.build_source.outputs.is_rc == 1 }}
working-directory: apache-beam-source-rc
env:
CIBW_BUILD: ${{ matrix.os_python.python }}
# TODO: https://github.com/apache/beam/issues/23048
CIBW_SKIP: "*-musllinux_*"
CIBW_ENVIRONMENT: "SETUPTOOLS_USE_DISTUTILS=stdlib"
CIBW_BEFORE_BUILD: pip install cython==0.29.36 numpy --config-settings=setup-args="-Dallow-noblas=true" && pip install --upgrade setuptools
run: cibuildwheel --print-build-identifiers && cibuildwheel --output-dir wheelhouse
shell: bash
- name: Add RC checksums
if: ${{ needs.build_source.outputs.is_rc == 1 }}
working-directory: apache-beam-source-rc/wheelhouse/
run: |
for file in *.whl; do
sha512sum $file > ${file}.sha512
done
shell: bash
- name: Upload RC wheels as artifacts
if: ${{ needs.build_source.outputs.is_rc == 1 }}
# Pinned to v3 because of https://github.com/actions/download-artifact/issues/249
uses: actions/upload-artifact@v3
with:
name: wheelhouse-rc${{ needs.build_source.outputs.rc_num }}-${{ matrix.os_python.os }}${{ (matrix.arch == 'aarch64' && '-aarch64') || '' }}
path: apache-beam-source-rc/wheelhouse/
upload_wheels_to_gcs:
name: Upload wheels to GCS bucket
needs:
- build_wheels
- check_env_variables
runs-on: ubuntu-latest
if: needs.check_env_variables.outputs.gcp-variables-set == 'true' && github.event_name != 'pull_request'
strategy:
matrix:
os : [ubuntu-latest, macos-latest, windows-latest]
arch: [auto]
include:
- os: "ubuntu-latest"
arch: aarch64
steps:
- name: Download wheels from artifacts
# Pinned to v3 because of https://github.com/actions/download-artifact/issues/249
uses: actions/download-artifact@v3
with:
name: wheelhouse-${{ matrix.os }}${{ (matrix.arch == 'aarch64' && '-aarch64') || '' }}
path: wheelhouse/
- name: Authenticate on GCP
uses: google-github-actions/setup-gcloud@v0
with:
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
service_account_key: ${{ secrets.GCP_SA_KEY }}
- name: Copy wheels to GCS bucket
run: gsutil cp -r -a public-read wheelhouse/* ${{ env.GCP_PATH }}
- name: Create github action information file on GCS bucket
run: |
cat > github_action_info <<EOF
GITHUB_WORKFLOW=$GITHUB_WORKFLOW
GITHUB_RUN_ID=$GITHUB_RUN_ID
GITHUB_RUN_NUMBER=$GITHUB_RUN_NUMBER
GITHUB_ACTION=$GITHUB_ACTION
GITHUB_ACTOR=$GITHUB_ACTOR
GITHUB_REPOSITORY=$GITHUB_REPOSITORY
GITHUB_EVENT_NAME=$GITHUB_EVENT_NAME
GITHUB_SHA=$GITHUB_SHA
GITHUB_REF=$GITHUB_REF
# only for forked repositiories
GITHUB_HEAD_REF=$GITHUB_HEAD_REF
GITHUB_BASE_REF=$GITHUB_BASE_REF
EOF
echo $(cat github_action_info)
gsutil cp -a public-read github_action_info ${{ env.GCP_PATH }}
- name: Upload GitHub event file to GCS bucket
run: gsutil cp -a public-read ${GITHUB_EVENT_PATH} ${{ env.GCP_PATH }}
list_files_on_gcs:
name: List files on Google Cloud Storage Bucket
needs:
- upload_wheels_to_gcs
- check_env_variables
runs-on: ubuntu-latest
if: needs.check_env_variables.outputs.gcp-variables-set == 'true' && github.event_name != 'pull_request'
steps:
- name: Authenticate on GCP
uses: google-github-actions/setup-gcloud@v0
with:
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
service_account_key: ${{ secrets.GCP_SA_KEY }}
- name: List file on Google Cloud Storage Bucket
run: gsutil ls "${{ env.GCP_PATH }}*"
branch_repo_nightly:
permissions:
contents: write
name: Branch repo nightly
needs:
- build_source
- build_wheels
runs-on: ubuntu-latest
timeout-minutes: 60
if: github.repository_owner == 'apache' && github.event_name == 'schedule'
steps:
- name: Checkout code on master branch
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Branch commit
run: |
BRANCH_NAME=${GITHUB_REF##*/}
echo "Updating nightly-${BRANCH_NAME}"
git branch -f nightly-${BRANCH_NAME} HEAD
- name: Push branch
uses: ./.github/actions/github-push-action
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
force: true
branch: nightly-${{ github.ref }}