CI/CD #2489
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: CI/CD | |
on: | |
push: | |
branches: | |
- 'master*' | |
- 'devel-*' | |
tags: | |
- '*' | |
pull_request: | |
schedule: | |
- cron: '0 16 * * *' | |
workflow_dispatch: | |
permissions: {} | |
jobs: | |
dependencies: | |
name: Test python/requirements.txt | |
runs-on: ubuntu-latest | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
- name: Check requirements.txt against requirements-direct.txt | |
run: | | |
(diff -w python/requirements-direct.txt python/requirements.txt || true) | (! grep -e "^<") | |
shell: bash | |
- name: Check for dependency updates | |
continue-on-error: true | |
run: | |
.github/upgrade-pip-packages.sh | |
shell: bash | |
test: | |
name: Test (python-${{ matrix.python-version }}, ${{ matrix.os }}) | |
runs-on: ${{ matrix.os }} | |
strategy: | |
fail-fast: false | |
matrix: | |
os: | |
- macos-11 | |
- macos-12 | |
- macos-latest | |
- ubuntu-20.04 | |
- ubuntu-22.04 | |
- ubuntu-latest | |
- windows-2019 | |
- windows-2022 | |
- windows-latest | |
python-version: ["3.8", "installed"] | |
include: | |
- os: macos-latest | |
python-version: "3.11" | |
- os: ubuntu-latest | |
python-version: "3.11" | |
# installing lxml fails for Python 3.11 on Windows | |
- os: macos-latest | |
python-version: "3.10" | |
- os: ubuntu-latest | |
python-version: "3.10" | |
- os: windows-latest | |
python-version: "3.10" | |
- os: macos-latest | |
python-version: "3.9" | |
- os: ubuntu-latest | |
python-version: "3.9" | |
- os: windows-latest | |
python-version: "3.9" | |
- os: macos-latest | |
python-version: "3.7" | |
- os: ubuntu-latest | |
python-version: "3.7" | |
- os: windows-latest | |
python-version: "3.7" | |
steps: | |
- name: Setup Ubuntu | |
if: startsWith(matrix.os, 'ubuntu') | |
run: | | |
sudo apt-get update | |
sudo apt-get install language-pack-en language-pack-de | |
shell: bash | |
- name: Setup Python | |
if: matrix.python-version != 'installed' | |
uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: Checkout | |
uses: actions/checkout@v3 | |
- name: Detect OS | |
id: os | |
env: | |
OS: ${{ matrix.os }} | |
run: | | |
case "$OS" in | |
ubuntu*) | |
echo "pip-cache=~/.cache/pip" >> $GITHUB_OUTPUT | |
;; | |
macos*) | |
echo "pip-cache=~/Library/Caches/pip" >> $GITHUB_OUTPUT | |
;; | |
windows*) | |
echo "pip-cache=~\\AppData\\Local\\pip\\Cache" >> $GITHUB_OUTPUT | |
;; | |
esac | |
echo "date=$(date +%Y%m%d 2> /dev/null || true)" >> $GITHUB_OUTPUT | |
shell: bash | |
- name: Cache PIP Packages | |
uses: actions/cache@v3 | |
id: cache | |
with: | |
path: ${{ steps.os.outputs.pip-cache }} | |
key: ${{ matrix.os }}-pip-test-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/constraints.txt') }}-${{ steps.os.outputs.date }} | |
restore-keys: | | |
${{ matrix.os }}-pip-test-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt', '**/constraints.txt') }}- | |
${{ matrix.os }}-pip-test-${{ matrix.python-version }}- | |
${{ matrix.os }}-pip-test- | |
- name: Install Python dependencies | |
run: | | |
python3 -V | |
python3 -m pip freeze | sort | |
python3 -m pip cache info || true | |
python3 -m pip cache list || true | |
python3 -m pip install --upgrade --force pip wheel | |
python3 -m pip install --force -r python/requirements.txt | |
python3 -m pip install --force -r python/test/requirements.txt -c python/test/constraints.txt | |
python3 -m pip freeze | sort | |
python3 -m pip cache info || true | |
python3 -m pip cache list || true | |
shell: bash | |
- name: Update expectation files | |
id: changes | |
continue-on-error: true | |
run: | | |
python/test/files/update_expectations.sh | |
git status | |
if ! git diff --exit-code || [[ $(git ls-files -o --exclude-standard | wc -l) -gt 0 ]] | |
then | |
zip changes.zip $(git diff --name-only) $(git ls-files -o --exclude-standard) | |
exit 1 | |
fi | |
shell: bash | |
- name: Upload changed expectation files | |
if: steps.changes.outcome == 'failure' | |
uses: actions/upload-artifact@v3 | |
with: | |
name: Changed expectations | |
path: changed-expectations.zip | |
- name: PyTest | |
env: | |
PYTHONPATH: .. | |
run: | | |
cd python/test | |
python3 -m pytest --capture=tee-sys --continue-on-collection-errors --junit-xml ../../test-results/pytest.xml | |
shell: bash | |
- name: PyTest (EST) | |
env: | |
TZ: US/Eastern | |
LANG: "en_US.UTF-8" | |
PYTHONPATH: .. | |
run: | | |
cd python/test | |
python3 -m pytest --capture=tee-sys --continue-on-collection-errors --junit-xml ../../test-results/pytest-est.xml | |
shell: bash | |
- name: PyTest (CET) | |
env: | |
TZ: Europe/Berlin | |
LANG: "de_DE.UTF-8" | |
PYTHONPATH: .. | |
run: | | |
cd python/test | |
python3 -m pytest --capture=tee-sys --continue-on-collection-errors --junit-xml ../../test-results/pytest-cet.xml | |
shell: bash | |
- name: Upload Test Results | |
if: always() | |
uses: actions/upload-artifact@v3 | |
with: | |
name: Test Results (python-${{ matrix.python-version }}, ${{ matrix.os }}) | |
path: | | |
test-results/*.xml | |
unit-test-results.json | |
publish-dockerfile: | |
name: Publish Test Results (Dockerfile) | |
needs: test | |
# we run the action from this branch whenever we can (when it runs in our repo's context) | |
if: > | |
always() && | |
github.event.sender.login != 'dependabot[bot]' && | |
( github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository ) | |
runs-on: ubuntu-latest | |
permissions: | |
checks: write | |
pull-requests: write | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
- name: Download Artifacts | |
uses: actions/download-artifact@v3 | |
with: | |
path: artifacts | |
- name: Prepare publish action from this branch | |
run: | | |
sed --in-place "s/image: .*/image: 'Dockerfile'/" action.yml | |
shell: bash | |
- name: Publish Test Results | |
id: test-results | |
uses: ./ | |
with: | |
check_name: Test Results (Dockerfile) | |
files: "artifacts/**/*.xml" | |
json_file: "tests.json" | |
json_suite_details: true | |
json_test_case_results: true | |
report_suite_logs: "any" | |
log_level: DEBUG | |
- name: JSON output | |
uses: ./misc/action/json-output | |
with: | |
json: '${{ steps.test-results.outputs.json }}' | |
json_file: 'tests.json' | |
publish-docker-image: | |
name: Publish Test Results (Docker Image) | |
needs: test | |
# we run the action from this branch whenever we can (when it runs in our repo's context) | |
if: > | |
always() && | |
github.event.sender.login != 'dependabot[bot]' && | |
( github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository ) | |
runs-on: ubuntu-latest | |
permissions: | |
checks: write | |
pull-requests: write | |
security-events: write | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
- name: Set up QEMU | |
uses: docker/setup-qemu-action@v2 | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v2 | |
- name: Build Docker image | |
id: build | |
uses: docker/build-push-action@v4 | |
with: | |
load: true | |
push: false | |
tags: enricomi/publish-unit-test-result-action:latest | |
outputs: type=docker | |
- name: Download Artifacts | |
uses: actions/download-artifact@v3 | |
with: | |
path: artifacts | |
- name: Publish Test Results | |
id: test-results | |
if: always() | |
env: | |
INPUT_GITHUB_TOKEN: ${{ github.token }} | |
INPUT_CHECK_NAME: Test Results (Docker Image) | |
INPUT_FILES: "artifacts/**/*.xml" | |
INPUT_JSON_FILE: "tests.json" | |
INPUT_JSON_SUITE_DETAILS: true | |
INPUT_JSON_TEST_CASE_RESULTS: true | |
INPUT_REPORT_SUITE_LOGS: "any" | |
run: | | |
docker run \ | |
--workdir $GITHUB_WORKSPACE \ | |
--rm \ | |
-e "INPUT_CHECK_NAME" \ | |
-e "INPUT_JSON_FILE" \ | |
-e "INPUT_JSON_SUITE_DETAILS" \ | |
-e "INPUT_JSON_TEST_CASE_RESULTS" \ | |
-e "INPUT_LOG_LEVEL" \ | |
-e "INPUT_ROOT_LOG_LEVEL" \ | |
-e "INPUT_GITHUB_TOKEN" \ | |
-e "INPUT_GITHUB_TOKEN_ACTOR" \ | |
-e "INPUT_GITHUB_RETRIES" \ | |
-e "INPUT_COMMIT" \ | |
-e "INPUT_COMMENT_TITLE" \ | |
-e "INPUT_COMMENT_MODE" \ | |
-e "INPUT_FAIL_ON" \ | |
-e "INPUT_ACTION_FAIL" \ | |
-e "INPUT_ACTION_FAIL_ON_INCONCLUSIVE" \ | |
-e "INPUT_FILES" \ | |
-e "INPUT_JUNIT_FILES" \ | |
-e "INPUT_NUNIT_FILES" \ | |
-e "INPUT_XUNIT_FILES" \ | |
-e "INPUT_TRX_FILES" \ | |
-e "INPUT_TIME_UNIT" \ | |
-e "INPUT_REPORT_INDIVIDUAL_RUNS" \ | |
-e "INPUT_REPORT_SUITE_LOGS" \ | |
-e "INPUT_DEDUPLICATE_CLASSES_BY_FILE_NAME" \ | |
-e "INPUT_LARGE_FILES" \ | |
-e "INPUT_IGNORE_RUNS" \ | |
-e "INPUT_JOB_SUMMARY" \ | |
-e "INPUT_COMPARE_TO_EARLIER_COMMIT" \ | |
-e "INPUT_PULL_REQUEST_BUILD" \ | |
-e "INPUT_EVENT_FILE" \ | |
-e "INPUT_EVENT_NAME" \ | |
-e "INPUT_TEST_CHANGES_LIMIT" \ | |
-e "INPUT_CHECK_RUN_ANNOTATIONS" \ | |
-e "INPUT_CHECK_RUN_ANNOTATIONS_BRANCH" \ | |
-e "INPUT_SECONDS_BETWEEN_GITHUB_READS" \ | |
-e "INPUT_SECONDS_BETWEEN_GITHUB_WRITES" \ | |
-e "INPUT_SECONDARY_RATE_LIMIT_WAIT_SECONDS" \ | |
-e "INPUT_JSON_THOUSANDS_SEPARATOR" \ | |
-e "INPUT_SEARCH_PULL_REQUESTS" \ | |
-e "HOME" \ | |
-e "GITHUB_JOB" \ | |
-e "GITHUB_REF" \ | |
-e "GITHUB_SHA" \ | |
-e "GITHUB_REPOSITORY" \ | |
-e "GITHUB_REPOSITORY_OWNER" \ | |
-e "GITHUB_RUN_ID" \ | |
-e "GITHUB_RUN_NUMBER" \ | |
-e "GITHUB_RETENTION_DAYS" \ | |
-e "GITHUB_RUN_ATTEMPT" \ | |
-e "GITHUB_ACTOR" \ | |
-e "GITHUB_TRIGGERING_ACTOR" \ | |
-e "GITHUB_WORKFLOW" \ | |
-e "GITHUB_HEAD_REF" \ | |
-e "GITHUB_BASE_REF" \ | |
-e "GITHUB_EVENT_NAME" \ | |
-e "GITHUB_SERVER_URL" \ | |
-e "GITHUB_API_URL" \ | |
-e "GITHUB_GRAPHQL_URL" \ | |
-e "GITHUB_REF_NAME" \ | |
-e "GITHUB_REF_PROTECTED" \ | |
-e "GITHUB_REF_TYPE" \ | |
-e "GITHUB_WORKSPACE" \ | |
-e "GITHUB_ACTION" \ | |
-e "GITHUB_EVENT_PATH" \ | |
-e "GITHUB_ACTION_REPOSITORY" \ | |
-e "GITHUB_ACTION_REF" \ | |
-e "GITHUB_PATH" \ | |
-e "GITHUB_ENV" \ | |
-e "GITHUB_STEP_SUMMARY" \ | |
-e "GITHUB_STATE" \ | |
-e "GITHUB_OUTPUT" \ | |
-e "RUNNER_OS" \ | |
-e "RUNNER_ARCH" \ | |
-e "RUNNER_NAME" \ | |
-e "RUNNER_TOOL_CACHE" \ | |
-e "RUNNER_TEMP" \ | |
-e "RUNNER_WORKSPACE" \ | |
-e "ACTIONS_RUNTIME_URL" \ | |
-e "ACTIONS_RUNTIME_TOKEN" \ | |
-e "ACTIONS_CACHE_URL" \ | |
-e GITHUB_ACTIONS=true \ | |
-e CI=true \ | |
-v "$RUNNER_TEMP":"$RUNNER_TEMP" \ | |
-v "/var/run/docker.sock":"/var/run/docker.sock" \ | |
-v "/home/runner/work/_temp/_github_home":"/github/home" \ | |
-v "/home/runner/work/_temp/_github_workflow":"/github/workflow" \ | |
-v "/home/runner/work/_temp/_runner_file_commands":"/github/file_commands" \ | |
-v "/home/runner/work/publish-unit-test-result-action/publish-unit-test-result-action":"$GITHUB_WORKSPACE" \ | |
enricomi/publish-unit-test-result-action:latest | |
shell: bash | |
- name: JSON output | |
uses: ./misc/action/json-output | |
with: | |
json: '${{ steps.test-results.outputs.json }}' | |
json_file: 'tests.json' | |
- name: Scan for vulnerabilities | |
id: scan | |
uses: crazy-max/ghaction-container-scan@v2 | |
with: | |
image: enricomi/publish-unit-test-result-action:latest | |
dockerfile: ./Dockerfile | |
annotations: true | |
- name: Upload SARIF artifact | |
uses: actions/upload-artifact@v3 | |
with: | |
name: SARIF | |
path: ${{ steps.scan.outputs.sarif }} | |
- name: Upload SARIF file | |
if: always() && steps.scan.outputs.sarif != '' | |
uses: github/codeql-action/upload-sarif@v2 | |
with: | |
sarif_file: ${{ steps.scan.outputs.sarif }} | |
publish-composite: | |
name: Publish Test Results (${{ matrix.os-label }} python ${{ matrix.python }}) | |
needs: test | |
# we run the action from this branch whenever we can (when it runs in our repo's context) | |
if: > | |
always() && | |
github.event.sender.login != 'dependabot[bot]' && | |
( github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository ) | |
runs-on: ${{ matrix.os }} | |
permissions: | |
checks: write | |
pull-requests: write | |
strategy: | |
fail-fast: false | |
max-parallel: 3 | |
matrix: | |
# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources | |
# test *-latest and newer (because newer eventually become 'latest' and should be tested to work before that) | |
include: | |
- os: macos-latest | |
os-label: macOS | |
python: "3.8" | |
- os: macos-latest | |
os-label: macOS | |
python: "installed" | |
- os: macos-11 | |
os-label: macOS 11 | |
python: "installed" | |
- os: ubuntu-latest | |
os-label: Linux | |
python: "3.8" | |
- os: ubuntu-latest | |
os-label: Linux | |
python: "installed" | |
- os: ubuntu-20.04 | |
os-label: Linux 20.04 | |
python: "installed" | |
- os: windows-latest | |
os-label: Windows | |
python: "installed" | |
- os: windows-2019 | |
os-label: Windows 2019 | |
python: "installed" | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
- name: Setup Python | |
if: matrix.python != 'installed' | |
uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python }} | |
- name: Download Artifacts | |
uses: actions/download-artifact@v3 | |
with: | |
path: artifacts | |
- name: Publish Test Results | |
id: test-results | |
uses: ./composite | |
with: | |
check_name: Test Results (${{ matrix.os-label }} python ${{ matrix.python }}) | |
files: | | |
artifacts/**/*.xml | |
artifacts\**\*.xml | |
json_file: "tests.json" | |
json_suite_details: true | |
json_test_case_results: true | |
report_suite_logs: "any" | |
- name: JSON output | |
uses: ./misc/action/json-output | |
with: | |
json: '${{ steps.test-results.outputs.json }}' | |
json_file: 'tests.json' | |
publish-test-files: | |
name: Publish Test Files | |
# does not really depend on 'tests' but can be executed together with other publish tasks just for good taste | |
needs: test | |
# we run the action from this branch whenever we can (when it runs in our repo's context) | |
if: > | |
always() && | |
github.event.sender.login != 'dependabot[bot]' && | |
( github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository ) | |
runs-on: ubuntu-latest | |
permissions: | |
checks: write | |
pull-requests: write | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
- name: Copy test result files | |
run: cp -rv python/test/files test-files | |
shell: bash | |
- name: Prepare publish action from this branch | |
run: | | |
sed --in-place "s/image: .*/image: 'Dockerfile'/" action.yml | |
shell: bash | |
- name: Publish Test Results | |
id: test-results | |
uses: ./ | |
with: | |
check_name: Test Results (Test Files) | |
fail_on: nothing | |
files: | | |
test-files/**/*.xml | |
test-files/**/*.trx | |
test-files/**/*.json | |
junit_files: "test-files/junit-xml/**/*.xml" | |
nunit_files: "test-files/nunit/**/*.xml" | |
xunit_files: "test-files/xunit/**/*.xml" | |
trx_files: "test-files/trx/**/*.trx" | |
json_file: "tests.json" | |
json_suite_details: true | |
json_test_case_results: true | |
report_suite_logs: "any" | |
log_level: DEBUG | |
- name: JSON output | |
uses: ./misc/action/json-output | |
with: | |
json: '${{ steps.test-results.outputs.json }}' | |
json_file: 'tests.json' | |
publish-test-file: | |
name: Publish Test File | |
# does not really depend on 'tests' but can be executed together with other publish tasks just for good taste | |
needs: test | |
# we run the action from this branch whenever we can (when it runs in our repo's context) | |
if: > | |
always() && | |
github.event.sender.login != 'dependabot[bot]' && | |
( github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository ) | |
runs-on: ubuntu-latest | |
permissions: | |
checks: write | |
pull-requests: write | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
- name: Copy test junit xml files | |
run: cp -rv python/test/files/junit-xml test-files | |
shell: bash | |
- name: Prepare publish action from this branch | |
run: | | |
sed --in-place "s/image: .*/image: 'Dockerfile'/" action.yml | |
shell: bash | |
- name: Publish Test Results | |
id: test-results | |
uses: ./ | |
with: | |
check_name: Test Results (Test File) | |
fail_on: nothing | |
files: "test-files/pytest/junit.gloo.standalone.xml" | |
json_file: "tests.json" | |
json_suite_details: true | |
json_test_case_results: true | |
report_suite_logs: "any" | |
log_level: DEBUG | |
- name: JSON output | |
uses: ./misc/action/json-output | |
with: | |
json: '${{ steps.test-results.outputs.json }}' | |
json_file: 'tests.json' | |
config-deploy: | |
name: Configure Deployment | |
needs: test | |
# do not build or deploy on forked repositories | |
if: github.repository_owner == 'EnricoMi' | |
runs-on: ubuntu-latest | |
outputs: | |
image: ${{ steps.action.outputs.image }} | |
image-exists: ${{ steps.image.outputs.exists }} | |
image-version: ${{ steps.action.outputs.version }} | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
- name: Extract action image and version | |
# we deploy from a specific commit on master (the one that mentions a new version the first time) | |
# so we need to tell docker/metadata-action to extract docker tags from that version | |
id: action | |
run: | | |
image=$(grep -A 10 "^runs:" action.yml | grep -E "^\s+image:\s" | sed -E -e "s/^\s+image:\s*'//" -e "s/docker:\/\///" -e "s/'\s*$//") | |
version=$(cut -d : -f 2 <<< "$image") | |
echo "image=$image" >>$GITHUB_OUTPUT | |
echo "version=$version" >>$GITHUB_OUTPUT | |
shell: bash | |
- name: Check action image existence | |
id: image | |
env: | |
DOCKER_CLI_EXPERIMENTAL: enabled | |
run: | | |
if docker manifest inspect '${{ steps.action.outputs.image }}' | |
then | |
echo "exists=true" >>$GITHUB_OUTPUT | |
fi | |
shell: bash | |
deploy: | |
name: Deploy to GitHub | |
needs: [test, publish-dockerfile, publish-docker-image, publish-composite, publish-test-file, publish-test-files, config-deploy] | |
# do not build or deploy on forked repositories | |
if: github.repository_owner == 'EnricoMi' | |
runs-on: ubuntu-latest | |
steps: | |
- name: Docker meta | |
id: docker-meta | |
uses: docker/metadata-action@v4 | |
with: | |
images: ghcr.io/EnricoMi/publish-unit-test-result-action | |
flavor: | | |
latest=false | |
prefix=v | |
tags: | | |
type=sha | |
type=ref,event=tag | |
type=semver,pattern={{major}},value=${{ needs.config-deploy.outputs.image-version }} | |
type=semver,pattern={{major}}.{{minor}},value=${{ needs.config-deploy.outputs.image-version }} | |
type=semver,pattern={{version}},value=${{ needs.config-deploy.outputs.image-version }} | |
- name: Set up QEMU | |
uses: docker/setup-qemu-action@v2 | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v2 | |
- name: Login to GitHub Container Registry | |
uses: docker/login-action@v2 | |
with: | |
registry: ghcr.io | |
username: ${{ github.repository_owner }} | |
password: ${{ secrets.CR_PAT }} | |
- name: Build and push Docker image | |
uses: docker/build-push-action@v4 | |
with: | |
tags: ${{ steps.docker-meta.outputs.tags }} | |
labels: ${{ steps.docker-meta.outputs.labels }} | |
pull: true | |
# deploy image actions from commits pushed to master and | |
# deploy Dockerfile actions from pushed version tags (no major versions) | |
push: | | |
${{ | |
github.event_name == 'push' && ( | |
needs.config-deploy.outputs.image != 'Dockerfile' && startsWith(github.ref, 'refs/heads/master') && needs.config-deploy.outputs.image-exists != 'true' || | |
needs.config-deploy.outputs.image == 'Dockerfile' && startsWith(github.ref, 'refs/tags/v') && contains(github.ref, '.') | |
) | |
}} | |
event_file: | |
name: "Event File" | |
runs-on: ubuntu-latest | |
steps: | |
- name: Upload | |
uses: actions/upload-artifact@v3 | |
with: | |
name: Event File | |
path: ${{ github.event_path }} |