Building and testing #168
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Test | |
| run-name: Building and testing | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| release: | |
| required: false | |
| type: boolean | |
| default: false | |
| release-to-pypi: | |
| required: false | |
| type: boolean | |
| default: false | |
| push: | |
| jobs: | |
| params: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| ocm_repository: ${{ steps.params.outputs.ocm_repository }} | |
| oci_repository: ${{ steps.params.outputs.oci_repository }} | |
| oci_platforms: ${{ steps.params.outputs.oci_platforms }} | |
| steps: | |
| - name: params | |
| id: params | |
| run: | | |
| repo_base=europe-docker.pkg.dev/gardener-project | |
| snapshots_repo="${repo_base}/snapshots" | |
| releases_repo="${repo_base}/releases" | |
| if ${{ inputs.release || false }}; then | |
| ocm_repository=${releases_repo} | |
| oci_repository=${releases_repo} | |
| else | |
| ocm_repository=${snapshots_repo} | |
| oci_repository=${snapshots_repo} | |
| fi | |
| echo "ocm_repository=${ocm_repository}" >> "${GITHUB_OUTPUT}" | |
| echo "oci_repository=${oci_repository}" >> "${GITHUB_OUTPUT}" | |
| echo "oci_platforms=linux/amd64,linux/arm64" >> "${GITHUB_OUTPUT}" | |
| version: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| effective_version: ${{ steps.version.outputs.effective_version }} | |
| next_version: ${{ steps.version.outputs.next_version }} | |
| repo_version: ${{ steps.version.outputs.repo_version }} | |
| setuptools_version: ${{ steps.version.outputs.setuptools_version }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: calculate-effective-version | |
| id: version | |
| run: | | |
| src_version=$(.ci/read-version) | |
| commit=${{ github.sha }} | |
| echo "commit-digest: ${commit}" | |
| major="$(echo ${src_version} | cut -d. -f1)" | |
| minor="$(echo ${src_version} | cut -d. -f2)" | |
| patch="$(echo ${src_version} | cut -d. -f3 | cut -d- -f1)" | |
| if ${{ inputs.release || false }}; then | |
| effective_version=${major}.${minor}.${patch} | |
| setuptools_version=${effective_version} | |
| # hardcode to bumping "minor" for now | |
| next_minor=$(( $minor + 1 )) | |
| next_version="${major}.${next_minor}.${patch}-dev" | |
| echo "next_version=${next_version}" >> "${GITHUB_OUTPUT}" | |
| echo "next_version=${next_version}" | |
| else | |
| effective_version=${major}.${minor}.${patch}-${commit} | |
| setuptools_version=${src_version} | |
| fi | |
| echo "effective-version: ${effective_version}" | |
| echo "effective_version=${effective_version}" >> "${GITHUB_OUTPUT}" | |
| echo "repo_version=${src_version}" >> "${GITHUB_OUTPUT}" | |
| echo "setuptools_version=${setuptools_version}" >> "${GITHUB_OUTPUT}" | |
| package: | |
| runs-on: ubuntu-latest | |
| environment: build | |
| outputs: | |
| ocm_resources: ${{ steps.package.outputs.ocm_resources }} | |
| needs: version | |
| container: | |
| image: python:alpine | |
| steps: | |
| - name: Install git, setuptools, node (for upload-artifact) | |
| run: | | |
| apk add \ | |
| bash \ | |
| file \ | |
| git \ | |
| nodejs \ | |
| xz | |
| pip3 install --root-user-action ignore \ | |
| setuptools \ | |
| pyyaml | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 # required by upload-artifact / `act` | |
| # see: https://github.com/nektos/act/issues/973 | |
| - name: create distribution package | |
| id: package | |
| run: | | |
| set -eu | |
| version=${{ needs.version.outputs.setuptools_version }} | |
| echo "version: ${version}" | |
| echo "${version}" | .ci/write-version | |
| pkg_dir=dist | |
| mkdir -p ${pkg_dir} | |
| pkg_dir="$(readlink -f dist)" | |
| echo "pkgdir: ${pkg_dir}" | |
| for path in \ | |
| setup.py \ | |
| setup.oci.py \ | |
| setup.ocm.py \ | |
| setup.whd.py \ | |
| ; do | |
| echo "building distribution package from ${path}" | |
| python3 ${path} \ | |
| bdist_wheel \ | |
| --dist-dir ${pkg_dir} | |
| set -x | |
| rm -rf build | |
| set +x | |
| done | |
| # special-case: cli-package (need to chdir in order to not confuse setuptools) | |
| ( | |
| cd cli | |
| python3 setup.py \ | |
| bdist_wheel \ | |
| --dist-dir ${pkg_dir} | |
| ) | |
| echo "Built packages" | |
| ls "${pkg_dir}" | |
| blobs_dir="${pkg_dir}/blobs.d" | |
| mkdir ${blobs_dir} | |
| resources_file=resources.yaml | |
| for package in \ | |
| gardener-cicd-whd \ | |
| gardener-cicd-cli \ | |
| gardener-cicd-libs \ | |
| gardener-oci \ | |
| gardener-ocm \ | |
| ; do | |
| path="$(echo $package | tr - _)-*" | |
| prefix=dist | |
| access_type='localBlob' | |
| outf="${pkg_dir}/${package}.tar.gz" | |
| tar cJf ${outf} -C ${pkg_dir} $(cd ${pkg_dir}; ls ${path}) | |
| mimetype=$(file -i ${outf} | cut -d: -f2 | cut -d' ' -f2-) | |
| leng=$(stat -c"%s" ${outf}) | |
| digest="$(sha256sum ${outf} | cut -d' ' -f1)" | |
| echo "\ | |
| - name: ${package} | |
| version: ${version} | |
| type: ${mimetype} | |
| relation: local | |
| access: | |
| type: localBlob | |
| localReference: sha256:${digest} | |
| size: ${leng} | |
| mediaType: ${mimetype}" \ | |
| >> ${resources_file} | |
| mv ${outf} ${blobs_dir}/${digest} | |
| done | |
| cp "${resources_file}" dist/ocm_resources.yaml | |
| echo "ocm_resources=dist/ocm_resources.yaml" >> "${GITHUB_OUTPUT}" | |
| find "${pkg_dir}" | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: distribution-packages | |
| path: dist/ | |
| component_descriptor: | |
| name: OCM + Release (only on manual triggering) | |
| runs-on: ubuntu-latest | |
| container: | |
| image: python:alpine | |
| permissions: | |
| contents: write | |
| id-token: write | |
| needs: | |
| - version | |
| - params | |
| - package | |
| - images | |
| - lint | |
| - unittests | |
| - documentation | |
| outputs: | |
| release_commit_digest: ${{ steps.releasecommit.commitdigest }} | |
| ocm_repository: ${{ steps.params.outputs.ocm_repository }} | |
| steps: | |
| - name: Install Packages | |
| run: | | |
| apk add --no-cache \ | |
| bash \ | |
| git | |
| git config --global --add safe.directory /__w/cc-utils/cc-utils | |
| - uses: actions/checkout@v4 | |
| - name: Retrieve Distribution Packages | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: distribution-packages | |
| path: /tmp/dist | |
| - name: Retrieve Linting Logs | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: linting-logs # targetpath: bandit.tar.gz | |
| path: /tmp/linting-logs | |
| - name: Retrieve Documentation | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: documentation | |
| path: /tmp/documentation-out.d | |
| - name: Create Release and Bump-Commits | |
| id: releasecommit | |
| if: ${{ inputs.release }} | |
| run: | | |
| echo "gha-creds*" >> .git/info/exclude | |
| git config --global --add safe.directory /__w/cc-utils/cc-utils | |
| git config user.name 'Gardener-CICD Bot' | |
| git config user.email 'gardener.ci.user@gmail.com' | |
| effective_version=${{ needs.version.outputs.effective_version }} | |
| echo "${effective_version}" | .ci/write-version | |
| git add . | |
| git commit -m "Release ${effective_version}" | |
| git show | |
| commit_digest=$(git rev-parse @) | |
| echo "release_commit_digest=${commit_digest}" >> "${GITHUB_OUTPUT}" | |
| tgt_ref="refs/tags/${effective_version}" | |
| echo "pushing release-commit ${commit_digest} to ${tgt_ref}" | |
| git push origin "@:${tgt_ref}" | |
| next_version=${{ needs.version.outputs.next_version }} | |
| echo "next version: ${next_version}" | |
| git reset --hard @~ | |
| echo "${next_version}" | .ci/write-version | |
| git add . | |
| git commit -m"Prepare next Dev-Cycle" | |
| git pull --rebase | |
| git push origin | |
| - name: GAR-Auth | |
| id: auth | |
| uses: ./.github/actions/gar-auth | |
| - name: component-descriptor | |
| run: | | |
| set -eu | |
| pip3 install --no-cache \ | |
| aiohttp \ | |
| dacite \ | |
| deprecated \ | |
| pyaml \ | |
| python-dateutil \ | |
| requests \ | |
| www-authenticate \ | |
| &>/dev/null | |
| python -c "import oci" | |
| version=${{ needs.version.outputs.effective_version }} | |
| ocm_repo=${{ needs.params.outputs.ocm_repository }} | |
| echo "generating component-descriptor" | |
| python3 -m ocm create \ | |
| --name github.com/${{ github.repository }} \ | |
| --version ${version} \ | |
| --ocm-repo ${ocm_repo} \ | |
| --provider sap-se \ | |
| --label '{ | |
| "name": "cloud.gardener.cnudie/responsibles", | |
| "value": [ | |
| { | |
| "type": "githubTeam", | |
| "teamname": "gardener/ci-maintainers", | |
| "github_hostname": "github.com" | |
| } | |
| ] | |
| }' \ | |
| > component-descriptor.yaml | |
| if ${{ inputs.release || false }}; then | |
| commit_digest=${{ steps.releasecommit.outputs.release_commit_digest || '' }} | |
| else | |
| commit_digest=${{ github.sha }} | |
| fi | |
| echo "adding main source" | |
| echo "\ | |
| name: main-source | |
| version: ${version} | |
| type: git | |
| labels: | |
| - name: cloud.gardener/cicd/source | |
| value: | |
| repository-classification: main | |
| access: | |
| type: github | |
| repoUrl: github.com/${{ github.repository }} | |
| version: ${version} | |
| commit: ${commit_digest} | |
| ref: ${{ github.ref }} | |
| " \ | |
| | python3 -m ocm append source \ | |
| --file component-descriptor.yaml | |
| echo "adding resources" | |
| ocm_resources=${{ needs.package.outputs.ocm_resources }} | |
| echo "ocm-resources-file: /tmp/${ocm_resources}" | |
| cat "/tmp/${ocm_resources}" | \ | |
| python3 -m ocm append resource \ | |
| --file component-descriptor.yaml | |
| echo "adding linting-evidence resource" | |
| linting_evidence=/tmp/linting-logs/bandit.tar.gz | |
| linting_digest=$(sha256sum ${linting_evidence} | cut -d' ' -f1) | |
| cp ${linting_evidence} /tmp/dist/blobs.d/${linting_digest} | |
| cat << EOF > linting_evidence.ocm-resource | |
| name: sast-linting-evidence | |
| version: ${version} | |
| type: application/gzip | |
| relation: local | |
| access: | |
| type: localBlob | |
| localReference: sha256:${linting_digest} | |
| size: $(stat -c"%s" ${linting_evidence}) | |
| labels: | |
| - name: gardener.cloud/purposes | |
| value: | |
| - lint | |
| - sast | |
| - pybandit | |
| - name: gardener.cloud/comment | |
| value: | | |
| we use bandit (linter) for SAST scans | |
| see: https://bandit.readthedocs.io/en/latest/ | |
| EOF | |
| cat linting_evidence.ocm-resource | \ | |
| python3 -m ocm append resource \ | |
| --file component-descriptor.yaml | |
| echo "${{ needs.images.outputs.ocm_resources }}" | base64 -d > oci_ocm_resources.yaml | |
| cat oci_ocm_resources.yaml | \ | |
| python3 -m ocm append resource \ | |
| --file component-descriptor.yaml | |
| echo "component-descriptor to be uploaded:" | |
| cat component-descriptor.yaml | |
| echo "uploading component-descriptor" | |
| python -m ocm upload \ | |
| --file component-descriptor.yaml \ | |
| --blobs-dir /tmp/dist/blobs.d | |
| - name: Publish Documentation | |
| if: ${{ inputs.release }} | |
| run: | | |
| release_commit_digest=${{ steps.releasecommit.outputs.release_commit_digest || '' }} | |
| git fetch origin gh-pages | |
| git checkout gh-pages | |
| git clean -dfx | |
| git status | |
| echo "let's hope our worktree is clean" | |
| tar c -C /tmp/documentation-out.d . | tar x -C. | |
| git status | |
| if [ -z "$(git status --porcelain)" ]; then | |
| echo "no changes in documentation - no need to update documentation" | |
| exit 0 | |
| fi | |
| git add -A | |
| git commit -m "update documentation" | |
| git push origin refs/heads/gh-pages | |
| git checkout master # needed for post-gar | |
| lint: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - package | |
| container: | |
| image: python:alpine | |
| steps: | |
| - name: install git | |
| run: | | |
| apk add --no-cache git | |
| - uses: actions/checkout@v4 | |
| - name: Retrieve Distribution Packages | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: distribution-packages | |
| path: /tmp/dist | |
| - name: lint | |
| run: | | |
| echo "install dependencies for python-packages" | |
| if ! apk add --no-cache $(cat gardener-cicd-libs.apk-packages) >/tmp/apk.log; then | |
| echo "error while trying to install apk-packages:" | |
| cat /tmp/apk.log | |
| exit 1 | |
| fi | |
| echo "installing linters" | |
| if ! pip3 install --upgrade --break-system-packages \ | |
| --find-links /tmp/dist \ | |
| gardener-cicd-libs \ | |
| gardener-cicd-cli \ | |
| gardener-oci \ | |
| gardener-ocm \ | |
| bandit \ | |
| flake8 \ | |
| setuptools \ | |
| pylama \ | |
| pylint \ | |
| > /tmp/pip3-install.log; then | |
| echo "error while trying to install packages:" | |
| cat /tmp/pip3-install.log | |
| fi | |
| echo "running linters" | |
| bandit_logfile=bandit.log | |
| bandit_evidence=bandit.tar.gz | |
| bandit_extra_args="-f txt -o ${bandit_logfile}" \ | |
| .ci/lint | |
| # pass bandit.log + used cfg (pyproject.toml) as evidence | |
| tar czf $bandit_evidence $bandit_logfile pyproject.toml | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: linting-logs | |
| path: bandit.tar.gz | |
| unittests: | |
| needs: | |
| - package | |
| permissions: | |
| contents: read | |
| runs-on: ubuntu-latest | |
| container: | |
| image: python:alpine | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Retrieve Distribution Packages | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: distribution-packages | |
| path: /tmp/dist | |
| - name: run-tests | |
| run: | | |
| set -eu | |
| echo "install dependencies for python-packages" | |
| apk add --no-cache $(cat gardener-cicd-libs.apk-packages) | |
| echo "install packages" | |
| if ! pip3 install --break-system-packages \ | |
| --find-links /tmp/dist \ | |
| gardener-cicd-libs \ | |
| gardener-cicd-cli \ | |
| gardener-oci \ | |
| gardener-ocm \ | |
| pytest \ | |
| setuptools \ | |
| > /tmp/pip3-install.log; then | |
| echo "error while trying to install packages:" | |
| cat /tmp/pip3-install.log | |
| fi | |
| pip3 list | |
| echo "running tests" | |
| mkdir /tmp/fake-cfg.d | |
| touch /tmp/fake-cfg.d/config_types.yaml | |
| export CC_CONFIG_DIR=/tmp/fake-cfg.d | |
| .ci/test | |
| pypi: | |
| if: ${{ inputs.release-to-pypi }} | |
| runs-on: ubuntu-latest | |
| name: Publish to PYPI | |
| needs: | |
| - version | |
| - package | |
| - params | |
| - lint | |
| - unittests | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - name: Retrieve Distribution Packages | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: distribution-packages | |
| path: /tmp/dist | |
| - name: prepare build-filesystem | |
| id: prepare | |
| run: | | |
| cp -r /tmp/dist . | |
| ls -lta dist/ | |
| rm -rf dist/blobs.d dist/ocm_resources.yaml | |
| ls -lta dist/ | |
| - name: publish to pypi | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| images: | |
| name: Build OCI Images | |
| needs: | |
| - version | |
| - package | |
| - params | |
| outputs: | |
| oci_image_ref: ${{ steps.prepare.outputs.oci_image_ref }} | |
| ocm_resources: ${{ steps.prepare.outputs.ocm_resources }} | |
| runs-on: ubuntu-latest | |
| environment: build | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Docker-Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: GAR-Auth | |
| id: auth | |
| uses: ./.github/actions/gar-auth | |
| - name: Retrieve Distribution Packages | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: distribution-packages | |
| path: /tmp/dist | |
| - name: prepare build-filesystem | |
| id: prepare | |
| run: | | |
| cp -r /tmp/dist . | |
| ls -lta | |
| setuptools_version=${{ needs.version.outputs.setuptools_version }} | |
| # workaround: set repository-version to setuptools-version so installation of | |
| # packages will succeed | |
| echo "${setuptools_version}" | .ci/write-version | |
| oci_repo=${{ needs.params.outputs.oci_repository }} | |
| image_tag=${{ needs.version.outputs.effective_version }} | |
| image_ref=${oci_repo}/cicd/job-image:${image_tag} | |
| echo "oci_image_ref=${image_ref}" >> ${GITHUB_OUTPUT} | |
| cat << EOF > ocm_resources.yaml | |
| name: job-image | |
| version: ${image_tag} | |
| type: ociImage | |
| access: | |
| type: ociRegistry | |
| imageReference: ${image_ref} | |
| relation: local | |
| labels: | |
| - name: cloud.gardener.cnudie/dso/scanning-hints/package-versions | |
| value: | |
| - name: containerd | |
| version: v1.6.15 | |
| - name: gardener.cloud/cve-categorisation | |
| value: | |
| authentication_enforced: true | |
| availability_requirement: low | |
| confidentiality_requirement: high | |
| integrity_requirement: high | |
| network_exposure: protected | |
| user_interaction: gardener-operator | |
| EOF | |
| echo "ocm_resources=$(cat ocm_resources.yaml | base64 -w0)" >> ${GITHUB_OUTPUT} | |
| - name: Build OCI Image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| push: true | |
| platforms: ${{ needs.params.outputs.oci_platforms }} | |
| tags: ${{ steps.prepare.outputs.oci_image_ref }} | |
| context: . # pass modified path rather than clean checkout | |
| documentation: | |
| name: Generate Documentation | |
| needs: | |
| - images | |
| runs-on: ubuntu-latest | |
| environment: build | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Docker-Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Generate Documentation | |
| run: | | |
| image_ref=${{ needs.images.outputs.oci_image_ref }} | |
| mkdir documentation-out.d | |
| docker run -v$PWD:/src \ | |
| -e GH_PAGES_PATH=/src/documentation-out.d \ | |
| ${image_ref} \ | |
| /src/.ci/generate_documentation | |
| ls documentation-out.d | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: documentation | |
| path: documentation-out.d |