Skip to content

Image - - refs/pull/89/merge #605

Image - - refs/pull/89/merge

Image - - refs/pull/89/merge #605

Workflow file for this run

name: Image
run-name: 'Image - ${{ inputs.oci-image-name }} - ${{ github.ref }}'
on:
push:
paths:
- "oci/*/image.y*ml"
pull_request:
paths:
- "oci/*/image.y*ml"
workflow_dispatch:
inputs:
oci-image-name:
description: "OCI image to build and test"
required: true
b64-image-trigger:
description: "(Base64 encoded) Pass the image.yaml as an argument"
required: false
type: string
upload:
description: "Upload image to GHCR"
required: true
type: boolean
default: false
external_ref_id: #(1)
description: 'Optional ID for unique run detection'
required: false
type: string
default: "default-id"
env:
VULNERABILITY_REPORT_SUFFIX: ".vulnerability-report.json"
jobs:
prepare-build:
runs-on: ubuntu-22.04
name: Prepare build
outputs:
build-matrix: ${{ steps.prepare-matrix.outputs.build-matrix }}
release-to: ${{ steps.prepare-matrix.outputs.release-to }}
oci-img-path: ${{ steps.validate-image.outputs.img-path }}
oci-img-name: ${{ steps.validate-image.outputs.img-name }}
revision-data-cache-key: ${{ steps.prepare-matrix.outputs.revision-data-cache-key }}
steps:
- name: ${{ inputs.external_ref_id }} #(2)
if: ${{ github.event_name == 'workflow_dispatch' }}
run: echo 'Started by ${{ inputs.external_ref_id }}' >> "$GITHUB_STEP_SUMMARY"
- uses: actions/checkout@v3
- name: Infer number of image triggers
uses: tj-actions/changed-files@v35
id: changed-files
if: github.event_name != 'workflow_dispatch'
with:
separator: ","
dir_names: "true"
files: |
oci/*/image.y*ml
- name: Validate image from dispatch
id: validate-image
run: |
set -ex
if [[ "${{ github.event_name }}" != "workflow_dispatch" ]]
then
img_path="${{ steps.changed-files.outputs.all_changed_files }}"
occurrences="${img_path//[^,]}"
if [ ${#occurrences} -ne 0 ]
then
echo "ERR: can only build 1 image at a time, but trying to trigger ${img_path}"
exit 1
fi
else
img_path="oci/${{ inputs.oci-image-name }}"
fi
test -d "${img_path}"
echo "img-name=$(basename ${img_path})" >> "$GITHUB_OUTPUT"
echo "img-path=${img_path}" >> "$GITHUB_OUTPUT"
- name: Use custom image trigger
if: ${{ inputs.b64-image-trigger != '' }}
run: echo ${{ inputs.b64-image-trigger }} | base64 -d > ${{ steps.validate-image.outputs.img-path }}/image.yaml
- uses: actions/setup-python@v4
with:
python-version: "3.x"
- run: pip install -r src/image/requirements.txt
- name: Get next revision number
id: get-next-revision
env:
OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
IMAGE_NAME: ${{ steps.validate-image.outputs.img-name }}
SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
run: ./src/image/define_image_revision.sh
- name: Validate and prepare builds matrix
id: prepare-matrix
env:
DATA_DIR: "revision-data"
run: |
mkdir ${{ env.DATA_DIR }}
./src/image/prepare_single_image_build_matrix.py \
--oci-path ${{ steps.validate-image.outputs.img-path }} \
--revision-data-dir ${{ env.DATA_DIR }} \
--next-revision ${{ steps.get-next-revision.outputs.revision }}
echo "revision-data-cache-key=${{ github.run_id }}-${{ env.DATA_DIR }}" >> "$GITHUB_OUTPUT"
- uses: actions/cache/save@v3
with:
path: ${{ steps.prepare-matrix.outputs.revision-data-dir }}
key: ${{ steps.prepare-matrix.outputs.revision-data-cache-key }}
run-build:
runs-on: ubuntu-22.04
needs: [prepare-build]
name: Build
strategy:
fail-fast: true
matrix: ${{ fromJSON(needs.prepare-build.outputs.build-matrix) }}
env:
OCI_ARCHIVE_NAME: ${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.revision }}
steps:
- name: Clone GitHub image repository
uses: actions/checkout@v3
id: clone-image-repo
continue-on-error: true
with:
repository: ${{ matrix.source }}
fetch-depth: 0
- name: Clone generic image repository
if: ${{ steps.clone-image-repo.outcome == 'failure' }}
run: |
git clone ${{ matrix.source }} .
- run: git checkout ${{ matrix.commit }}
- name: Validate image naming and base
working-directory: ${{ matrix.directory }}
run: |
if [ -z ${{ matrix.dockerfile-build }} ]
then
rock_name=`cat rockcraft.y*ml | yq -r .name`
if [[ "${{ matrix.path }}" != *"${rock_name}"* ]]
then
echo "ERROR: the ROCK's name '${rock_name}' must match the OCI folder name!"
exit 1
fi
else
grep 'FROM' Dockerfile | tail -1 | grep ubuntu || \
(echo "ERROR: the image '${rock_name}' must be based on Ubuntu!" \
&& exit 1)
fi
# If this is a ROCK...
- name: Build ROCK ${{ matrix.name }}
if: matrix.dockerfile-build == ''
id: rockcraft
uses: canonical/craft-actions/rockcraft-pack@main
with:
path: "${{ matrix.directory }}"
verbosity: debug
- name: Rename ROCK OCI archive
if: matrix.dockerfile-build == ''
run: |
mv ${{ steps.rockcraft.outputs.rock }} ${{ env.OCI_ARCHIVE_NAME }}
# If this is a Dockerfile-based image...
- name: Set up QEMU
if: matrix.dockerfile-build != ''
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
if: matrix.dockerfile-build != ''
id: buildx
uses: docker/setup-buildx-action@v2
- name: Form the platforms string
if: matrix.dockerfile-build != ''
id: buildx-platforms
run: |
sudo apt install -y jq
platforms=$(echo linux/$(echo '${{ toJSON(matrix.dockerfile-build.platforms) }}' | jq -r 'join(",linux/")'))
echo "platforms=${platforms}" >> "$GITHUB_OUTPUT"
- name: Build OCI image ${{ matrix.name }}
if: matrix.dockerfile-build != ''
uses: docker/build-push-action@v4
with:
context: "${{ matrix.directory }}"
outputs: "type=oci,dest=${{ env.OCI_ARCHIVE_NAME }}"
platforms: ${{ steps.buildx-platforms.outputs.platforms }}
push: false
cache-from: type=gha
cache-to: type=gha,mode=max
provenance: false
sbom: false
- uses: actions/cache/save@v3
with:
path: ${{ env.OCI_ARCHIVE_NAME }}
key: ${{ github.run_id }}-${{ env.OCI_ARCHIVE_NAME }}
test:
needs: [prepare-build, run-build]
name: Test
strategy:
fail-fast: true
matrix: ${{ fromJSON(needs.prepare-build.outputs.build-matrix) }}
uses: canonical/oci-factory/.github/workflows/Tests.yaml@main
with:
oci-image-name: "${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.revision }}"
oci-image-path: "oci/${{ matrix.name }}"
is-a-rock: ${{ matrix.dockerfile-build == '' && true || false }}
test-from: "cache"
cache-key: "${{ github.run_id }}-${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.revision }}"
secrets: inherit
upload:
runs-on: ubuntu-22.04
needs: [prepare-build, run-build, test]
name: Upload
if: ${{ inputs.upload || (github.ref_name == 'main' && github.event_name == 'push') }}
strategy:
fail-fast: true
matrix: ${{ fromJSON(needs.prepare-build.outputs.build-matrix) }}
env:
OCI_ARCHIVE_NAME: ${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.revision }}
outputs:
artefacts-hashes: ${{ steps.artefacts-hashes.outputs.hashes }}
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Setup environment
env:
ROCKS_DEV_LP_SSH_PRIVATE: ${{ secrets.ROCKS_DEV_LP_SSH_PRIVATE }}
ROCKS_DEV_LP_USERNAME: ${{ secrets.ROCKS_DEV_LP_USERNAME }}
CPC_BUILD_TOOLS_REPO: git.launchpad.net/~cloudware/cloudware/+git/cpc_build_tools
# CPC_BUILD_TOOLS_REPO_REF: 9b716ed8a8ba728d036b54b1bb17a8f49dbda434
SKOPEO_BRANCH: "v1.9.1"
SKOPEO_URL: "https://github.com/containers/skopeo"
run: |
./src/image/requirements.sh
./src/uploads/requirements.sh
pip install -r src/uploads/requirements.txt -r src/image/requirements.txt
- name: Clone GitHub image repository
uses: actions/checkout@v3
id: clone-image-repo
continue-on-error: true
with:
repository: ${{ matrix.source }}
fetch-depth: 0
path: source
- name: Clone generic image repository
if: ${{ steps.clone-image-repo.outcome == 'failure' }}
run: |
git clone ${{ matrix.source }} source
- run: cd source && git checkout ${{ matrix.commit }}
- uses: actions/cache/restore@v3
with:
path: ${{ env.OCI_ARCHIVE_NAME }}
key: ${{ github.run_id }}-${{ env.OCI_ARCHIVE_NAME }}
fail-on-cache-miss: true
- name: Infer track name
id: get-track
env:
DOCKERFILE_IMAGE_VERSION: ${{ matrix.dockerfile-build.version }}
run: |
./src/uploads/infer_image_track.py --recipe-dirname source/${{ matrix.directory }}
- name: Name output artefact
id: rename-oci-archive
run: |
# Rename the OCI archive tarball
canonical_tag="${{ steps.get-track.outputs.track }}_${{ matrix.revision }}"
name="${{ matrix.name }}_${canonical_tag}"
mv ${{ env.OCI_ARCHIVE_NAME }} $name
echo "name=${name}" >> "$GITHUB_OUTPUT"
echo "canonical-tag=${canonical_tag}" >> "$GITHUB_OUTPUT"
- uses: actions/cache/save@v3
with:
path: ${{ steps.rename-oci-archive.outputs.name }}
key: ${{ github.run_id }}-${{ steps.rename-oci-archive.outputs.name }}
- name: Install Syft
uses: anchore/sbom-action/download-syft@v0
with:
syft-version: "v0.75.0"
- name: Infer architectures
id: get-arches
run: |
set -ex
arches=$(skopeo inspect --raw \
oci-archive:${{ steps.rename-oci-archive.outputs.name }} \
| jq -r 'if has("manifests") then .manifests[].platform.architecture else "${{ runner.arch }}" end' \
| jq -nRcr '[inputs] | join(",")')
echo "Detected architectures: ${arches}"
echo "arches='${arches}'" >> "$GITHUB_OUTPUT"
- name: Generate SBOMs
id: generate-sboms
run: |
set -ex
echo "Generating SBOMs for ${{ steps.get-arches.outputs.arches }}"
IFS=',' read -ra arch_list <<< ${{ steps.get-arches.outputs.arches }}
for arch in "${arch_list[@]}"; do
if [[ "${arch}" == "unknown" ]]; then
continue
fi
echo "Generate SBOM for ${arch}"
skopeo --override-arch $arch copy \
oci-archive:${{ steps.rename-oci-archive.outputs.name }} \
oci-archive:${{ steps.rename-oci-archive.outputs.name }}.${arch}
syft oci-archive:${{ steps.rename-oci-archive.outputs.name }}.${arch} \
-o spdx-json \
--file ${{ steps.rename-oci-archive.outputs.name }}.${arch}.sbom.spdx.json
done
all_sboms_zip="${{ steps.rename-oci-archive.outputs.name }}.sbom.spdx.zip"
zip "${all_sboms_zip}" ${{ steps.rename-oci-archive.outputs.name }}.*.sbom.spdx.json
echo "sboms=${all_sboms_zip}" >> "$GITHUB_OUTPUT"
- name: Fetch vulnerability artifacts for hashing
uses: actions/cache/restore@v3
with:
path: ${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}
key: ${{ github.run_id }}-${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}
fail-on-cache-miss: true
# https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/generic/README.md
- name: Calculate artefacts hashes
id: artefacts-hashes
env:
VULN_REPORT: ${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}
SBOMS: ${{ steps.generate-sboms.outputs.sboms }}
OCI_IMAGE_ARCHIVE: ${{ steps.rename-oci-archive.outputs.name }}
run: |
set -ex
echo "hashes=$(sha256sum ${{ env.VULN_REPORT }} ${{ env.OCI_IMAGE_ARCHIVE }} ${{ env.SBOMS }} | base64 -w0)"
- name: Login to GHCR
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Upload SBOM
uses: actions/upload-artifact@v3
with:
name: ${{ steps.generate-sboms.outputs.sboms }}
path: ${{ steps.generate-sboms.outputs.sboms }}
if-no-files-found: error
- name: Upload image
uses: actions/upload-artifact@v3
with:
name: ${{ steps.rename-oci-archive.outputs.name }}
path: ${{ steps.rename-oci-archive.outputs.name }}
if-no-files-found: error
- name: Upload to GHCR
id: upload-image
env:
GHCR_REPO: ghcr.io/${{ github.repository_owner }}/oci-factory
GHCR_USERNAME: ${{ github.actor }}
GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
run: |
oci_images="${PWD}/images-oci"
rm -fr $oci_images
mkdir -p $oci_images
tar -xf ${{ steps.rename-oci-archive.outputs.name }} -C $oci_images
source="oci:${oci_images}"
digest="$(skopeo inspect $source | jq -r .Digest)"
echo "digest=$digest" >> $GITHUB_OUTPUT
./src/image/tag_and_publish.sh "${source}" \
${{ matrix.name }} \
${{ steps.rename-oci-archive.outputs.canonical-tag }}
- name: Upload build metadata to Swift
env:
OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
IMAGE_NAME: ${{ matrix.name }}
SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
run: |
jq --arg base "${{ steps.get-track.outputs.base }}" \
--arg digest "${{ steps.upload-image.outputs.digest }}" \
'. + {base: $base, digest: $digest}' <<< '${{ toJSON(matrix) }}' > build_metadata.json
./src/uploads/upload_to_swift.sh \
${{ matrix.name }} \
${{ steps.get-track.outputs.track }} \
${{ matrix.revision }} \
build_metadata.json \
${{ steps.generate-sboms.outputs.sboms }} \
${{ env.OCI_ARCHIVE_NAME }}${{ env.VULNERABILITY_REPORT_SUFFIX }}
- name: Create Git tag
uses: rickstaa/action-create-tag@v1
with:
tag: "${{ steps.rename-oci-archive.outputs.name }}"
message: "upload(${{ matrix.name }}): Build and upload new image revision ${{ matrix.revision }}"
force_push_tag: true
github_token: ${{ secrets.ROCKSBOT_TOKEN }}
# We need this job because we want to make the releases all in one go,
# so that we know which revisions to release, and so that we can update
# and commit the _releases.json file in a single commit, outside a matrix job
dispatch-releases:
name: Dispatch releases
needs: [prepare-build, upload]
runs-on: ubuntu-22.04
if: ${{ needs.prepare-build.outputs.release-to != '' }}
concurrency:
group: ${{ needs.prepare-build.outputs.oci-img-path }}
cancel-in-progress: true
env:
REVISION_DATA_DIR: revision-data
steps:
- uses: actions/setup-python@v4
with:
python-version: "3.x"
- uses: actions/checkout@v3
with:
persist-credentials: false
- name: Use custom image trigger
if: ${{ inputs.b64-image-trigger != '' }}
run: echo ${{ inputs.b64-image-trigger }} | base64 -d > ${{ needs.prepare-build.outputs.oci-img-path }}/image.yaml
- uses: actions/cache/restore@v3
with:
path: ${{ env.REVISION_DATA_DIR }}
key: ${{ needs.prepare-build.outputs.revision-data-cache-key }}
fail-on-cache-miss: true
- run: pip install -r src/image/requirements.txt
- name: Merge release requests
id: merge-release-requests
env:
PYTHONUNBUFFERED: 1
run: |
set -ex
for revision_file in `ls ${{ env.REVISION_DATA_DIR }}`
do
ret=0
release=$(jq -er .release < ${{ env.REVISION_DATA_DIR }}/$revision_file) || ret=1
if [ $ret -eq 1 ]
then
echo "Revision $revision_file not marked for release"
continue
fi
echo "Merge revision $revision_file with requested releases"
./src/image/merge_release_info.py \
--image-trigger "${{ needs.prepare-build.outputs.oci-img-path }}/image.yaml" \
--revision-data-file "${{ env.REVISION_DATA_DIR }}/${revision_file}"
done
- uses: actions/cache/save@v3
with:
path: ${{ needs.prepare-build.outputs.oci-img-path }}/image.yaml
key: ${{ github.run_id }}-image-trigger
- name: Dispatch Releases workflow
# Using this action because others can have this problem:
# https://github.com/convictional/trigger-workflow-and-wait/issues/61
uses: mathze/workflow-dispatch-action@v1.2.0
id: run-releases
with:
token: ${{ secrets.GITHUB_TOKEN }}
ref: ${{ github.ref_name }}
fail-on-error: true
workflow-name: Release.yaml
payload: '{ "oci-image-name": "${{ needs.prepare-build.outputs.oci-img-name }}", "image-trigger-cache-key": "${{ github.run_id }}-image-trigger"}'
run-id: dummy
trigger-timeout: "10m"
wait-timeout: "20m"
use-marker-step: true
- name: Write step summary
run: |
url='${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.run-releases.outputs.run-id }}'
echo " - Triggered releases for '${{ needs.prepare-build.outputs.oci-img-name }}' at [${url}](${url})" >> "$GITHUB_STEP_SUMMARY"
- name: Enforce release conclusion
if: ${{ steps.run-releases.outputs.run-conclusion != 'success' }}
# The previous step doesn't always raise an error
run: |
url='${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.run-releases.outputs.run-id }}'
echo "Failed to release '${{ needs.prepare-build.outputs.oci-img-name }}' at [${url}](${url})."
exit 1
generate-provenance:
name: Generate SLSA provenance
needs: [upload]
runs-on: ubuntu-22.04
steps:
- uses: actions/download-artifact@v3
id: download
with:
path: artifacts
- name: Generate provenance
uses: philips-labs/slsa-provenance-action@v0.8.0
with:
command: generate
subcommand: files
arguments: --artifact-path ${{ steps.download.outputs.download-path }} --output-path provenance.json
# - uses: sigstore/cosign-installer@v3.0.1
# with:
# cosign-release: 'v2.0.0'
# - name: Sign provenance file
# env:
# PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
# COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
# PUBLIC_KEY: ${{ secrets.COSIGN_PUBLIC_KEY }}
# run: |
# echo "${PRIVATE_KEY}" > cosign.key
# echo "${PUBLIC_KEY}" > cosign.pub
# cosign sign-blob --key cosign.key --output-signature provenance.json.sig provenance.json
# - name: Upload verification keys
# uses: actions/upload-artifact@v3
# with:
# name: verify
# path: |
# cosign.pub
# provenance.json.sig
# if-no-files-found: error
# - name: Generate checksums
# working-directory: ${{ steps.download.outputs.download-path }}
# run: sha256sum * > SHA256SUMS || true
# - name: Upload SHA256SUMS file
# uses: actions/upload-artifact@v3
# with:
# name: SHA256SUMS
# path: SHA256SUMS
# if-no-files-found: error
- name: Upload provenance file
uses: actions/upload-artifact@v3
with:
name: provenance
path: provenance.json
if-no-files-found: error
# generate-provenance:
# name: Generate SLSA provenance
# needs: [upload]
# permissions:
# actions: read # Needed for detection of GitHub Actions environment.
# id-token: write # Needed for provenance signing and ID
# contents: write # Needed for release uploads
# uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.5.0
# with:
# base64-subjects: "${{ needs.upload.outputs.artefacts-hashes }}"
notify:
runs-on: ubuntu-22.04
name: Notify
needs: [prepare-build, run-build, upload, dispatch-releases, generate-provenance]
if: ${{ always() && contains(needs.*.result, 'failure') && github.event_name != 'pull_request' }}
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.x'
- name: Summarize workflow failure message
id: get-summary
run: |
echo '${{ toJson(needs) }}' > jobs.json
./src/notifications/summarize_workflow_results.py --jobs-file jobs.json
- name: Get contacts for ${{ needs.prepare-build.outputs.oci-img-name }}
id: get-contacts
working-directory: ${{ needs.prepare-build.outputs.oci-img-path }}
run: |
mm_channels=$(yq -r '.notify | ."mattermost-channels" | join(",")' < contacts.y*ml)
echo "mattermost-channels=${mm_channels}" >> "$GITHUB_OUTPUT"
- name: Notify via Mattermost
env:
MM_BOT_TOKEN: ${{ secrets.MM_BOT_TOKEN }}
FINAL_STATUS: failure
MM_SERVER: ${{ secrets.MM_SERVER }}
URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
SUMMARY: ${{ steps.get-summary.outputs.summary }}
FOOTER: "Triggered by ${{ github.triggering_actor }}. Ref: ${{ github.ref }}. Attempts: ${{ github.run_attempt }}"
TITLE: '${{ needs.prepare-build.outputs.oci-img-name }}: failed to build->upload->release'
run: |
for channel in $(echo ${{ steps.get-contacts.outputs.mattermost-channels }} | tr ',' ' ')
do
MM_CHANNEL_ID="${channel}" ./src/notifications/send_to_mattermost.sh
done