From 2caf5c3e36222b0914c11156c2aaca0b68e91c54 Mon Sep 17 00:00:00 2001 From: jiaxiao zhou Date: Thu, 4 Apr 2024 22:51:03 +0000 Subject: [PATCH 01/16] ci(*): modify the release pipeline to create a PR automatically Signed-off-by: jiaxiao zhou --- .github/workflows/release.yml | 109 ++++++++++++---------------------- main.log | 7 +++ 2 files changed, 45 insertions(+), 71 deletions(-) create mode 100644 main.log diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ad8e7837d..39e8e3c13 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -83,79 +83,46 @@ jobs: cargo owner --add github:containerd:runwasi-committers ${{ inputs.crate }} env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_PUBLISH_TOKEN }} - - name: Verify version matches + - name: Update crate version and make a PR + if: ${{ !inputs.dry_run }} run: | - if [ "$(grep -c "version = \"${{ inputs.version }}\"" crates/${{ inputs.crate }}/Cargo.toml)" -ne 1 ]; then - echo "::error::Version in Cargo.toml does not match the version input" - exit 1 + # replace the version inline in the Cargo.toml + set -ex + git fetch origin + + sed -i -E 's/^version.+=.+".+"/version = "${{ inputs.version }}"/' crates/${{ inputs.crate }}/Cargo.toml + git diff + git config user.name "${GITHUB_ACTOR}" + git config user.email "${GITHUB_ACTOR}@users.noreply.github.com" + + if [ "${{ inputs.dry_run }}" = true ]; then + TITLE="[dry-run] Release ${{ inputs.crate }} v${{ inputs.version }}" + else + TITLE="Release ${{ inputs.crate }} v${{ inputs.version }}" fi - build-and-sign: - permissions: - id-token: write - needs: - - pre-release - strategy: - matrix: - arch: ["x86_64", "aarch64"] - include: - - ${{ needs.pre-release.outputs }} - uses: ./.github/workflows/action-build.yml - with: - os: "ubuntu-22.04" - runtime: ${{ matrix.runtime }} - target: "${{ matrix.arch }}-unknown-linux-musl" - slug: "${{ matrix.arch }}-linux-musl" - arch: ${{ matrix.arch }} - sign: true + git commit --allow-empty -a -F-<> $GITHUB_ENV + echo "PR_TITLE=$TITLE" >> $GITHUB_ENV + echo "PR_BASE=main" >> $GITHUB_ENV + cat > pr-body <<-EOF + This is an automated pull request from CI to release + ${{ inputs.crate }} v${{ inputs.version }} when merged. The commit + message for this PR has a marker that is detected by CI to create + tags and publish crate artifacts. + + When first opened this PR will not have CI run because it is generated + by a bot. A maintainer should close this PR and then reopen it to + trigger CI to execute which will then enable merging this PR. + EOF + - name: Make a PR + run: gh pr create -B "$PR_BASE" -H "$PR_HEAD" --title "$PR_TITLE" --body "$(cat ./pr-body)" env: - GH_TOKEN: ${{ github.token }} - RELEASE_NAME: ${{ matrix.crate }}/v${{ matrix.version }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/main.log b/main.log new file mode 100644 index 000000000..667df0a17 --- /dev/null +++ b/main.log @@ -0,0 +1,7 @@ +commit 2c0eaa5efdc37064480bc0400027eaf54054f137 +Author: jiaxiao zhou +Date: Wed Apr 3 23:35:13 2024 +0000 + + remove permission for write contents in build action + + Signed-off-by: jiaxiao zhou From c6b3fa7001e0b772956c0e313a6dedbb4c379e62 Mon Sep 17 00:00:00 2001 From: jiaxiao zhou Date: Thu, 4 Apr 2024 23:44:19 +0000 Subject: [PATCH 02/16] ci(*): automate the version bump process this commit automates the version bump process to ask the bot to create a PR to bump the respective crate's version to the given one on behalf of humans. the PR created by the bot contains magic message that will be parsed by another workflow `publish` which triggers on merge to the main branch. The `publish` workflow parses the commit message to get crate, version and dry-run information to be able to publish the crate, release binary and push tags. the motivation for this commit is to further simplify the release process Signed-off-by: jiaxiao zhou --- .github/workflows/publish.yml | 101 ++++++++++++++++++++++++++++++++++ .github/workflows/release.yml | 15 ----- scripts/parse-crate.sh | 33 +++++++++++ 3 files changed, 134 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/publish.yml create mode 100755 scripts/parse-crate.sh diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..16d603b1f --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,101 @@ + +name: Publish Artifacts + +on: + push: + branches: [main] + +permissions: + contents: write + +jobs: + parse: + runs-on: ubuntu-latest + outputs: + push_tag: ${{ steps.tag.outputs.push_tag }} + dry_run: ${{ steps.parse.outputs.dry_run }} + crate: ${{ steps.parse.outputs.crate }} + version: ${{ steps.parse.outputs.version }} + is_binary: ${{ steps.parse_crate.outputs.is_binary }} + is_crate: ${{ steps.parse_crate.outputs.is_crate }} + steps: + - name: Test if tag is needed + id: tag + run: | + git log ${{ github.event.before }}...${{ github.event.after }} | tee main.log + if grep -q "automatically-tag-and-release-this-commit" main.log; then + echo push-tag + echo "push_tag=yes" >> $GITHUB_OUTPUT + else + echo no-push-tag + echo "push_tag=no" >> $GITHUB_OUTPUT + fi + - name: Parse commit message + id: parse + if: steps.tag.outputs.push_tag == 'yes' + run: | + dry_run=false + crate=$(grep 'Release ' main.log | sed 's/.*Release \([a-zA-Z0-9_-]*\).*/\1/') + version=$(grep 'Release ' main.log | sed 's/.* v\(.*\)/\1/') + if grep -q '\[dry-run\]' main.log; then + dry_run=true + fi + + echo "dry_run: $dry_run" + echo "crate: $crate" + echo "version: $version" + + echo "dry_run:$dry_run" >> $GITHUB_OUTPUT + echo "crate:$crate" >> $GITHUB_OUTPUT + echo "version:$version" >> $GITHUB_OUTPUT + + - name: parse crate + id: parse_crate + if: steps.tag.outputs.push_tag == 'yes' + run: | + ./scripts/parse-crate.sh ${{ steps.parse.outputs.crate }} >> $GITHUB_OUTPUT + + release: + runs-on: "ubuntu-latest" + needs: parse + if: needs.parse.outputs.push_tag == 'yes' + steps: + - uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.parse.outputs.is_binary == 'true' + uses: actions/download-artifact@master + with: + path: release + + - name: Cargo publish + if: needs.parse.outputs.is_crate == 'true' + run: | + echo "DRY_RUN_FLAG=$( [ '${{ needs.parse.outputs.dry_run }}' = 'true' ] && echo '--dry-run' || echo '' )" >> $GITHUB_ENV + cargo publish $DRY_RUN_FLAG --package ${{ needs.parse.outputs.crate }} --verbose --locked + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_PUBLISH_TOKEN }} + + - name: Tag the the release + if: needs.parse.outputs.dry_run == 'false' + run: | + git tag "${{ needs.parse.outputs.crate }}/v${{ needs.parse.outputs.version }}" + git push origin "${{ needs.parse.outputs.crate }}/v${{ needs.parse.outputs.version }}" + + - name: Create release + if: needs.parse.outputs.dry_run == 'false' + run: | + gh release create 'refs/tags/${{ needs.parse.outputs.crate }}/v${{ needs.parse.outputs.version }}' --generate-notes + env: + GH_TOKEN: ${{ github.token }} + RELEASE_NAME: ${{ needs.parse.outputs.crate }}/v${{ needs.parse.outputs.version }} + + - name: Upload release artifacts + if: needs.parse.outputs.dry_run == 'false' + run: | + for i in release/*/*; do + gh release upload ${RELEASE_NAME} $i + done + env: + GH_TOKEN: ${{ github.token }} + RELEASE_NAME: ${{ needs.parse.outputs.crate }}/v${{ needs.parse.outputs.version }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 39e8e3c13..95f4b5c43 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,12 +38,6 @@ jobs: pre-release: name: pre-release checks runs-on: "ubuntu-latest" - outputs: - crate: ${{ inputs.crate }} - runtime: ${{ steps.runtime_sub.outputs.runtime }} - version: ${{ inputs.version }} - ### is_shim is a string, not a boolean, so use: is_shim == 'true' - is_shim: ${{ steps.runtime_sub.outputs.is_shim }} steps: - name: Fail if branch is not main if: github.event_name == 'workflow_dispatch' && github.ref != 'refs/heads/main' @@ -51,15 +45,6 @@ jobs: echo "::error::This workflow should not be triggered with workflow_dispatch on a branch other than main" exit 1 - uses: actions/checkout@v4 - ### Determine the name of the runtime and if it is a binary release or crates.io - - name: verify version input - uses: actions/github-script@v7 - with: - script: | - const version = '${{ inputs.version }}'; - if(!version.match(/^[0-9]+.[0-9]+.*/)) { - core.setFailed(`The version '${version}' does not match regex /^[0-9]+.[0-9]+.*/.`); - } - name: substring runtime id: runtime_sub uses: actions/github-script@v7 diff --git a/scripts/parse-crate.sh b/scripts/parse-crate.sh new file mode 100755 index 000000000..8fb9338f9 --- /dev/null +++ b/scripts/parse-crate.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +crate_name=$1 +is_binary="false" +is_crate="false" + +# Define ground truth for binary crates +declare -A binary_map=( + ["oci-tar-builder"]=true + ["containerd-shim-wasmtime"]=true + ["containerd-shim-wasmer"]=true + ["containerd-shim-wasmedge"]=true +) + +# Define ground truth for crate items +declare -A crate_map=( + ["oci-tar-builder"]=true + ["containerd-shim-wasm"]=true + ["containerd-shim-wasm-test-modules"]=true +) + +# Check and assign based on the binary_map +if [[ "${binary_map[$crate_name]}" == "true" ]]; then + is_binary="true" +fi + +# Check and assign based on the crate_map +if [[ "${crate_map[$crate_name]}" == "true" ]]; then + is_crate="true" +fi + +echo "is_binary=$is_binary" +echo "is_crate=$is_crate" \ No newline at end of file From a9c52e8b76714a4ce07a7360d0496dce081abb0f Mon Sep 17 00:00:00 2001 From: jiaxiao zhou Date: Thu, 4 Apr 2024 23:56:41 +0000 Subject: [PATCH 03/16] ci: remove dry-run on make a pr step Signed-off-by: jiaxiao zhou --- .github/workflows/release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 95f4b5c43..998fa5e54 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -69,7 +69,6 @@ jobs: env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_PUBLISH_TOKEN }} - name: Update crate version and make a PR - if: ${{ !inputs.dry_run }} run: | # replace the version inline in the Cargo.toml set -ex From ada18bbc03006038e7542fd03986019c87bc5ff3 Mon Sep 17 00:00:00 2001 From: jiaxiao zhou Date: Fri, 5 Apr 2024 00:02:55 +0000 Subject: [PATCH 04/16] ci: add checkout to parse job Signed-off-by: jiaxiao zhou --- .github/workflows/publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 16d603b1f..da342d6e6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -19,6 +19,7 @@ jobs: is_binary: ${{ steps.parse_crate.outputs.is_binary }} is_crate: ${{ steps.parse_crate.outputs.is_crate }} steps: + - uses: actions/checkout@v4 - name: Test if tag is needed id: tag run: | From 5468e8f396e53832368e6cc2f35aee59e0ecbdbb Mon Sep 17 00:00:00 2001 From: jiaxiao zhou Date: Fri, 5 Apr 2024 00:18:07 +0000 Subject: [PATCH 05/16] ci: add fetch origin to job Signed-off-by: jiaxiao zhou --- .github/workflows/publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index da342d6e6..b466e511a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -23,6 +23,7 @@ jobs: - name: Test if tag is needed id: tag run: | + git fetch origin git log ${{ github.event.before }}...${{ github.event.after }} | tee main.log if grep -q "automatically-tag-and-release-this-commit" main.log; then echo push-tag From 1698184350bc40db2cde2daccc4dd8239dc7f5ef Mon Sep 17 00:00:00 2001 From: jiaxiao zhou Date: Fri, 5 Apr 2024 00:28:01 +0000 Subject: [PATCH 06/16] ci: add fetch depty 0 to checkout Signed-off-by: jiaxiao zhou --- .github/workflows/publish.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b466e511a..bfb3f101e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,12 +18,16 @@ jobs: version: ${{ steps.parse.outputs.version }} is_binary: ${{ steps.parse_crate.outputs.is_binary }} is_crate: ${{ steps.parse_crate.outputs.is_crate }} + # e0ef1cf000bfb3f12e5b8613977ac9d901e39fbd...1ab3eb4318fb824cc44e36530b48e600fe18f258 steps: - uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 - name: Test if tag is needed id: tag run: | - git fetch origin + git log -n 2 | cat git log ${{ github.event.before }}...${{ github.event.after }} | tee main.log if grep -q "automatically-tag-and-release-this-commit" main.log; then echo push-tag From 8e154f914b2d47fbc737c37e074667640a037b27 Mon Sep 17 00:00:00 2001 From: jiaxiao zhou Date: Fri, 5 Apr 2024 00:31:20 +0000 Subject: [PATCH 07/16] ci: fix parse step Signed-off-by: jiaxiao zhou --- .github/workflows/publish.yml | 7 +++---- main.log | 7 ------- 2 files changed, 3 insertions(+), 11 deletions(-) delete mode 100644 main.log diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bfb3f101e..30b89db6d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,7 +18,6 @@ jobs: version: ${{ steps.parse.outputs.version }} is_binary: ${{ steps.parse_crate.outputs.is_binary }} is_crate: ${{ steps.parse_crate.outputs.is_crate }} - # e0ef1cf000bfb3f12e5b8613977ac9d901e39fbd...1ab3eb4318fb824cc44e36530b48e600fe18f258 steps: - uses: actions/checkout@v4 with: @@ -51,9 +50,9 @@ jobs: echo "crate: $crate" echo "version: $version" - echo "dry_run:$dry_run" >> $GITHUB_OUTPUT - echo "crate:$crate" >> $GITHUB_OUTPUT - echo "version:$version" >> $GITHUB_OUTPUT + echo "dry_run=$dry_run" >> $GITHUB_OUTPUT + echo "crate=$crate" >> $GITHUB_OUTPUT + echo "version=$version" >> $GITHUB_OUTPUT - name: parse crate id: parse_crate diff --git a/main.log b/main.log deleted file mode 100644 index 667df0a17..000000000 --- a/main.log +++ /dev/null @@ -1,7 +0,0 @@ -commit 2c0eaa5efdc37064480bc0400027eaf54054f137 -Author: jiaxiao zhou -Date: Wed Apr 3 23:35:13 2024 +0000 - - remove permission for write contents in build action - - Signed-off-by: jiaxiao zhou From 0f3fe7cc4fad1ff4e26d347bc2afaf8ba5e5a66a Mon Sep 17 00:00:00 2001 From: jiaxiao zhou Date: Tue, 16 Apr 2024 23:52:34 +0000 Subject: [PATCH 08/16] ci(*): remove signing from the build CI this commit removes signing from the build CI as the release pipeline will now handle signing of the binaries downloaded using actions/download-artifact. the biggest change is that the parse script is re-write to be able to output all the necessary variables such as `runtime`, `dry_run` and more. the action-sign workflow is also modified to take an extra `path` input to replace the hard-coded `dist/bin` path Signed-off-by: jiaxiao zhou --- .github/workflows/action-build.yml | 11 +--------- .github/workflows/action-sign.yml | 19 ++++++++-------- .github/workflows/publish.yml | 34 ++++++++++------------------- .github/workflows/release.yml | 21 +++++------------- scripts/parse-crate.sh | 35 ++++++++++++++++++++++++++---- 5 files changed, 58 insertions(+), 62 deletions(-) diff --git a/.github/workflows/action-build.yml b/.github/workflows/action-build.yml index 566ef1348..159952ef4 100644 --- a/.github/workflows/action-build.yml +++ b/.github/workflows/action-build.yml @@ -22,12 +22,9 @@ on: arch: required: false type: string - sign: - default: false - type: boolean jobs: - build-sign-upload: + build-upload: permissions: id-token: write name: build for ${{ inputs.slug }} @@ -60,12 +57,6 @@ jobs: run: | make test-${{ inputs.runtime }} if: ${{ inputs.arch == 'x86_64' }} - - name: Sign the binary - if: ${{ inputs.runtime != 'common' && inputs.slug != 'windows' && inputs.sign }} - uses: ./.github/workflows/action-sign.yml - with: - runtime: ${{ inputs.runtime }} - os: ${{ inputs.os }} - name: Package artifacts if: ${{ inputs.runtime != 'common' }} shell: bash diff --git a/.github/workflows/action-sign.yml b/.github/workflows/action-sign.yml index aed870300..874b254ad 100644 --- a/.github/workflows/action-sign.yml +++ b/.github/workflows/action-sign.yml @@ -8,14 +8,15 @@ on: runtime: required: true type: string - os: + path: required: true type: string jobs: sign: - name: Sign the binaries on ${{ inputs.os }} - runs-on: ${{ inputs.os }} + name: Sign the binaries + runs-on: "ubuntu-latest" steps: + - uses: actions/checkout@v4 - name: Setup cosign for signing uses: sigstore/cosign-installer@v3.3.0 with: @@ -24,29 +25,29 @@ jobs: run: | make dist-${{ inputs.runtime }} # Check if there's any files to archive as tar fails otherwise - if stat dist/bin/* >/dev/null 2>&1; then + if stat ${{ inputs.path }}/* >/dev/null 2>&1; then echo "::notice::Signing the binary" cosign sign-blob --yes \ --output-signature containerd-shim-${{ inputs.runtime }}-v1.sig \ --output-certificate containerd-shim-${{ inputs.runtime }}-v1.pem \ --bundle containerd-shim-${{ inputs.runtime }}-v1.bundle \ - dist/bin/containerd-shim-${{ inputs.runtime }}-v1 + ${{ inputs.path }}/containerd-shim-${{ inputs.runtime }}-v1 cosign sign-blob --yes \ --output-signature containerd-shim-${{ inputs.runtime }}d-v1.sig \ --output-certificate containerd-shim-${{ inputs.runtime }}d-v1.pem \ --bundle containerd-shim-${{ inputs.runtime }}d-v1.bundle \ - dist/bin/containerd-shim-${{ inputs.runtime }}d-v1 + ${{ inputs.path }}/containerd-shim-${{ inputs.runtime }}d-v1 cosign sign-blob --yes \ --output-signature containerd-${{ inputs.runtime }}d.sig \ --output-certificate containerd-${{ inputs.runtime }}d.pem \ --bundle containerd-${{ inputs.runtime }}d.bundle \ - dist/bin/containerd-${{ inputs.runtime }}d + ${{ inputs.path }}/containerd-${{ inputs.runtime }}d # Copy the certs to the dist/bin folder - cp *.sig dist/bin/ - cp *.pem dist/bin/ + cp *.sig ${{ inputs.path }}/ + cp *.pem ${{ inputs.path }}/ else echo "::warning::No files to sign" fi \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 30b89db6d..6edfc05cb 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -15,9 +15,10 @@ jobs: push_tag: ${{ steps.tag.outputs.push_tag }} dry_run: ${{ steps.parse.outputs.dry_run }} crate: ${{ steps.parse.outputs.crate }} + runtime: ${{ steps.parse.outputs.runtime }} version: ${{ steps.parse.outputs.version }} - is_binary: ${{ steps.parse_crate.outputs.is_binary }} - is_crate: ${{ steps.parse_crate.outputs.is_crate }} + is_binary: ${{ steps.parse.outputs.is_binary }} + is_crate: ${{ steps.parse.outputs.is_crate }} steps: - uses: actions/checkout@v4 with: @@ -39,27 +40,7 @@ jobs: id: parse if: steps.tag.outputs.push_tag == 'yes' run: | - dry_run=false - crate=$(grep 'Release ' main.log | sed 's/.*Release \([a-zA-Z0-9_-]*\).*/\1/') - version=$(grep 'Release ' main.log | sed 's/.* v\(.*\)/\1/') - if grep -q '\[dry-run\]' main.log; then - dry_run=true - fi - - echo "dry_run: $dry_run" - echo "crate: $crate" - echo "version: $version" - - echo "dry_run=$dry_run" >> $GITHUB_OUTPUT - echo "crate=$crate" >> $GITHUB_OUTPUT - echo "version=$version" >> $GITHUB_OUTPUT - - - name: parse crate - id: parse_crate - if: steps.tag.outputs.push_tag == 'yes' - run: | - ./scripts/parse-crate.sh ${{ steps.parse.outputs.crate }} >> $GITHUB_OUTPUT - + ./scripts/parse-crate.sh main.log >> $GITHUB_OUTPUT release: runs-on: "ubuntu-latest" needs: parse @@ -72,6 +53,13 @@ jobs: uses: actions/download-artifact@master with: path: release + + - name: Sign + if: needs.parse.outputs.is_binary == 'true' + uses: ./.github/workflows/action-sign.yml + with: + path: release + runtime: ${{ needs.parse.outputs.runtime }} - name: Cargo publish if: needs.parse.outputs.is_crate == 'true' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 998fa5e54..6637a7cd7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,24 +45,13 @@ jobs: echo "::error::This workflow should not be triggered with workflow_dispatch on a branch other than main" exit 1 - uses: actions/checkout@v4 - - name: substring runtime - id: runtime_sub - uses: actions/github-script@v7 - with: - script: | - const crate = '${{ inputs.crate }}'; - const non_shim_crates = ['wasm', 'wasm-test-modules', 'oci-tar-builder']; - if non_shim_crates.includes(runtime) { - core.setOutput('runtime', 'common'); - core.setOutput('is_shim', false) - } else { - const runtime = crate.replace(/^containerd-shim-/, ''); - core.setOutput('runtime', runtime); - core.setOutput('is_shim', true); - } + - name: parse crate + id: parse_crate + run: | + ./scripts/parse-crate.sh ${{ steps.parse.outputs.crate }} >> $GITHUB_OUTPUT ### If we are releasing a crate rather than producing a bin, check for crates.io access - name: Check crates.io ownership - if: ${{ steps.runtime_sub.outputs.is_shim != 'true' }} + if: steps.parse_crate.outputs.is_crate == 'true' run: | cargo owner --list ${{ inputs.crate }} | grep github:containerd:runwasi-committers || \ cargo owner --add github:containerd:runwasi-committers ${{ inputs.crate }} diff --git a/scripts/parse-crate.sh b/scripts/parse-crate.sh index 8fb9338f9..c50e2774a 100755 --- a/scripts/parse-crate.sh +++ b/scripts/parse-crate.sh @@ -1,6 +1,20 @@ #!/bin/bash -crate_name=$1 +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 +fi + +log_file="$1" + +# extract crate and version from log file +dry_run=false +crate=$(grep 'Release ' "$log_file" | sed 's/.*Release \([a-zA-Z0-9_-]*\).*/\1/') +version=$(grep 'Release ' "$log_file" | sed 's/.* v\(.*\)/\1/') +if grep -q '\[dry-run\]' "$log_file"; then + dry_run=true +fi + is_binary="false" is_crate="false" @@ -20,14 +34,27 @@ declare -A crate_map=( ) # Check and assign based on the binary_map -if [[ "${binary_map[$crate_name]}" == "true" ]]; then +if [[ "${binary_map[$crate]}" == "true" ]]; then is_binary="true" fi # Check and assign based on the crate_map -if [[ "${crate_map[$crate_name]}" == "true" ]]; then +if [[ "${crate_map[$crate]}" == "true" ]]; then is_crate="true" fi +# Runtime logic +declare -a non_shim_crates=("containerd-shim-wasm" "containerd-shim-wasm-test-modules" "oci-tar-builder") +runtime="" + +if printf '%s\n' "${non_shim_crates[@]}" | grep -q "^$crate$"; then + runtime="common" +else + runtime="${crate#containerd-shim-}" +fi +echo "dry_run=$dry_run" +echo "crate=$crate" +echo "version=$version" echo "is_binary=$is_binary" -echo "is_crate=$is_crate" \ No newline at end of file +echo "is_crate=$is_crate" +echo "runtime=$runtime" \ No newline at end of file From d24cf906e7991d03dfb0baf0fc4051de84a3cc34 Mon Sep 17 00:00:00 2001 From: jiaxiao zhou Date: Wed, 17 Apr 2024 00:05:24 +0000 Subject: [PATCH 09/16] feat(ci): bring back the runtime_sub logic to release Signed-off-by: jiaxiao zhou --- .github/workflows/release.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6637a7cd7..833807f06 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,13 +45,18 @@ jobs: echo "::error::This workflow should not be triggered with workflow_dispatch on a branch other than main" exit 1 - uses: actions/checkout@v4 - - name: parse crate - id: parse_crate - run: | - ./scripts/parse-crate.sh ${{ steps.parse.outputs.crate }} >> $GITHUB_OUTPUT + - name: substring runtime + id: runtime_sub + uses: actions/github-script@v7 + with: + script: | + const crate = '${{ inputs.crate }}'; + let runtime = crate.replace(/^containerd-shim-/, ''); + const non_shim_crates = ['wasm', 'wasm-test-modules', 'oci-tar-builder']; + core.setOutput('is_shim', !non_shim_crates.includes(runtime)); ### If we are releasing a crate rather than producing a bin, check for crates.io access - name: Check crates.io ownership - if: steps.parse_crate.outputs.is_crate == 'true' + if: ${{ steps.runtime_sub.outputs.is_shim != 'true' }} run: | cargo owner --list ${{ inputs.crate }} | grep github:containerd:runwasi-committers || \ cargo owner --add github:containerd:runwasi-committers ${{ inputs.crate }} From 0e3262449ea70389725604a6dfacd6a2a107e285 Mon Sep 17 00:00:00 2001 From: jiaxiao zhou Date: Wed, 17 Apr 2024 00:15:36 +0000 Subject: [PATCH 10/16] feat(ci): add annotation Signed-off-by: jiaxiao zhou --- .github/workflows/publish.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6edfc05cb..44db72e99 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -46,8 +46,10 @@ jobs: needs: parse if: needs.parse.outputs.push_tag == 'yes' steps: + - name: describe runner + run: | + echo "::notice::Running job with push_tag: '${{ needs.parse.outputs.push_tag }}', dry_run: '${{ needs.parse.outputs.dry_run }}', crate: '${{ needs.parse.outputs.crate }}', runtime: '${{ needs.parse.outputs.runtime }}', version: '${{ needs.parse.outputs.version }}', is_binary: '${{ needs.parse.outputs.is_binary }}', is_crate: '${{ needs.parse.outputs.is_crate }}'" - uses: actions/checkout@v4 - - name: Download artifacts if: needs.parse.outputs.is_binary == 'true' uses: actions/download-artifact@master From e19cf5f155e5b833f5ccce600ff81f5e2bc5ac4f Mon Sep 17 00:00:00 2001 From: jiaxiao zhou Date: Wed, 17 Apr 2024 00:26:08 +0000 Subject: [PATCH 11/16] feat(ci): make the commit message parsing more robust Signed-off-by: jiaxiao zhou --- .github/workflows/publish.yml | 1 + .github/workflows/release.yml | 2 ++ scripts/parse-crate.sh | 4 ++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 44db72e99..f34ee3bec 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -49,6 +49,7 @@ jobs: - name: describe runner run: | echo "::notice::Running job with push_tag: '${{ needs.parse.outputs.push_tag }}', dry_run: '${{ needs.parse.outputs.dry_run }}', crate: '${{ needs.parse.outputs.crate }}', runtime: '${{ needs.parse.outputs.runtime }}', version: '${{ needs.parse.outputs.version }}', is_binary: '${{ needs.parse.outputs.is_binary }}', is_crate: '${{ needs.parse.outputs.is_crate }}'" + - uses: actions/checkout@v4 - name: Download artifacts if: needs.parse.outputs.is_binary == 'true' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 833807f06..220a54a08 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -83,6 +83,8 @@ jobs: $TITLE [automatically-tag-and-release-this-commit] + Version: v${{ inputs.version }} + Crate: ${{ inputs.crate }} EOF git push origin HEAD:ci/release-${{ inputs.crate }}-${{ inputs.version }} diff --git a/scripts/parse-crate.sh b/scripts/parse-crate.sh index c50e2774a..3944bbc73 100755 --- a/scripts/parse-crate.sh +++ b/scripts/parse-crate.sh @@ -9,8 +9,8 @@ log_file="$1" # extract crate and version from log file dry_run=false -crate=$(grep 'Release ' "$log_file" | sed 's/.*Release \([a-zA-Z0-9_-]*\).*/\1/') -version=$(grep 'Release ' "$log_file" | sed 's/.* v\(.*\)/\1/') +crate=$(grep 'Crate: ' "$log_file" | sed 's/.*Crate: \([a-zA-Z0-9_-]*\).*/\1/') +version=$(grep 'Version: ' "$log_file" | sed 's/.*Version: v\(.*\)/\1/') if grep -q '\[dry-run\]' "$log_file"; then dry_run=true fi From 9fbf4d6d612f5bcf5a626c6d90c2e6858fffc3cc Mon Sep 17 00:00:00 2001 From: jiaxiao zhou Date: Wed, 17 Apr 2024 00:28:09 +0000 Subject: [PATCH 12/16] feat(ci): add more annotation Signed-off-by: jiaxiao zhou --- .github/workflows/publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f34ee3bec..2854dc000 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -35,6 +35,7 @@ jobs: else echo no-push-tag echo "push_tag=no" >> $GITHUB_OUTPUT + echo "::notice::No tag needed, skipping release" fi - name: Parse commit message id: parse From 10272458d53efef9ef0557d5bd2724215919b3e0 Mon Sep 17 00:00:00 2001 From: jiaxiao zhou Date: Wed, 17 Apr 2024 00:54:53 +0000 Subject: [PATCH 13/16] feat(ci): refactor action-sign Signed-off-by: jiaxiao zhou --- .github/workflows/action-sign.yml | 49 ++++++++++++++++++++++++------- .github/workflows/publish.yml | 15 +++++----- 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/.github/workflows/action-sign.yml b/.github/workflows/action-sign.yml index 874b254ad..dd9827a2c 100644 --- a/.github/workflows/action-sign.yml +++ b/.github/workflows/action-sign.yml @@ -8,46 +8,73 @@ on: runtime: required: true type: string - path: + is_binary: required: true - type: string + type: boolean jobs: sign: name: Sign the binaries runs-on: "ubuntu-latest" + if: ${{ inputs.is_binary }} steps: - uses: actions/checkout@v4 - name: Setup cosign for signing uses: sigstore/cosign-installer@v3.3.0 with: cosign-release: 'v2.2.2' + - name: download artifact from main + uses: dawidd6/action-download-artifact@v3 + with: + github_token: ${{secrets.GITHUB_TOKEN}} + branch: main + path: artifacts + - name: unpack artifact + run: | + mkdir -p dist + if stat artifacts/* >/dev/null 2>&1; then + tar -xzf artifacts/containerd-shim-${{ inputs.runtime }}.tar.gz -C dist + else + echo "::warning::No artifacts" + fi + - name: Sign the binaries run: | - make dist-${{ inputs.runtime }} # Check if there's any files to archive as tar fails otherwise - if stat ${{ inputs.path }}/* >/dev/null 2>&1; then + if stat dist/* >/dev/null 2>&1; then echo "::notice::Signing the binary" cosign sign-blob --yes \ --output-signature containerd-shim-${{ inputs.runtime }}-v1.sig \ --output-certificate containerd-shim-${{ inputs.runtime }}-v1.pem \ --bundle containerd-shim-${{ inputs.runtime }}-v1.bundle \ - ${{ inputs.path }}/containerd-shim-${{ inputs.runtime }}-v1 + dist/containerd-shim-${{ inputs.runtime }}-v1 cosign sign-blob --yes \ --output-signature containerd-shim-${{ inputs.runtime }}d-v1.sig \ --output-certificate containerd-shim-${{ inputs.runtime }}d-v1.pem \ --bundle containerd-shim-${{ inputs.runtime }}d-v1.bundle \ - ${{ inputs.path }}/containerd-shim-${{ inputs.runtime }}d-v1 + dist/containerd-shim-${{ inputs.runtime }}d-v1 cosign sign-blob --yes \ --output-signature containerd-${{ inputs.runtime }}d.sig \ --output-certificate containerd-${{ inputs.runtime }}d.pem \ --bundle containerd-${{ inputs.runtime }}d.bundle \ - ${{ inputs.path }}/containerd-${{ inputs.runtime }}d + dist/containerd-${{ inputs.runtime }}d - # Copy the certs to the dist/bin folder - cp *.sig ${{ inputs.path }}/ - cp *.pem ${{ inputs.path }}/ + # Copy the certs to the dist folder + cp *.sig dist/ + cp *.pem dist/ else echo "::warning::No files to sign" - fi \ No newline at end of file + fi + - name: package artifacts + run: | + if stat dist/* >/dev/null 2>&1; then + tar -czf dist-${{ inputs.runtime }}.tar.gz -C dist . + else + echo "::warning::No files to package" + fi + - name: Upload the signed binaries + uses: actions/upload-artifact@v2 + with: + name: dist-${{ inputs.runtime }} + path: dist/dist-${{ inputs.runtime }}.tar.gz \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2854dc000..eb704cb64 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -42,9 +42,15 @@ jobs: if: steps.tag.outputs.push_tag == 'yes' run: | ./scripts/parse-crate.sh main.log >> $GITHUB_OUTPUT + sign: + uses: ./.github/workflows/action-sign.yml + with: + runtime: ${{ needs.parse.outputs.runtime }} + is_binary: ${{ needs.parse.outputs.is_binary }} + release: runs-on: "ubuntu-latest" - needs: parse + needs: [parse, sign] if: needs.parse.outputs.push_tag == 'yes' steps: - name: describe runner @@ -57,13 +63,6 @@ jobs: uses: actions/download-artifact@master with: path: release - - - name: Sign - if: needs.parse.outputs.is_binary == 'true' - uses: ./.github/workflows/action-sign.yml - with: - path: release - runtime: ${{ needs.parse.outputs.runtime }} - name: Cargo publish if: needs.parse.outputs.is_crate == 'true' From c4f8089f09994d86e2afebbb4802f63ebea80e40 Mon Sep 17 00:00:00 2001 From: jiaxiao zhou Date: Wed, 17 Apr 2024 00:56:23 +0000 Subject: [PATCH 14/16] feat(ci): fix publish.yml Signed-off-by: jiaxiao zhou --- .github/workflows/action-sign.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/action-sign.yml b/.github/workflows/action-sign.yml index dd9827a2c..4cac31ad9 100644 --- a/.github/workflows/action-sign.yml +++ b/.github/workflows/action-sign.yml @@ -10,12 +10,12 @@ on: type: string is_binary: required: true - type: boolean + type: string jobs: sign: name: Sign the binaries runs-on: "ubuntu-latest" - if: ${{ inputs.is_binary }} + if: ${{ inputs.is_binary == 'true' }} steps: - uses: actions/checkout@v4 - name: Setup cosign for signing From 2543ed3447f49e4f57365f319f1d07fc91d14597 Mon Sep 17 00:00:00 2001 From: jiaxiao zhou Date: Wed, 17 Apr 2024 00:57:10 +0000 Subject: [PATCH 15/16] feat(ci): add condition on signing Signed-off-by: jiaxiao zhou --- .github/workflows/publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index eb704cb64..d4e354985 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -43,6 +43,8 @@ jobs: run: | ./scripts/parse-crate.sh main.log >> $GITHUB_OUTPUT sign: + needs: [parse] + if: needs.parse.outputs.push_tag == 'yes' uses: ./.github/workflows/action-sign.yml with: runtime: ${{ needs.parse.outputs.runtime }} From d58bc86786d0f94474fe3b90f5fc1ce695e75eeb Mon Sep 17 00:00:00 2001 From: Mossaka Date: Wed, 17 Apr 2024 01:00:15 +0000 Subject: [PATCH 16/16] [dry-run] Release containerd-shim-wasmtime v0.5.0 [automatically-tag-and-release-this-commit] Version: v0.5.0 Crate: containerd-shim-wasmtime --- crates/containerd-shim-wasmtime/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/containerd-shim-wasmtime/Cargo.toml b/crates/containerd-shim-wasmtime/Cargo.toml index 05a8e7fa1..7fe9653ca 100644 --- a/crates/containerd-shim-wasmtime/Cargo.toml +++ b/crates/containerd-shim-wasmtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "containerd-shim-wasmtime" -version = "0.4.0" +version = "0.5.0" edition.workspace = true [dependencies]