Skip to content
Stackable Build Pipeline

feat: Add cache auto-purge and --old arg to cache clean command #296

Sign in to view logs

feat: Add cache auto-purge and --old arg to cache clean command

feat: Add cache auto-purge and --old arg to cache clean command #296

Workflow file for this run

.github/workflows/build.yml at f7b0df5
---
name: Stackable Build Pipeline
on:
push:
branches:
- main
- staging
- trying
- "renovate/**"
tags:
- "[0-9][0-9].[0-9]+.[0-9]+"
pull_request:
merge_group:
env:
RUST_VERSION: 1.70.0
CARGO_TERM_COLOR: always
CARGO_INCREMENTAL: "0"
CARGO_PROFILE_DEV_DEBUG: "0"
RUSTFLAGS: "-D warnings"
RUSTDOCFLAGS: "-D warnings"
RUST_LOG: "info"
DEV_REPO_HELM_URL: https://repo.stackable.tech/repository/helm-dev
TEST_REPO_HELM_URL: https://repo.stackable.tech/repository/helm-test
STABLE_REPO_HELM_URL: https://repo.stackable.tech/repository/helm-stable
jobs:
# Identify unused dependencies
run_udeps:
name: Run Cargo Udeps
runs-on: ubuntu-latest
env:
RUSTC_BOOTSTRAP: 1
steps:
- name: Install host dependencies
run: |
sudo apt-get update
sudo apt-get install protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
with:
submodules: recursive
- uses: dtolnay/rust-toolchain@0e66bd3e6b38ec0ad5312288c83e47c143e6b09e # v1
with:
toolchain: ${{ env.RUST_VERSION }}
- uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
with:
node-version: 18
cache: yarn
- run: yarn install --frozen-lockfile
- uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0
with:
key: udeps
- run: cargo install cargo-udeps
- run: cargo udeps --workspace
# This job evaluates the github environment to determine why this action is running and selects the appropriate
# target repository for published Helm charts based on this.
#
# The following scenarios are identified:
# - all pull requests land in the test repository:
# condition: github.event_name == "pull_request"
# repository: test
#
# - all tagged releases land in stable:
# condition: github.event_name == 'create' & github.ref.startswith('refs/tags/')
# repository: stable
#
# - all pushes to main (i.e. PR-merges) land in dev:
# condition: github.event_name == 'push' & github.ref == 'refs/heads/main'
# repository: dev
#
# Any other scenarios (e.g. when a branch is created/pushed) will cause the publish step to be skipped, most commonly this is expected to happen for the
# branches that bors uses internally (staging, trying) for which the checks need to run, but we do not want artifacts
# to be published.
select_helm_repo:
name: Select target helm repository based on action trigger
runs-on: ubuntu-latest
outputs:
helm_repository: ${{ steps.selecthelmrepo.outputs.helm_repo }}
steps:
- id: selecthelmrepo
env:
TRIGGER: ${{ github.event_name }}
GITHUB_REF: ${{ github.ref }}
run: |
if [[ $TRIGGER == "pull_request" ]]; then
echo "exporting test as target helm repo: ${{ env.TEST_REPO_HELM_URL }}"
echo "helm_repo=${{ env.TEST_REPO_HELM_URL }}" >> $GITHUB_OUTPUT
elif [[ $TRIGGER == "push" && $GITHUB_REF == "refs/heads/main" ]]; then
echo "exporting dev as target helm repo: ${{ env.DEV_REPO_HELM_URL }}"
echo "helm_repo=${{ env.DEV_REPO_HELM_URL }}" >> $GITHUB_OUTPUT
elif [[ ( $TRIGGER == "create" || $TRIGGER == "push" ) && $GITHUB_REF == refs/tags/* ]]; then
echo "exporting stable as target helm repo: ${{ env.STABLE_REPO_HELM_URL }}"
echo "helm_repo=${{ env.STABLE_REPO_HELM_URL }}" >> $GITHUB_OUTPUT
else
echo "Unknown trigger and ref combination encountered, skipping publish step: $TRIGGER $GITHUB_REF"
echo "helm_repo=skip" >> $GITHUB_OUTPUT
fi
run_cargodeny:
name: Run Cargo Deny
runs-on: ubuntu-latest
strategy:
matrix:
checks:
- advisories
- bans licenses sources
# Prevent sudden announcement of a new advisory from failing ci:
continue-on-error: ${{ matrix.checks == 'advisories' }}
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
with:
submodules: recursive
- uses: EmbarkStudios/cargo-deny-action@8a8607bd8e2b3a514d5a40174cc7c55b229d9ba7 # v1.4.0
with:
command: check ${{ matrix.checks }}
run_rustfmt:
name: Run Rustfmt
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
with:
submodules: recursive
- uses: dtolnay/rust-toolchain@0e66bd3e6b38ec0ad5312288c83e47c143e6b09e # v1
with:
toolchain: ${{ env.RUST_VERSION }}
components: rustfmt
- run: cargo fmt --all -- --check
run_clippy:
name: Run Clippy
runs-on: ubuntu-latest
steps:
- name: Install host dependencies
run: |
sudo apt-get update
sudo apt-get install protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
with:
submodules: recursive
- uses: dtolnay/rust-toolchain@0e66bd3e6b38ec0ad5312288c83e47c143e6b09e # v1
with:
toolchain: ${{ env.RUST_VERSION }}
components: clippy
- uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0
with:
key: clippy
- uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
with:
node-version: 18
cache: yarn
- run: yarn install --frozen-lockfile
- name: Run clippy action to produce annotations
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: actions-rs/clippy-check@b5b5f21f4797c02da247df37026fcd0a5024aa4d # renovate: tag=v1.0.7
if: env.GITHUB_TOKEN != null
with:
args: --all-targets -- -D warnings
token: ${{ secrets.GITHUB_TOKEN }}
- name: Run clippy manually without annotations
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
if: env.GITHUB_TOKEN == null
run: cargo clippy --all-targets -- -D warnings
run_rustdoc:
name: Run RustDoc
runs-on: ubuntu-latest
steps:
- name: Install host dependencies
run: |
sudo apt-get update
sudo apt-get install protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
with:
submodules: recursive
- uses: dtolnay/rust-toolchain@0e66bd3e6b38ec0ad5312288c83e47c143e6b09e # v1
with:
toolchain: ${{ env.RUST_VERSION }}
components: rustfmt
- uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0
with:
key: doc
- uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
with:
node-version: 18
cache: yarn
- run: yarn install --frozen-lockfile
- run: cargo doc --document-private-items
run_tests:
name: Run Cargo Tests
runs-on: ubuntu-latest
steps:
- name: Install host dependencies
run: |
sudo apt-get update
sudo apt-get install protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
with:
submodules: recursive
- uses: dtolnay/rust-toolchain@0e66bd3e6b38ec0ad5312288c83e47c143e6b09e # v1
with:
toolchain: ${{ env.RUST_VERSION }}
- uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0
with:
key: test
- uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
with:
node-version: 18
cache: yarn
- run: yarn install --frozen-lockfile
- run: cargo test
# Similar to check_charts, this tries to render the README, and see if there are unintended changes.
# This will save us from merging changes to the wrong file (instead of the templated source), and from
# forgetting to render out modifications to the README.
# check_readme:
# name: Check if committed README is the one we would render from the available parts
# runs-on: ubuntu-latest
# steps:
# - name: Checkout
# uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
# with:
# submodules: recursive
# - uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.6.1
# with:
# python-version: '3.11'
# - name: Install jinja2-cli
# run: pip install jinja2-cli==0.8.2
# - name: Regenerate charts
# run: make render-readme
# - name: Check if committed README were up to date
# run: git diff --exit-code
# - name: Git Diff showed uncommitted changes
# if: ${{ failure() }}
# uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6
# with:
# script: |
# core.setFailed('Committed README are not up to date, please make sure to apply them to the templated partials, and re-commit!')
# This job cleans up the CRDs and Helm charts, followed by rebuilding them
# It then runs a `git diff` and fails the entire workflow, if any difference is encountered.
#
# Since CRD files are generated during the 'cargo build' process we need to run this once after
# removing the CRD files to ensure that the checked in versions match what the code expects.
#
# The reason for this step is, that developers are expected to check in up-to-date versions of charts
# as we'd otherwise have to build these in CI and commit them back to the PR, which
# creates all kinds of problems.
# This failsafe simply aborts anything that has not had charts rebuilt before pushing.
check_charts:
name: Check if committed Helm charts are up to date
runs-on: ubuntu-latest
steps:
- name: Install host dependencies
run: |
sudo apt-get update
sudo apt-get install protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config
- name: Checkout
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
with:
submodules: recursive
- name: Set up Helm
uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5
with:
version: v3.6.2
- name: Set up cargo
uses: dtolnay/rust-toolchain@1.68.2
- name: Set up rust-cache
uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0
with:
key: charts
# - name: Regenerate charts
# run: make regenerate-charts
- name: Check if committed charts were up to date
run: git diff --exit-code
- name: Git Diff showed uncommitted changes
if: ${{ failure() }}
uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6
with:
script: |
core.setFailed('Committed charts were not up to date, please regenerate and re-commit!')
tests_passed:
name: All tests passed
needs:
- run_udeps
- run_cargodeny
- run_clippy
- run_rustfmt
- run_rustdoc
- run_tests
- check_charts
# - check_readme
runs-on: ubuntu-latest
steps:
- name: log
run: echo All tests have passed!
package_and_publish:
name: Package Charts, Build Docker Image and publish them
needs:
- tests_passed
- select_helm_repo
runs-on: ubuntu-latest
env:
NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
HELM_REPO: ${{ needs.select_helm_repo.outputs.helm_repository }}
if: needs.select_helm_repo.outputs.helm_repository != 'skip'
outputs:
IMAGE_TAG: ${{ steps.printtag.outputs.IMAGE_TAG }}
steps:
- name: Install host dependencies
run: |
sudo apt-get update
sudo apt-get install protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config
- name: Checkout
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
with:
submodules: recursive
- uses: dtolnay/rust-toolchain@1.68.2
with:
components: rustfmt
# This step checks if the current run was triggered by a push to a pr (or a pr being created).
# If this is the case it changes the version of this project in all Cargo.toml files to include the suffix
# "-pr<prnumber>" so that the published artifacts can be linked to this PR.
- name: Update version if PR
if: ${{ github.event_name == 'pull_request' }}
run: |
cargo install cargo-edit --version 0.11.11
cargo set-version --offline --workspace 0.0.0-pr${{ github.event.pull_request.number }}
# Recreate charts and publish charts and docker image. The "-e" is needed as we want to override the
# default value in the makefile if called from this action, but not otherwise (i.e. when called locally).
# This is needed for the HELM_REPO variable.
- name: Publish Docker image and Helm chart
run: make -e publish
# Output the name of the published image to the Job output for later use
- id: printtag
name: Output image name and tag
run: echo "IMAGE_TAG=$(make -e print-docker-tag)" >> $GITHUB_OUTPUT
openshift_preflight:
name: Run the OpenShift Preflight check on the published images
needs:
- package_and_publish
runs-on: ubuntu-latest
env:
IMAGE_TAG: ${{ needs.package_and_publish.outputs.IMAGE_TAG }}
steps:
- name: Install preflight
run: |
wget https://github.com/redhat-openshift-ecosystem/openshift-preflight/releases/latest/download/preflight-linux-amd64
chmod +x preflight-linux-amd64
- name: Check container
run: ./preflight-linux-amd64 check container "$IMAGE_TAG" > preflight.out
- name: "Passed?"
run: '[ "$(./preflight-linux-amd64 check container "$IMAGE_TAG" | jq -r .passed)" == true ]'