From 8c0fda9994685eddaf6b87d2fb7bae93ecce8192 Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Mon, 7 Oct 2024 00:03:35 -0400 Subject: [PATCH] chore: Cleanup workflows to be run from just --- .github/workflows/build-pr.yml | 213 ++++++------------ .github/workflows/build.yml | 159 ++++++------- Cargo.lock | 1 + Cargo.toml | 2 + Earthfile | 9 +- .../test-repo/recipes/akmods.yml | 1 - justfile | 127 ++++++++++- process/Cargo.toml | 2 +- process/drivers/opts/inspect.rs | 1 + process/drivers/types.rs | 2 +- src/commands/generate.rs | 47 ++-- template/src/lib.rs | 2 +- template/templates/Containerfile.j2 | 2 +- template/templates/modules/modules.j2 | 4 +- template/templates/stages.j2 | 2 +- utils/src/constants.rs | 1 + 16 files changed, 310 insertions(+), 265 deletions(-) diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml index d840d4c7..2b4c22e4 100644 --- a/.github/workflows/build-pr.yml +++ b/.github/workflows/build-pr.yml @@ -14,7 +14,7 @@ env: jobs: test: - timeout-minutes: 20 + timeout-minutes: 10 runs-on: ubuntu-latest steps: @@ -32,7 +32,7 @@ jobs: earthly --ci +test lint: - timeout-minutes: 20 + timeout-minutes: 10 runs-on: ubuntu-latest steps: @@ -50,22 +50,16 @@ jobs: earthly --ci +lint arm64-prebuild: - timeout-minutes: 60 + timeout-minutes: 10 runs-on: ubuntu-latest + if: github.repository == 'blue-build/cli' steps: - - name: Maximize build space - uses: ublue-os/remove-unwanted-software@v6 - env: - EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} - if: env.EARTHLY_SAT_TOKEN == null - - uses: earthly/actions-setup@v1 - name: Earthly login env: EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} - if: env.EARTHLY_SAT_TOKEN != null run: | earthly account login --token ${{ secrets.EARTHLY_SAT_TOKEN }} >> /dev/null earthly org s blue-build @@ -79,7 +73,6 @@ jobs: - name: Login to GitHub Container Registry uses: docker/login-action@v3 - if: github.token != null with: registry: ghcr.io username: ${{ github.actor }} @@ -91,22 +84,16 @@ jobs: earthly --ci --push -P +prebuild amd64-prebuild: - timeout-minutes: 60 + timeout-minutes: 10 runs-on: ubuntu-latest + if: github.repository == 'blue-build/cli' steps: - - name: Maximize build space - uses: ublue-os/remove-unwanted-software@v6 - env: - EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} - if: env.EARTHLY_SAT_TOKEN == null - - uses: earthly/actions-setup@v1 - name: Earthly login env: EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} - if: env.EARTHLY_SAT_TOKEN != null run: | earthly account login --token ${{ secrets.EARTHLY_SAT_TOKEN }} >> /dev/null earthly org s blue-build @@ -120,7 +107,6 @@ jobs: - name: Login to GitHub Container Registry uses: docker/login-action@v3 - if: github.token != null with: registry: ghcr.io username: ${{ github.actor }} @@ -131,28 +117,19 @@ jobs: run: | earthly --ci --push -P +prebuild - build: - timeout-minutes: 60 + build-images: + timeout-minutes: 30 runs-on: ubuntu-latest + if: github.repository == 'blue-build/cli' needs: - arm64-prebuild - amd64-prebuild - outputs: - push: ${{ steps.build.outputs.push }} - steps: - - name: Maximize build space - uses: ublue-os/remove-unwanted-software@v6 - env: - EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} - if: env.EARTHLY_SAT_TOKEN == null - - uses: earthly/actions-setup@v1 - name: Earthly login env: EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} - if: env.EARTHLY_SAT_TOKEN != null run: | earthly account login --token ${{ secrets.EARTHLY_SAT_TOKEN }} >> /dev/null earthly org s blue-build @@ -166,7 +143,6 @@ jobs: - name: Login to GitHub Container Registry uses: docker/login-action@v3 - if: github.token != null with: registry: ghcr.io username: ${{ github.actor }} @@ -175,21 +151,38 @@ jobs: - name: Run build id: build run: | - if [ -n "${{ github.token }}" ]; then - earthly --ci --push -P +build - echo "push=true" >> "$GITHUB_OUTPUT" - else - earthly --ci -P +build - fi + earthly --ci --push -P +build-images + + build-scripts: + timeout-minutes: 5 + runs-on: ubuntu-latest + if: github.repository == 'blue-build/cli' + steps: + - uses: earthly/actions-setup@v1 + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Run build + id: build + run: | + earthly --ci --push -P +build-scripts integration-tests: permissions: packages: write timeout-minutes: 60 runs-on: ubuntu-latest - needs: - - build - if: needs.build.outputs.push == 'true' steps: - name: Maximize build space @@ -210,15 +203,12 @@ jobs: earthly --ci -P ./integration-tests+all docker-build: - timeout-minutes: 60 + timeout-minutes: 20 runs-on: ubuntu-latest permissions: contents: read packages: write id-token: write - needs: - - build - if: needs.build.outputs.push == 'true' steps: - name: Maximize build space @@ -239,35 +229,26 @@ jobs: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} - - name: Install bluebuild - run: | - cargo install --path . --debug --all-features - - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 + - uses: extractions/setup-just@v1 + - name: Run Build env: GH_TOKEN: ${{ github.token }} GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} BB_BUILDKIT_CACHE_GHA: true - run: | - cd integration-tests/test-repo - bluebuild template -vv | tee Containerfile - grep -q 'ARG IMAGE_REGISTRY=ghcr.io/blue-build' Containerfile || exit 1 - bluebuild build --retry-push -B docker -I docker -S sigstore --push -vv recipes/recipe.yml recipes/recipe-39.yml + run: just test-docker-build arm64-build: - timeout-minutes: 60 + timeout-minutes: 40 runs-on: ubuntu-latest permissions: contents: read packages: write id-token: write - needs: - - build - if: needs.build.outputs.push == 'true' steps: - name: Maximize build space @@ -288,38 +269,26 @@ jobs: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} - - name: Install bluebuild - run: | - cargo install --path . --debug --all-features - - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 + - uses: extractions/setup-just@v1 + - name: Run Build env: GH_TOKEN: ${{ github.token }} GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} BB_BUILDKIT_CACHE_GHA: true - run: | - cd integration-tests/test-repo - bluebuild build \ - --retry-push \ - --platform linux/arm64 \ - --push \ - -vv \ - recipes/recipe-arm64.yml + run: just test-arm64-build docker-build-external-login: - timeout-minutes: 60 + timeout-minutes: 20 runs-on: ubuntu-latest permissions: contents: read packages: write id-token: write - needs: - - build - if: needs.build.outputs.push == 'true' steps: - name: Maximize build space @@ -347,34 +316,25 @@ jobs: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} - - name: Install bluebuild - run: | - cargo install --path . --debug --all-features - - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 + - uses: extractions/setup-just@v1 + - name: Run Build env: GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} BB_BUILDKIT_CACHE_GHA: true - run: | - cd integration-tests/test-repo - bluebuild template -vv | tee Containerfile - grep -q 'ARG IMAGE_REGISTRY=ghcr.io/blue-build' Containerfile || exit 1 - bluebuild build --retry-push -S sigstore --push -vv recipes/recipe.yml recipes/recipe-39.yml + run: just test-docker-build-external-login docker-build-oauth-login: - timeout-minutes: 60 + timeout-minutes: 20 runs-on: ubuntu-latest permissions: contents: read packages: write id-token: write - needs: - - build - if: needs.build.outputs.push == 'true' steps: - name: Google Auth @@ -413,39 +373,25 @@ jobs: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} - - name: Install bluebuild - run: | - cargo install --path . --debug --all-features - - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 + - uses: extractions/setup-just@v1 + - name: Run Build env: GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} BB_BUILDKIT_CACHE_GHA: true - run: | - cd integration-tests/test-repo - bluebuild template -vv | tee Containerfile - bluebuild build \ - --registry us-east1-docker.pkg.dev \ - --registry-namespace bluebuild-oidc/bluebuild \ - --retry-push \ - --push \ - -vv \ - recipes/recipe.yml recipes/recipe-39.yml + run: just test-docker-build-oauth-login podman-build: - timeout-minutes: 60 + timeout-minutes: 20 runs-on: ubuntu-latest permissions: contents: read packages: write id-token: write - needs: - - build - if: needs.build.outputs.push == 'true' steps: - name: Maximize build space @@ -473,31 +419,22 @@ jobs: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} - - name: Install bluebuild - run: | - cargo install --path . --debug --all-features + - uses: extractions/setup-just@v1 - name: Run Build env: GH_TOKEN: ${{ github.token }} GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} - run: | - cd integration-tests/test-repo - bluebuild template -vv | tee Containerfile - grep -q 'ARG IMAGE_REGISTRY=ghcr.io/blue-build' Containerfile || exit 1 - bluebuild build --retry-push -B podman -I podman -S sigstore --push -vv recipes/recipe.yml recipes/recipe-39.yml + run: just test-podman-build buildah-build: - timeout-minutes: 15 + timeout-minutes: 20 runs-on: ubuntu-latest permissions: contents: read packages: write id-token: write - needs: - - build - if: needs.build.outputs.push == 'true' steps: - name: Maximize build space @@ -525,31 +462,22 @@ jobs: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} - - name: Install bluebuild - run: | - cargo install --path . --debug --all-features + - uses: extractions/setup-just@v1 - name: Run Build env: GH_TOKEN: ${{ github.token }} GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} - run: | - cd integration-tests/test-repo - bluebuild template -vv | tee Containerfile - grep -q 'ARG IMAGE_REGISTRY=ghcr.io/blue-build' Containerfile || exit 1 - bluebuild build --retry-push -B buildah -I podman -S sigstore --squash --push -vv recipes/recipe.yml recipes/recipe-39.yml + run: just test-buildah-build iso-from-image: - timeout-minutes: 60 + timeout-minutes: 20 runs-on: ubuntu-latest permissions: contents: read packages: write id-token: write - needs: - - build - if: needs.build.outputs.push == 'true' steps: - name: Maximize build space @@ -570,33 +498,26 @@ jobs: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} - - name: Install bluebuild - run: | - cargo install --path . --debug --all-features - - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 + - uses: extractions/setup-just@v1 + - name: Run Build env: GH_TOKEN: ${{ github.token }} GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} BB_BUILDKIT_CACHE_GHA: true - run: | - cd integration-tests/test-repo - bluebuild generate-iso image ghcr.io/blue-build/cli/test:40 + run: just test-generate-iso-image iso-from-recipe: - timeout-minutes: 60 + timeout-minutes: 20 runs-on: ubuntu-latest permissions: contents: read packages: write id-token: write - needs: - - build - if: needs.build.outputs.push == 'true' steps: - name: Maximize build space @@ -617,19 +538,15 @@ jobs: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} - - name: Install bluebuild - run: | - cargo install --path . --debug --all-features - - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 + - uses: extractions/setup-just@v1 + - name: Run Build env: GH_TOKEN: ${{ github.token }} GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} BB_BUILDKIT_CACHE_GHA: true - run: | - cd integration-tests/test-repo - bluebuild generate-iso -vv recipe recipes/recipe.yml + run: just test-generate-iso-recipe diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 86d64bef..2adc2746 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -134,7 +134,47 @@ jobs: run: | earthly --ci --push -P +prebuild - build: + build-scripts: + timeout-minutes: 60 + runs-on: ubuntu-latest + if: github.repository == 'blue-build/cli' + steps: + - name: Maximize build space + uses: ublue-os/remove-unwanted-software@v6 + env: + EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} + if: env.EARTHLY_SAT_TOKEN == null + + - uses: earthly/actions-setup@v1 + + - name: Earthly login + env: + EARTHLY_SAT_TOKEN: ${{ secrets.EARTHLY_SAT_TOKEN }} + if: env.EARTHLY_SAT_TOKEN != null + run: | + earthly account login --token ${{ secrets.EARTHLY_SAT_TOKEN }} >> /dev/null + earthly org s blue-build + earthly sat s pr + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Run build + id: build + run: | + earthly --ci --push -P +build-scripts + + build-images: permissions: packages: write timeout-minutes: 60 @@ -188,7 +228,7 @@ jobs: runs-on: ubuntu-latest if: github.repository == 'blue-build/cli' needs: - - build + - build-scripts steps: - name: Maximize build space @@ -216,7 +256,7 @@ jobs: id-token: write if: github.repository == 'blue-build/cli' needs: - - build + - build-scripts steps: - name: Maximize build space @@ -236,28 +276,21 @@ jobs: with: ref: main - - - name: Install bluebuild - run: | - cargo install --path . --debug --all-features - - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 + - uses: extractions/setup-just@v1 + - name: Run Build env: GH_TOKEN: ${{ github.token }} GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} BB_BUILDKIT_CACHE_GHA: true - run: | - cd integration-tests/test-repo - bluebuild template -vv | tee Containerfile - grep -q 'ARG IMAGE_REGISTRY=ghcr.io/blue-build' Containerfile || exit 1 - bluebuild build --retry-push -B docker -I docker -S sigstore --push -vv recipes/recipe.yml recipes/recipe-39.yml + run: just test-docker-build arm64-build: - timeout-minutes: 60 + timeout-minutes: 40 runs-on: ubuntu-latest permissions: contents: read @@ -265,7 +298,7 @@ jobs: id-token: write if: github.repository == 'blue-build/cli' needs: - - build + - build-scripts steps: - name: Maximize build space @@ -286,27 +319,18 @@ jobs: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} - - name: Install bluebuild - run: | - cargo install --path . --debug --all-features - - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 + - uses: extractions/setup-just@v1 + - name: Run Build env: GH_TOKEN: ${{ github.token }} GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} BB_BUILDKIT_CACHE_GHA: true - run: | - cd integration-tests/test-repo - bluebuild build \ - --retry-push \ - --platform linux/arm64 \ - --push \ - -vv \ - recipes/recipe-arm64.yml + run: just test-arm64-build docker-build-external-login: timeout-minutes: 60 @@ -317,7 +341,7 @@ jobs: id-token: write if: github.repository == 'blue-build/cli' needs: - - build + - build-scripts steps: - name: Maximize build space @@ -345,23 +369,17 @@ jobs: ref: main - - name: Install bluebuild - run: | - cargo install --path . --debug --all-features - - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 + - uses: extractions/setup-just@v1 + - name: Run Build env: GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} BB_BUILDKIT_CACHE_GHA: true - run: | - cd integration-tests/test-repo - bluebuild template -vv | tee Containerfile - grep -q 'ARG IMAGE_REGISTRY=ghcr.io/blue-build' Containerfile || exit 1 - bluebuild build --retry-push -S sigstore --push -vv recipes/recipe.yml recipes/recipe-39.yml + run: just test-docker-build-external-login docker-build-oauth-login: timeout-minutes: 60 @@ -371,7 +389,7 @@ jobs: packages: write id-token: write needs: - - build + - build-scripts if: github.repository == 'blue-build/cli' steps: @@ -409,28 +427,17 @@ jobs: with: ref: main - - name: Install bluebuild - run: | - cargo install --path . --debug --all-features - - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 + - uses: extractions/setup-just@v1 + - name: Run Build env: GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} BB_BUILDKIT_CACHE_GHA: true - run: | - cd integration-tests/test-repo - bluebuild template -vv | tee Containerfile - bluebuild build \ - --registry us-east1-docker.pkg.dev \ - --registry-namespace bluebuild-oidc/bluebuild \ - --retry-push \ - --push \ - -vv \ - recipes/recipe.yml recipes/recipe-39.yml + run: just test-docker-build-oauth-login podman-build: timeout-minutes: 60 @@ -441,7 +448,7 @@ jobs: id-token: write if: github.repository == 'blue-build/cli' needs: - - build + - build-scripts steps: - name: Maximize build space @@ -469,20 +476,14 @@ jobs: ref: main - - name: Install bluebuild - run: | - cargo install --path . --debug --all-features + - uses: extractions/setup-just@v1 - name: Run Build env: GH_TOKEN: ${{ github.token }} GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} - run: | - cd integration-tests/test-repo - bluebuild template -vv | tee Containerfile - grep -q 'ARG IMAGE_REGISTRY=ghcr.io/blue-build' Containerfile || exit 1 - bluebuild build --retry-push -B podman -I podman -S sigstore --push -vv recipes/recipe.yml recipes/recipe-39.yml + run: just test-podman-build buildah-build: timeout-minutes: 60 @@ -493,7 +494,7 @@ jobs: id-token: write if: github.repository == 'blue-build/cli' needs: - - build + - build-scripts steps: - name: Maximize build space @@ -521,20 +522,14 @@ jobs: ref: main - - name: Install bluebuild - run: | - cargo install --path . --debug --all-features + - uses: extractions/setup-just@v1 - name: Run Build env: GH_TOKEN: ${{ github.token }} GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} - run: | - cd integration-tests/test-repo - bluebuild template -vv | tee Containerfile - grep -q 'ARG IMAGE_REGISTRY=ghcr.io/blue-build' Containerfile || exit 1 - bluebuild build --retry-push -B buildah -I podman -S sigstore --squash --push -vv recipes/recipe.yml recipes/recipe-39.yml + run: just test-buildah-build iso-from-image: timeout-minutes: 60 @@ -544,7 +539,7 @@ jobs: packages: write id-token: write needs: - - build + - build-scripts if: github.repository == 'blue-build/cli' steps: @@ -566,22 +561,18 @@ jobs: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} - - name: Install bluebuild - run: | - cargo install --path . --debug --all-features - - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 + - uses: extractions/setup-just@v1 + - name: Run Build env: GH_TOKEN: ${{ github.token }} GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} BB_BUILDKIT_CACHE_GHA: true - run: | - cd integration-tests/test-repo - bluebuild generate-iso image ghcr.io/blue-build/cli/test:40 + run: just test-generate-iso-image iso-from-recipe: timeout-minutes: 60 @@ -591,7 +582,7 @@ jobs: packages: write id-token: write needs: - - build + - build-scripts if: github.repository == 'blue-build/cli' steps: @@ -613,19 +604,15 @@ jobs: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} - - name: Install bluebuild - run: | - cargo install --path . --debug --all-features - - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v3 + - uses: extractions/setup-just@v1 + - name: Run Build env: GH_TOKEN: ${{ github.token }} GH_PR_EVENT_NUMBER: ${{ github.event.number }} COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }} BB_BUILDKIT_CACHE_GHA: true - run: | - cd integration-tests/test-repo - bluebuild generate-iso -vv recipe recipes/recipe.yml + run: just test-generate-iso-image diff --git a/Cargo.lock b/Cargo.lock index 98b612a6..d629c79d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -321,6 +321,7 @@ dependencies = [ "blue-build-template", "blue-build-utils", "bon", + "cached", "clap", "clap-verbosity-flag", "clap_complete", diff --git a/Cargo.toml b/Cargo.toml index b2f6e821..ecac5147 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ version = "0.8.20" [workspace.dependencies] bon = "2" +cached = "0.53" chrono = "0.4" clap = "4" colored = "2" @@ -71,6 +72,7 @@ requestty = { version = "0.5", features = ["macros", "termion"] } shadow-rs = "0.26" urlencoding = "2" +cached.workspace = true clap = { workspace = true, features = ["derive", "cargo", "unicode", "env"] } colored.workspace = true indicatif.workspace = true diff --git a/Earthfile b/Earthfile index e9da8397..b6a29c85 100644 --- a/Earthfile +++ b/Earthfile @@ -12,12 +12,6 @@ all: BUILD +build BUILD ./integration-tests+all -build: - WAIT - BUILD --platform=linux/amd64 --platform=linux/arm64 +build-scripts - END - BUILD --platform=linux/amd64 --platform=linux/arm64 +build-images - run-checks: BUILD +lint BUILD +test @@ -103,7 +97,7 @@ build-scripts: chmod +x "scripts/${script}" END - DO --pass-args +SAVE_IMAGE --SUFFIX="-build-scripts" + DO --pass-args +SAVE_IMAGE --IMAGE="$IMAGE/build-scripts" blue-build-cli-prebuild: ARG BASE_IMAGE="registry.fedoraproject.org/fedora-toolbox" @@ -229,6 +223,7 @@ INSTALL: SAVE_IMAGE: FUNCTION ARG SUFFIX="" + ARG IMAGE="$IMAGE" ARG TAGGED="false" COPY --platform=native +version/version / diff --git a/integration-tests/test-repo/recipes/akmods.yml b/integration-tests/test-repo/recipes/akmods.yml index 9feb668b..ffc076de 100644 --- a/integration-tests/test-repo/recipes/akmods.yml +++ b/integration-tests/test-repo/recipes/akmods.yml @@ -2,6 +2,5 @@ modules: # Tests installing rpms from a combo image stage - type: akmods install: - - nct6687d - openrazer - openrgb diff --git a/justfile b/justfile index f6242894..79462014 100644 --- a/justfile +++ b/justfile @@ -1,5 +1,3 @@ -#!/usr/bin/env just --justfile - export RUST_BACKTRACE := "1" set dotenv-load := true @@ -9,6 +7,20 @@ set positional-arguments := true default: @just --list +# Clean up development files and images +clean: + cargo clean + command -v docker \ + && docker buildx --builder bluebuild prune -f \ + && docker system prune -f \ + || true + command -v podman \ + && podman system prune -f \ + || true + command -v earthly \ + && earthly prune --reset \ + || true + # Install bluebuild using cargo with release optimization install: cargo install --path . @@ -69,9 +81,10 @@ watch-lint: watch-lint-all-features: cargo watch -c -x 'clippy --all-features' +# Expand the macros of a module for debugging expand *args: cargo expand $@ > ./expand.rs - echo "Expansion located in ./expand.rs" + $EDITOR ./expand.rs # Installs cargo tools that help with development tools: @@ -99,3 +112,111 @@ release *args: git tag "v${VERSION}" git push origin "v${VERSION}" gh release create --generate-notes --latest "v${VERSION}" + +should_push := if env('GITHUB_ACTIONS', '') != '' { + if env('COSIGN_PRIVATE_KEY', '') != '' { + '--push' + } else { + '' + } +} else { + '' +} + +# Run all integration tests +integration-tests: test-docker-build test-arm64-build test-podman-build test-buildah-build test-generate-iso-image test-generate-iso-recipe + +# Run docker driver integration test +test-docker-build: install-debug-all-features + #!/usr/bin/env bash + set -eu + cd integration-tests/test-repo + bluebuild build \ + --retry-push \ + -B docker \ + -I docker \ + -S sigstore \ + {{ should_push }} \ + -vv \ + recipes/recipe.yml recipes/recipe-39.yml + +# Run arm integration test +test-arm64-build: install-debug-all-features + #!/usr/bin/env bash + set -eu + cd integration-tests/test-repo + bluebuild build \ + --retry-push \ + --platform linux/arm64 \ + {{ should_push }} \ + -vv \ + recipes/recipe-arm64.yml + +# Run docker driver external login integration test +test-docker-build-external-login: install-debug-all-features + #!/usr/bin/env bash + set -eu + cd integration-tests/test-repo + bluebuild build \ + --retry-push \ + -S sigstore \ + {{ should_push }} \ + -vv \ + recipes/recipe.yml recipes/recipe-39.yml + +# Run docker driver oauth login integration test +test-docker-build-oauth-login: install-debug-all-features + #!/usr/bin/env bash + set -eu + cd integration-tests/test-repo + bluebuild build \ + --registry us-east1-docker.pkg.dev \ + --registry-namespace bluebuild-oidc/bluebuild \ + --retry-push \ + {{ should_push }} \ + -vv \ + recipes/recipe.yml recipes/recipe-39.yml + +# Run podman driver integration test +test-podman-build: install-debug-all-features + #!/usr/bin/env bash + set -eu + cd integration-tests/test-repo + bluebuild build \ + --retry-push \ + -B podman \ + -I podman \ + -S sigstore \ + {{ should_push }} \ + -vv \ + recipes/recipe.yml recipes/recipe-39.yml + +# Run buildah driver integration test +test-buildah-build: install-debug-all-features + #!/usr/bin/env bash + set -eu + cd integration-tests/test-repo + bluebuild build \ + --retry-push \ + -B buildah \ + -I podman \ + -S sigstore \ + {{ should_push }} \ + -vv \ + recipes/recipe.yml recipes/recipe-39.yml + +# Run ISO generator for images +test-generate-iso-image: install-debug-all-features + #!/usr/bin/env bash + set -eu + ISO_OUT=$(mktemp -d) + bluebuild generate-iso -vv --output-dir "$ISO_OUT" image ghcr.io/blue-build/cli/test:40 + +# Run ISO generator for images +test-generate-iso-recipe: install-debug-all-features + #!/usr/bin/env bash + set -eu + ISO_OUT=$(mktemp -d) + cd integration-tests/test-repo + bluebuild generate-iso -vv --output-dir "$ISO_OUT" recipe recipes/recipe.yml + diff --git a/process/Cargo.toml b/process/Cargo.toml index e251b494..18e43158 100644 --- a/process/Cargo.toml +++ b/process/Cargo.toml @@ -28,6 +28,7 @@ sigstore = { version = "0.10", features = ["full-rustls-tls", "cached-client", " tokio = { version = "1.39.2", features = ["rt", "rt-multi-thread"], optional = true } zeroize = { version = "1", features = ["aarch64", "derive", "serde"] } +cached.workspace = true chrono.workspace = true clap = { workspace = true, features = ["derive", "env"] } colored.workspace = true @@ -42,7 +43,6 @@ tempdir.workspace = true bon.workspace = true users.workspace = true uuid.workspace = true -cached = "0.53.1" [dev-dependencies] rstest.workspace = true diff --git a/process/drivers/opts/inspect.rs b/process/drivers/opts/inspect.rs index 0f75422d..169c2aa6 100644 --- a/process/drivers/opts/inspect.rs +++ b/process/drivers/opts/inspect.rs @@ -5,6 +5,7 @@ use bon::Builder; use crate::drivers::types::Platform; #[derive(Debug, Clone, Builder)] +#[builder(derive(Clone))] pub struct GetMetadataOpts<'scope> { #[builder(into)] pub image: Cow<'scope, str>, diff --git a/process/drivers/types.rs b/process/drivers/types.rs index 4285ea26..7395ec79 100644 --- a/process/drivers/types.rs +++ b/process/drivers/types.rs @@ -170,7 +170,7 @@ impl DetermineDriver for Option { } } -#[derive(Debug, Default, Clone, Copy, ValueEnum)] +#[derive(Debug, Default, Clone, Copy, ValueEnum, PartialEq, Eq, Hash)] pub enum Platform { #[default] #[value(name = "native")] diff --git a/src/commands/generate.rs b/src/commands/generate.rs index e4adf398..95736771 100644 --- a/src/commands/generate.rs +++ b/src/commands/generate.rs @@ -3,14 +3,17 @@ use std::{ path::{Path, PathBuf}, }; -use blue_build_process_management::drivers::{types::Platform, CiDriver, Driver, DriverArgs}; +use blue_build_process_management::drivers::{ + opts::GetMetadataOpts, types::Platform, CiDriver, Driver, DriverArgs, InspectDriver, +}; use blue_build_recipe::Recipe; use blue_build_template::{ContainerFileTemplate, Template}; use blue_build_utils::{ - constants::{CONFIG_PATH, RECIPE_FILE, RECIPE_PATH}, + constants::{BUILD_SCRIPTS_IMAGE_REF, CONFIG_PATH, RECIPE_FILE, RECIPE_PATH}, syntax_highlighting::{self, DefaultThemes}, }; use bon::Builder; +use cached::proc_macro::cached; use clap::{crate_version, Args}; use log::{debug, info, trace, warn}; use miette::{IntoDiagnostic, Result}; @@ -132,17 +135,7 @@ impl GenerateCommand { .recipe_path(recipe_path.as_path()) .registry(registry) .repo(Driver::get_repo_url()?) - .exports_tag({ - #[allow(clippy::const_is_empty)] - if shadow::COMMIT_HASH.is_empty() { - // This is done for users who install via - // cargo. Cargo installs do not carry git - // information via shadow - format!("v{}", crate_version!()) - } else { - shadow::COMMIT_HASH.to_string() - } - }) + .build_scripts_image(determine_scripts_tag(self.platform)?) .build(); let output_str = template.render().into_diagnostic()?; @@ -159,3 +152,31 @@ impl GenerateCommand { Ok(()) } } + +#[cached( + result = true, + key = "Platform", + convert = r#"{ platform }"#, + sync_writes = true +)] +fn determine_scripts_tag(platform: Platform) -> Result { + let version = format!("v{}", crate_version!()); + let opts = GetMetadataOpts::builder() + .image(BUILD_SCRIPTS_IMAGE_REF) + .platform(platform); + + Driver::get_metadata(&opts.clone().tag(shadow::COMMIT_HASH).build()) + .inspect_err(|e| trace!("{e:?}")) + .map(|_| format!("{BUILD_SCRIPTS_IMAGE_REF}:{}", shadow::COMMIT_HASH)) + .or_else(|_| { + Driver::get_metadata(&opts.clone().tag(shadow::BRANCH).build()) + .inspect_err(|e| trace!("{e:?}")) + .map(|_| format!("{BUILD_SCRIPTS_IMAGE_REF}:{}", shadow::BRANCH)) + }) + .or_else(|_| { + Driver::get_metadata(&opts.tag(&version).build()) + .inspect_err(|e| trace!("{e:?}")) + .map(|_| format!("{BUILD_SCRIPTS_IMAGE_REF}:{version}")) + }) + .inspect(|image| debug!("Using build scripts image: {image}")) +} diff --git a/template/src/lib.rs b/template/src/lib.rs index 1ffc87e6..d9f9eb15 100644 --- a/template/src/lib.rs +++ b/template/src/lib.rs @@ -25,7 +25,7 @@ pub struct ContainerFileTemplate<'a> { build_id: Uuid, os_version: u64, registry: Cow<'a, str>, - exports_tag: Cow<'a, str>, + build_scripts_image: Cow<'a, str>, repo: Cow<'a, str>, } diff --git a/template/templates/Containerfile.j2 b/template/templates/Containerfile.j2 index 9d51e78e..10a52422 100644 --- a/template/templates/Containerfile.j2 +++ b/template/templates/Containerfile.j2 @@ -38,7 +38,7 @@ RUN --mount=type=bind,from=stage-bins,src=/bins,dst=/tmp/bins \ {% call modules::main_modules_run(recipe.modules_ext, os_version) %} -RUN --mount=type=bind,from=ghcr.io/blue-build/cli:{{ exports_tag }}-build-scripts,src=/scripts/,dst=/scripts/ \ +RUN --mount=type=bind,from={{ build_scripts_image }},src=/scripts/,dst=/scripts/ \ /scripts/post_build.sh # Labels are added last since they cause cache misses with buildah diff --git a/template/templates/modules/modules.j2 b/template/templates/modules/modules.j2 index ef21d3c0..01adef73 100644 --- a/template/templates/modules/modules.j2 +++ b/template/templates/modules/modules.j2 @@ -25,7 +25,7 @@ RUN \ {%- if module.module_type == "akmods" %} --mount=type=bind,from=stage-akmods-{{ module.generate_akmods_info(os_version).stage_name }},src=/rpms,dst=/tmp/rpms,rw \ {%- endif %} - --mount=type=bind,from=ghcr.io/blue-build/cli:{{ exports_tag }}-build-scripts,src=/scripts/,dst=/tmp/scripts/ \ + --mount=type=bind,from={{ build_scripts_image }},src=/scripts/,dst=/tmp/scripts/ \ --mount=type=cache,dst=/var/cache/rpm-ostree,id=rpm-ostree-cache-{{ recipe.name }}-{{ recipe.image_version }},sharing=locked \ /tmp/scripts/run_module.sh '{{ module.module_type }}' '{{ module|json|safe }}' \ && ostree container commit @@ -58,7 +58,7 @@ RUN \ {%- else %} --mount=type=bind,from=stage-modules,src=/modules,dst=/tmp/modules,rw \ {%- endif %} - --mount=type=bind,from=ghcr.io/blue-build/cli:{{ exports_tag }}-build-scripts,src=/scripts/,dst=/tmp/scripts/ \ + --mount=type=bind,from={{ build_scripts_image }},src=/scripts/,dst=/tmp/scripts/ \ /tmp/scripts/run_module.sh '{{ module.module_type }}' '{{ module|json|safe }}' {%- endif %} {%- endif %} diff --git a/template/templates/stages.j2 b/template/templates/stages.j2 index 505fa691..bbcddaf4 100644 --- a/template/templates/stages.j2 +++ b/template/templates/stages.j2 @@ -62,7 +62,7 @@ ARG RUST_LOG_STYLE=always {%- if stage.from != "scratch" %} # Add compatibility for modules RUN --mount=type=bind,from=stage-bins,src=/bins/,dst=/tmp/bins/ \ - --mount=type=bind,from=ghcr.io/blue-build/cli:{{ exports_tag }}-build-scripts,src=/scripts/,dst=/tmp/scripts/ \ + --mount=type=bind,from={{ build_scripts_image }},src=/scripts/,dst=/tmp/scripts/ \ /tmp/scripts/setup.sh {%- if files_dir_exists %} diff --git a/utils/src/constants.rs b/utils/src/constants.rs index 65733f68..4fb6e784 100644 --- a/utils/src/constants.rs +++ b/utils/src/constants.rs @@ -72,6 +72,7 @@ pub const LC_TERMINAL_VERSION: &str = "LC_TERMINAL_VERSION"; pub const XDG_RUNTIME_DIR: &str = "XDG_RUNTIME_DIR"; // Misc +pub const BUILD_SCRIPTS_IMAGE_REF: &str = "ghcr.io/blue-build/cli/build-scripts"; pub const COSIGN_IMAGE: &str = "ghcr.io/sigstore/cosign/cosign:latest"; pub const OCI_ARCHIVE: &str = "oci-archive"; pub const OSTREE_IMAGE_SIGNED: &str = "ostree-image-signed";