[backport] CI refactor to v2.1.x branch (#2146)

* Fix stackable sources for build-iso (#2061) * Fix stackable sources for build-iso * Move syncFunc as a parameter of DumpSource Signed-off-by: David Cassany <dcassany@suse.com> (cherry picked from commit 22baf14) * Fix error return Signed-off-by: David Cassany <dcassany@suse.com> (cherry picked from commit f9934d7) * Change default root password Signed-off-by: David Cassany <dcassany@suse.com> (cherry picked from commit 46a88e1) * Allow key pair of ssh connection in tests Signed-off-by: David Cassany <dcassany@suse.com> (cherry picked from commit a5f526d) * Set legacy password for upgrade test Signed-off-by: David Cassany <dcassany@suse.com> (cherry picked from commit 6f0b28e) * Allow insecure registries (#2131) * Allow skipping TLS verification on registries Signed-off-by: David Cassany <dcassany@suse.com> * Improve error logging Signed-off-by: David Cassany <dcassany@suse.com> * Fix flag default value Signed-off-by: David Cassany <dcassany@suse.com> * Fix image extractor mock and use default TLS verification for tests Signed-off-by: David Cassany <dcassany@suse.com> --------- Signed-off-by: David Cassany <dcassany@suse.com> (cherry picked from commit 22996ef) * Fix permissions * Set token to explicit content read permissions (cherry picked from commit 3d657c9) * Move permissions at top level (cherry picked from commit 8b8ae12) * Fix workflows, permissions at top level (cherry picked from commit d3da1cf) * Leftover from d3da1cf (cherry picked from commit 3206436) * Fix nested permissions definition Top level permissions can't be increased for specific jobs, hence setting the permission on each specific job on workflows that require more fine grain approach. In addition it removes the pull-request permission as this is mostly required for PR decorators, read/write labels, etc. (cherry picked from commit 1a0903a) * Refactor ci (#2137) * Refactor CI to use a local registry * Cache image and fetch image tarballs * Composite fetch images * Push to insecure registry with podman * Simplify workflow runs * Improve cache cleanup * Improve ip detection Signed-off-by: David Cassany <dcassany@suse.com> (cherry picked from commit 644cdc7) * Fix cache cleaning for PRs and main pushes (#2144) * Fix cache cleaning for PRs and main pushes * Update .github/workflows/cache-cleanup.yaml Co-authored-by: Francesco Giudici <francesco.giudici@gmail.com> Signed-off-by: David Cassany Viladomat <dcassany@suse.com> (cherry picked from commit f54f683) * Remove unused steps in workflow Signed-off-by: David Cassany <dcassany@suse.com> (cherry picked from commit 51a85a7) (cherry picked from commit 9f8d14fea58c2e458f8840d274422352f0b0b1c2) * Do not fetch upstream go dependencies Signed-off-by: David Cassany <dcassany@suse.com> * Update cmd/cloud-init.go Missing err Co-authored-by: Francesco Giudici <francesco.giudici@gmail.com> Signed-off-by: David Cassany Viladomat <dcassany@suse.com> --------- Signed-off-by: David Cassany <dcassany@suse.com> Signed-off-by: David Cassany Viladomat <dcassany@suse.com> Co-authored-by: Francesco Giudici <francesco.giudici@gmail.com>
rancher · Jul 29, 2024 · 216ab93 · 216ab93
1 parent e536b50
commit 216ab93
Show file tree

Hide file tree

Showing 46 changed files with 718 additions and 408 deletions.
diff --git a/.github/actions/fetchimages/action.yml b/.github/actions/fetchimages/action.yml
@@ -0,0 +1,58 @@
+name: Pull images
+descriptions: Fetches and loads images from the cache
+
+inputs:
+  version:
+    description: key of the cache and tag of the images
+    required: true
+    type: string
+  flavor:
+    description: flavor of the OS image
+    required: false
+    type: string
+  toolkit:
+    description: fetch toolkit image
+    required: false
+    default: 'true'
+    type: 'boolean'
+  os:
+    description: fetch OS image
+    required: false
+    default: 'true'
+    type: 'boolean'
+
+runs:
+  using: composite
+  steps:
+    - if: ${{ inputs.toolkit == 'true' }}
+      name: Fetch toolkit image
+      id: cache-toolkit
+      uses: actions/cache/restore@v4
+      env:
+        cache-name: toolkit-build-x86_64-${{ github.event_name }}
+      with:
+        path: /tmp/toolkit.tar
+        key: ${{ env.cache-name }}-${{ inputs.version }}
+        fail-on-cache-miss: true
+    - if: ${{ inputs.toolkit == 'true' }}
+      name: Load toolkit image
+      id: load-toolkit
+      shell: bash
+      run: |
+        docker load -i /tmp/toolkit.tar
+    - if: ${{ inputs.os == 'true' }}
+      name: Fetch OS image
+      id: cache-os
+      uses: actions/cache/restore@v4
+      env:
+        cache-name: os-build-x86_64-${{ inputs.flavor }}-${{ github.event_name }}
+      with:
+        path: /tmp/os.tar
+        key: ${{ env.cache-name }}-${{ inputs.version }}
+        fail-on-cache-miss: true
+    - if: ${{ inputs.os == 'true' }}
+      name: Load OS image
+      id: load-os
+      shell: bash
+      run: |
+          docker load -i /tmp/os.tar
diff --git a/.github/actions/version/action.yml b/.github/actions/version/action.yml
@@ -0,0 +1,20 @@
+name: Version hash
+descriptions: Computes the version hash from the current context
+
+outputs:
+  version:
+    description: computed hash from current context
+    value: ${{ steps.version.outputs.version }}
+
+runs:
+  using: composite
+  steps:
+    - name: Define version
+      id: version
+      shell: bash
+      env:
+        hash: ${{ hashFiles('Dockerfile', '**/go.sum', '**/pkg/**', '**/examples/**', '**/cmd/**', '**/vendor/**', '**/Makefile', '**/main.go') }}
+      run: |
+        version="${{ env.hash }}"
+        version=${version::16}
+        echo "version=${version}" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -1,26 +1,18 @@
 name: Build
 on:
+  pull_request:
   push:
-    tags:
-      - v**
-  pull_request_target:
-    types:
-      - opened
-      - synchronize
-      - reopened
-    paths:
-      - tests/**
-      - make/**
-      - Makefile
-      - .github/**
-      - pkg/**
-      - cmd/**
-      - go.mod
-      - go.sum
-      - examples/**
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}
+  cancel-in-progress: true
 
 jobs:
   detect:
+    permissions:
+      contents: read
     runs-on: ubuntu-latest
     outputs:
       flavor: ${{ steps.set-matrix.outputs.flavor }}
@@ -42,29 +34,49 @@ jobs:
         fi
 
   build-toolkit:
+    permissions:
+      contents: read
     needs:
       - detect
-    permissions:
-      packages: write
     runs-on: ubuntu-latest
     env:
       PLATFORM: ${{ needs.detect.outputs.platform }}
-      TOOLKIT_REPO: ghcr.io/${{github.repository}}/elemental-cli 
+    outputs:
+      version: ${{ steps.version.outputs.version }}
     steps:
       - uses: actions/checkout@v4
         with:
           ref: "${{ github.event.pull_request.head.sha }}"
       - run: |
           git fetch --prune --unshallow
-      - name: Log in to ghcr.io
-        uses: docker/login-action@v3
+      - name: Define version
+        id: version
+        uses: ./.github/actions/version
+      - name: Check cache for Toolkit image
+        id: cache-toolkit
+        uses: actions/cache/restore@v4
+        env:
+          cache-name: toolkit-build-x86_64-${{ github.event_name }}
+          lookup-only: true
         with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Build toolkit
+          path: /tmp/toolkit.tar
+          key: ${{ env.cache-name }}-${{ steps.version.outputs.version }}
+      - if: ${{ steps.cache-toolkit.outputs.cache-hit != 'true' }} 
+        name: Build toolkit
+        env:
+          VERSION: ${{ steps.version.outputs.version }} 
         run: |
-          make DOCKER_ARGS=--push build
+          make build-save
+          mv build/elemental-toolkit*.tar /tmp/toolkit.tar
+      - if: ${{ steps.cache-toolkit.outputs.cache-hit != 'true' }}
+        name: Save toolkit image in cache
+        id: save-toolkit
+        uses: actions/cache/save@v4
+        env:
+          cache-name: toolkit-build-x86_64-${{ github.event_name }}
+        with:
+          path: /tmp/toolkit.tar
+          key: ${{ env.cache-name }}-${{ steps.version.outputs.version }}
 
   build-matrix:
     needs:
@@ -77,3 +89,4 @@ jobs:
     uses: ./.github/workflows/build_and_test_x86.yaml
     with:
       flavor: ${{ matrix.flavor }}
+      version: ${{ needs.build-toolkit.outputs.version }}
diff --git a/.github/workflows/build_and_test_arm.yaml b/.github/workflows/build_and_test_arm.yaml
@@ -11,11 +11,13 @@ on:
         type: string
 
 concurrency:
-  group: ci-${{ inputs.flavor }}-aarch64-${{ github.head_ref || github.ref }}-${{ github.repository }}
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}-${{ inputs.flavor }}-aarch64
   cancel-in-progress: true
 
-jobs:
+permissions:
+  contents: read
 
+jobs:
   build-iso:
     needs: detect
     runs-on: [self-hosted, arm64]
@@ -120,9 +122,6 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
-      - name: Install deps
-        run: |
-          make test-deps
       - run: |
           git fetch --prune --unshallow
       - name: Cached Disk
@@ -174,9 +173,6 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
-      - name: Install deps
-        run: |
-          make test-deps
       - run: |
           git fetch --prune --unshallow
       - name: Cached ISO