Merge branch 'dev' into dockerfiles-layers

ThePorgs · Jul 25, 2023 · 831ed6d · 831ed6d
2 parents dcec90f + cd52086
commit 831ed6d
Show file tree

Hide file tree

Showing 28 changed files with 696 additions and 55 deletions.
diff --git a/.github/workflows/entrypoint_nightly.yml b/.github/workflows/entrypoint_nightly.yml
@@ -5,7 +5,7 @@ on:
     branches:
       - dev
     paths-ignore:
-      - ".github/**"
+#      - ".github/**"
       - "**.md"
 
 env:
@@ -89,12 +89,117 @@ jobs:
           docker manifest push ${DOCKER_TARGET_REPO}:${IMAGE_BASE_NAME}-${IMAGE_VERSION}
           docker manifest rm ${DOCKER_TARGET_REPO}:${IMAGE_BASE_NAME}-${IMAGE_VERSION}
 
+  tools_list:
+    name: Export tools list
+    needs:
+      - varset
+      - publish
+      - build_belt
+    # only exporting tools list if publish was a success AND the tests were a success
+    if: always() && ( needs.publish.result == 'success' && !contains(needs.build_belt.outputs.build, 'failure'))
+    strategy:
+      fail-fast: false
+      # only 1 job at a time, else there will be a conflict. Runner 2 will pull exegol-docs before runner 1 pushes changes.
+      max-parallel: 1
+      matrix:
+        arch: [ arm64, amd64 ]
+    runs-on:
+      - self-hosted
+      - builder
+      - ${{ matrix.arch }}
+    steps:
+      - name: Checkout Exegol-images
+        uses: actions/checkout@v3
+      - name: Prepare image version
+        id: prepare
+        run: |
+          COMMIT_ID=$(git rev-parse "$GITHUB_SHA")
+          if [ "$IMAGE_VERSION" == "" ]; then
+            IMAGE_VERSION=${COMMIT_ID:0:8}
+          fi
+          echo "image_version=${IMAGE_VERSION}"
+          echo "image_version=${IMAGE_VERSION}" >> $GITHUB_OUTPUT
+      - name: Checkout Exegol-docs
+        uses: actions/checkout@v3
+        with:
+          repository: 'ThePorgs/Exegol-docs'
+          ref: 'dev-images'
+      - name: Inspect the built image
+        run: docker inspect ${{ needs.varset.outputs.DOCKER_BUILD_REPO }}:${{ needs.varset.outputs.IMAGE_BASE_NAME }}-${{ matrix.arch }}
+      - name: Create a container from the built image
+        run: |
+          docker run --name exegol-${{ needs.varset.outputs.IMAGE_BASE_NAME }}-${{ matrix.arch }} --rm -t -d ${{ needs.varset.outputs.DOCKER_BUILD_REPO }}:${{ needs.varset.outputs.IMAGE_BASE_NAME }}-${{ matrix.arch }} endless
+      - name: Export the tools list
+        if: success()
+        run: |
+          docker cp exegol-${{ needs.varset.outputs.IMAGE_BASE_NAME }}-${{ matrix.arch }}:/.exegol/installed_tools.csv installed_tools.csv
+      - name: Debug print installed_tools.csv
+        id: list_exists
+        run: cat installed_tools.csv
+      - name: Sanity check for installed_tools.csv
+        if: success()
+        run: |
+          grep -qE '([^,]*,[^,]*){3,}' installed_tools.csv \
+            && (echo '[-] Wrong number of columns on the following lines' \
+            && grep -oE '([^,]*,[^,]*){3,}' installed_tools.csv || exit 1) \
+            || (echo '[+] List contains right number of columns' && exit 0)
+      - name: Stop the container
+        if: always()
+        run: docker stop exegol-${{ needs.varset.outputs.IMAGE_BASE_NAME }}-${{ matrix.arch }}
+      - name: Adding list to repo and tables
+        if: always() && steps.list_exists.outcome == 'success'
+        run: |
+          mkdir -p source/assets/installed_tools/lists
+          echo '[*] [SHOULD BE NIGHTLY ONLY] Moving tools list to latest_nightly_arch.csv'
+          mv installed_tools.csv source/assets/installed_tools/lists/latest_nightly_${{ matrix.arch }}.csv
+          echo '[*] Changing lists.csv so that new tools list appears'
+          echo '[*] [SHOULD BE NIGHTLY ONLY] Removing occurences of nightly,whateverversion,arch so that only the latest nightly is shown'
+          (head -n 1 source/assets/installed_tools/lists.csv; \
+            echo "${{ needs.varset.outputs.IMAGE_BASE_NAME }},${{ steps.prepare.outputs.image_version }},${{ matrix.arch }},$(date -u +"%Y-%m-%dT%H:%M:%SZ"),:download:\`${{ needs.varset.outputs.IMAGE_BASE_NAME }}_${{ steps.prepare.outputs.image_version }}_${{ matrix.arch }}.csv \
+              </assets/installed_tools/lists/latest_nightly_${{ matrix.arch }}.csv>\`"; \
+            ( \
+              tail -n +2 source/assets/installed_tools/lists.csv | grep -Ev "${{ needs.varset.outputs.IMAGE_BASE_NAME }},[0-9a-fA-F]{8},${{ matrix.arch }}" \
+            ) \
+          ) | tee source/assets/installed_tools/new_lists.csv
+          mv source/assets/installed_tools/new_lists.csv source/assets/installed_tools/lists.csv
+      - name: Debug print lists.csv
+        if: always()
+        id: final_list_exists
+        run: cat source/assets/installed_tools/lists.csv
+      - name: Push Exegol-docs
+        if: always() && steps.final_list_exists.outcome == 'success'
+        env:
+          SSH_DEPLOY_KEY: ${{ secrets.EXEGOL_DOCS_SSH_DEPLOY_KEY }}
+        run: |
+          echo '[*] Setting up git env for SSH use'
+          mkdir -p "$HOME/.ssh"
+          DEPLOY_KEY_FILE="$HOME/.ssh/deploy_key"
+          echo "${SSH_DEPLOY_KEY}" > "$DEPLOY_KEY_FILE"
+          chmod 600 "$DEPLOY_KEY_FILE"
+          SSH_KNOWN_HOSTS_FILE="$HOME/.ssh/known_hosts"
+          ssh-keyscan -H github.com > "$SSH_KNOWN_HOSTS_FILE"
+          export GIT_SSH_COMMAND="ssh -i "$DEPLOY_KEY_FILE" -o UserKnownHostsFile=$SSH_KNOWN_HOSTS_FILE"
+          GIT_CMD_REPOSITORY="git@github.com:ThePorgs/Exegol-docs.git"
+          echo '[*] Setting git config'
+          git config --global user.name "exegol-images[pipeline]"
+          git config --global user.email "pipeline@exegol.images"
+          echo '[*] Staging changes'
+          echo '[*] [SHOULD BE NIGHTLY ONLY] Staging latest_nightly_arch.csv'
+          git add --verbose source/assets/installed_tools/lists/latest_nightly_${{ matrix.arch }}.csv
+          git add --verbose source/assets/installed_tools/lists.csv
+          echo '[*] Commiting changes'
+          git commit --verbose -m "PIPELINE: tools list for ${{ needs.varset.outputs.IMAGE_BASE_NAME }}_${{ steps.prepare.outputs.image_version }}_${{ matrix.arch }}"
+          echo '[*] Pushing changes'
+          git push --verbose "$GIT_CMD_REPOSITORY"
+
   clean_runners:
     name: Clean runner
     needs:
+      - tools_list
       - publish
       - build_belt
     # clean runners, if publish was a success, or at least if build succeeded
+    # we don't care if tools_list worked or not
     if: always() && ( needs.publish.result == 'success' || !contains(needs.build_belt.outputs.build, 'failure'))
     # even if this job fails, it won't affect the success/fail status of the whole workflow
     continue-on-error: true

diff --git a/.github/workflows/entrypoint_preprod_ad.yml b/.github/workflows/entrypoint_preprod_ad.yml
@@ -94,9 +94,112 @@ jobs:
           docker manifest push ${DOCKER_TARGET_REPO}:${IMAGE_BASE_NAME}-${IMAGE_VERSION}
           docker manifest rm ${DOCKER_TARGET_REPO}:${IMAGE_BASE_NAME}-${IMAGE_VERSION}
 
+  tools_list:
+    name: Export tools list
+    needs:
+      - varset
+      - publish
+      - build_belt
+    # only exporting tools list if publish was a success AND the tests were a success
+    if: always() && ( needs.publish.result == 'success' && !contains(needs.build_belt.outputs.build, 'failure'))
+    strategy:
+      fail-fast: false
+      # only 1 job at a time, else there will be a conflict. Runner 2 will pull exegol-docs before runner 1 pushes changes.
+      max-parallel: 1
+      matrix:
+        arch: [ arm64, amd64 ]
+    runs-on:
+      - self-hosted
+      - builder
+      - ${{ matrix.arch }}
+    steps:
+      - name: Checkout Exegol-images
+        uses: actions/checkout@v3
+      - name: Prepare image version
+        id: prepare
+        run: |
+          COMMIT_ID=$(git rev-parse "$GITHUB_SHA")
+          if [ "$IMAGE_VERSION" == "" ]; then
+            IMAGE_VERSION=${COMMIT_ID:0:8}
+          fi
+          echo "image_version=${IMAGE_VERSION}"
+          echo "image_version=${IMAGE_VERSION}" >> $GITHUB_OUTPUT
+      - name: Checkout Exegol-docs
+        uses: actions/checkout@v3
+        with:
+          repository: 'ThePorgs/Exegol-docs'
+          ref: 'dev-images'
+      - name: Inspect the built image
+        run: docker inspect ${{ needs.varset.outputs.DOCKER_BUILD_REPO }}:${{ needs.varset.outputs.IMAGE_BASE_NAME }}-${{ matrix.arch }}
+      - name: Create a container from the built image
+        run: |
+          docker run --name exegol-${{ needs.varset.outputs.IMAGE_BASE_NAME }}-${{ matrix.arch }} --rm -t -d ${{ needs.varset.outputs.DOCKER_BUILD_REPO }}:${{ needs.varset.outputs.IMAGE_BASE_NAME }}-${{ matrix.arch }} endless
+      - name: Export the tools list
+        if: success()
+        run: |
+          docker cp exegol-${{ needs.varset.outputs.IMAGE_BASE_NAME }}-${{ matrix.arch }}:/.exegol/installed_tools.csv installed_tools.csv
+      - name: Debug print installed_tools.csv
+        id: list_exists
+        run: cat installed_tools.csv
+      - name: Sanity check for installed_tools.csv
+        if: success()
+        run: |
+          grep -qE '([^,]*,[^,]*){3,}' installed_tools.csv \
+            && (echo '[-] Wrong number of columns on the following lines' \
+            && grep -oE '([^,]*,[^,]*){3,}' installed_tools.csv || exit 1) \
+            || (echo '[+] List contains right number of columns' && exit 0)
+      - name: Stop the container
+        if: always()
+        run: docker stop exegol-${{ needs.varset.outputs.IMAGE_BASE_NAME }}-${{ matrix.arch }}
+      - name: Adding list to repo and tables
+        if: always() && steps.list_exists.outcome == 'success'
+        run: |
+          mkdir -p source/assets/installed_tools/lists
+          echo '[*] Moving tools list to imagetag_version_arch.csv'
+          mv installed_tools.csv source/assets/installed_tools/lists/${{ needs.varset.outputs.IMAGE_BASE_NAME }}_${{ steps.prepare.outputs.image_version }}_${{ matrix.arch }}.csv
+          echo '[*] Changing lists.csv so that new tools list appears'
+          echo '[*] [SHOULD BE RELEASES ONLY] Removing occurences of image,version,arch. This is because this workflow runs in the prerelease pipeline, meaning that there's a possibility tools list is pushed to exegol-docs even if the images are not released for some reasons (e.g. imageA prerelease works but not for imageB). Doing this grep -v will ensure that lists.csv doesn't have duplicates to the same tag, same version and same arch'
+          (head -n 1 source/assets/installed_tools/lists.csv; \
+            echo "${{ needs.varset.outputs.IMAGE_BASE_NAME }},${{ steps.prepare.outputs.image_version }},${{ matrix.arch }},$(date -u +"%Y-%m-%dT%H:%M:%SZ"),:download:\`${{ needs.varset.outputs.IMAGE_BASE_NAME }}_${{ steps.prepare.outputs.image_version }}_${{ matrix.arch }}.csv \
+              </assets/installed_tools/lists/${{ needs.varset.outputs.IMAGE_BASE_NAME }}_${{ steps.prepare.outputs.image_version }}_${{ matrix.arch }}.csv>\`"; \
+            ( \
+              tail -n +2 source/assets/installed_tools/lists.csv | grep -Ev "${{ needs.varset.outputs.IMAGE_BASE_NAME }},${{ steps.prepare.outputs.image_version }},${{ matrix.arch }}" \
+            ) \
+          ) | tee source/assets/installed_tools/new_lists.csv
+          mv source/assets/installed_tools/new_lists.csv source/assets/installed_tools/lists.csv
+      - name: Debug print lists.csv
+        if: always()
+        id: final_list_exists
+        run: cat source/assets/installed_tools/lists.csv
+      - name: Push Exegol-docs
+        if: always() && steps.final_list_exists.outcome == 'success'
+        env:
+          SSH_DEPLOY_KEY: ${{ secrets.EXEGOL_DOCS_SSH_DEPLOY_KEY }}
+        run: |
+          echo '[*] Setting up git env for SSH use'
+          mkdir -p "$HOME/.ssh"
+          DEPLOY_KEY_FILE="$HOME/.ssh/deploy_key"
+          echo "${SSH_DEPLOY_KEY}" > "$DEPLOY_KEY_FILE"
+          chmod 600 "$DEPLOY_KEY_FILE"
+          SSH_KNOWN_HOSTS_FILE="$HOME/.ssh/known_hosts"
+          ssh-keyscan -H github.com > "$SSH_KNOWN_HOSTS_FILE"
+          export GIT_SSH_COMMAND="ssh -i "$DEPLOY_KEY_FILE" -o UserKnownHostsFile=$SSH_KNOWN_HOSTS_FILE"
+          GIT_CMD_REPOSITORY="git@github.com:ThePorgs/Exegol-docs.git"
+          echo '[*] Setting git config'
+          git config --global user.name "exegol-images[pipeline]"
+          git config --global user.email "pipeline@exegol.images"
+          echo '[*] Staging changes'
+          git add --verbose source/assets/installed_tools/lists/${{ needs.varset.outputs.IMAGE_BASE_NAME }}_${{ steps.prepare.outputs.image_version }}_${{ matrix.arch }}.csv
+          git add --verbose source/assets/installed_tools/lists.csv
+          echo '[*] Commiting changes'
+          git commit --verbose -m "PIPELINE: tools list for ${{ needs.varset.outputs.IMAGE_BASE_NAME }}_${{ steps.prepare.outputs.image_version }}_${{ matrix.arch }}"
+          echo '[*] Pushing changes'
+          git push --verbose "$GIT_CMD_REPOSITORY"
+
   clean_runners:
     name: Clean runner
     needs:
+      - tools_list
       - publish
       - build_belt
     # clean runners, if publish was a success, or at least if build succeeded

diff --git a/.github/workflows/entrypoint_preprod_full.yml b/.github/workflows/entrypoint_preprod_full.yml
@@ -94,9 +94,112 @@ jobs:
           docker manifest push ${DOCKER_TARGET_REPO}:${IMAGE_BASE_NAME}-${IMAGE_VERSION}
           docker manifest rm ${DOCKER_TARGET_REPO}:${IMAGE_BASE_NAME}-${IMAGE_VERSION}
 
+  tools_list:
+    name: Export tools list
+    needs:
+      - varset
+      - publish
+      - build_belt
+    # only exporting tools list if publish was a success AND the tests were a success
+    if: always() && ( needs.publish.result == 'success' && !contains(needs.build_belt.outputs.build, 'failure'))
+    strategy:
+      fail-fast: false
+      # only 1 job at a time, else there will be a conflict. Runner 2 will pull exegol-docs before runner 1 pushes changes.
+      max-parallel: 1
+      matrix:
+        arch: [ arm64, amd64 ]
+    runs-on:
+      - self-hosted
+      - builder
+      - ${{ matrix.arch }}
+    steps:
+      - name: Checkout Exegol-images
+        uses: actions/checkout@v3
+      - name: Prepare image version
+        id: prepare
+        run: |
+          COMMIT_ID=$(git rev-parse "$GITHUB_SHA")
+          if [ "$IMAGE_VERSION" == "" ]; then
+            IMAGE_VERSION=${COMMIT_ID:0:8}
+          fi
+          echo "image_version=${IMAGE_VERSION}"
+          echo "image_version=${IMAGE_VERSION}" >> $GITHUB_OUTPUT
+      - name: Checkout Exegol-docs
+        uses: actions/checkout@v3
+        with:
+          repository: 'ThePorgs/Exegol-docs'
+          ref: 'dev-images'
+      - name: Inspect the built image
+        run: docker inspect ${{ needs.varset.outputs.DOCKER_BUILD_REPO }}:${{ needs.varset.outputs.IMAGE_BASE_NAME }}-${{ matrix.arch }}
+      - name: Create a container from the built image
+        run: |
+          docker run --name exegol-${{ needs.varset.outputs.IMAGE_BASE_NAME }}-${{ matrix.arch }} --rm -t -d ${{ needs.varset.outputs.DOCKER_BUILD_REPO }}:${{ needs.varset.outputs.IMAGE_BASE_NAME }}-${{ matrix.arch }} endless
+      - name: Export the tools list
+        if: success()
+        run: |
+          docker cp exegol-${{ needs.varset.outputs.IMAGE_BASE_NAME }}-${{ matrix.arch }}:/.exegol/installed_tools.csv installed_tools.csv
+      - name: Debug print installed_tools.csv
+        id: list_exists
+        run: cat installed_tools.csv
+      - name: Sanity check for installed_tools.csv
+        if: success()
+        run: |
+          grep -qE '([^,]*,[^,]*){3,}' installed_tools.csv \
+            && (echo '[-] Wrong number of columns on the following lines' \
+            && grep -oE '([^,]*,[^,]*){3,}' installed_tools.csv || exit 1) \
+            || (echo '[+] List contains right number of columns' && exit 0)
+      - name: Stop the container
+        if: always()
+        run: docker stop exegol-${{ needs.varset.outputs.IMAGE_BASE_NAME }}-${{ matrix.arch }}
+      - name: Adding list to repo and tables
+        if: always() && steps.list_exists.outcome == 'success'
+        run: |
+          mkdir -p source/assets/installed_tools/lists
+          echo '[*] Moving tools list to imagetag_version_arch.csv'
+          mv installed_tools.csv source/assets/installed_tools/lists/${{ needs.varset.outputs.IMAGE_BASE_NAME }}_${{ steps.prepare.outputs.image_version }}_${{ matrix.arch }}.csv
+          echo '[*] Changing lists.csv so that new tools list appears'
+          echo '[*] [SHOULD BE RELEASES ONLY] Removing occurences of image,version,arch. This is because this workflow runs in the prerelease pipeline, meaning that there's a possibility tools list is pushed to exegol-docs even if the images are not released for some reasons (e.g. imageA prerelease works but not for imageB). Doing this grep -v will ensure that lists.csv doesn't have duplicates to the same tag, same version and same arch'
+          (head -n 1 source/assets/installed_tools/lists.csv; \
+            echo "${{ needs.varset.outputs.IMAGE_BASE_NAME }},${{ steps.prepare.outputs.image_version }},${{ matrix.arch }},$(date -u +"%Y-%m-%dT%H:%M:%SZ"),:download:\`${{ needs.varset.outputs.IMAGE_BASE_NAME }}_${{ steps.prepare.outputs.image_version }}_${{ matrix.arch }}.csv \
+              </assets/installed_tools/lists/${{ needs.varset.outputs.IMAGE_BASE_NAME }}_${{ steps.prepare.outputs.image_version }}_${{ matrix.arch }}.csv>\`"; \
+            ( \
+              tail -n +2 source/assets/installed_tools/lists.csv | grep -Ev "${{ needs.varset.outputs.IMAGE_BASE_NAME }},${{ steps.prepare.outputs.image_version }},${{ matrix.arch }}" \
+            ) \
+          ) | tee source/assets/installed_tools/new_lists.csv
+          mv source/assets/installed_tools/new_lists.csv source/assets/installed_tools/lists.csv
+      - name: Debug print lists.csv
+        if: always()
+        id: final_list_exists
+        run: cat source/assets/installed_tools/lists.csv
+      - name: Push Exegol-docs
+        if: always() && steps.final_list_exists.outcome == 'success'
+        env:
+          SSH_DEPLOY_KEY: ${{ secrets.EXEGOL_DOCS_SSH_DEPLOY_KEY }}
+        run: |
+          echo '[*] Setting up git env for SSH use'
+          mkdir -p "$HOME/.ssh"
+          DEPLOY_KEY_FILE="$HOME/.ssh/deploy_key"
+          echo "${SSH_DEPLOY_KEY}" > "$DEPLOY_KEY_FILE"
+          chmod 600 "$DEPLOY_KEY_FILE"
+          SSH_KNOWN_HOSTS_FILE="$HOME/.ssh/known_hosts"
+          ssh-keyscan -H github.com > "$SSH_KNOWN_HOSTS_FILE"
+          export GIT_SSH_COMMAND="ssh -i "$DEPLOY_KEY_FILE" -o UserKnownHostsFile=$SSH_KNOWN_HOSTS_FILE"
+          GIT_CMD_REPOSITORY="git@github.com:ThePorgs/Exegol-docs.git"
+          echo '[*] Setting git config'
+          git config --global user.name "exegol-images[pipeline]"
+          git config --global user.email "pipeline@exegol.images"
+          echo '[*] Staging changes'
+          git add --verbose source/assets/installed_tools/lists/${{ needs.varset.outputs.IMAGE_BASE_NAME }}_${{ steps.prepare.outputs.image_version }}_${{ matrix.arch }}.csv
+          git add --verbose source/assets/installed_tools/lists.csv
+          echo '[*] Commiting changes'
+          git commit --verbose -m "PIPELINE: tools list for ${{ needs.varset.outputs.IMAGE_BASE_NAME }}_${{ steps.prepare.outputs.image_version }}_${{ matrix.arch }}"
+          echo '[*] Pushing changes'
+          git push --verbose "$GIT_CMD_REPOSITORY"
+
   clean_runners:
     name: Clean runner
     needs:
+      - tools_list
       - publish
       - build_belt
     # clean runners, if publish was a success, or at least if build succeeded