Skip to content

Commit

Permalink
update builder
Browse files Browse the repository at this point in the history
  • Loading branch information
madebyTimo committed Nov 9, 2024
1 parent 8584f67 commit 9fd966e
Show file tree
Hide file tree
Showing 10 changed files with 250 additions and 152 deletions.
102 changes: 102 additions & 0 deletions .github/workflows/build-release.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
name: "build and release"

on:
pull_request:
push:
branches:
- "main"
schedule:
- cron: "00 01 * * *"
workflow_dispatch:

jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Create cache folder
run: |
sudo mkdir --parents /media/saved-cache
sudo chown -R "$(whoami)" /media/saved-cache
- name: Get cache
uses: actions/cache/restore@v4
with:
path: /media/saved-cache
key: docker-builder-build-cache-${{ github.run_id }}
restore-keys: docker-builder-build-cache
- name: Import cache
run: |
if (ls /media/saved-cache/*.tar.zst); then
docker run --pull always --rm \
--volume "/media/saved-cache:/media/saved-cache" \
--volume "docker-builder-build-cache:/media/build-cache" \
--workdir /media \
madebytimo/scripts \
compress.sh --decompress /media/saved-cache/*.tar.zst
rm /media/saved-cache/*.tar.zst
fi
- name: Set secrets and variables
run: |
mkdir data-local
echo "latest_version=$(git describe --tags --abbrev=0)" >> "$GITHUB_ENV"
echo "version=$(cat Version.txt)" >> "$GITHUB_ENV"
if [[ -n '${{ secrets.UNITY_LICENSE_FILE }}' ]]; then
echo '${{ secrets.UNITY_LICENSE_FILE }}' > data-local/unity-license.ulf
fi
if [[ -n '${{ secrets.DOCKER_REGISTRY_USERNAME }}' ]]; then
echo ${{ secrets.DOCKER_REGISTRY_PASSWORD }} | \
docker login --username ${{ secrets.DOCKER_REGISTRY_USERNAME }} \
--password-stdin ${{ secrets.DOCKER_REGISTRY_URL }}
fi
- name: Prepare environment
run: |
if [[ -f builder/docker.sh ]]; then
docker buildx create --use
fi
- name: Build
if: ${{ github.event_name != 'schedule'}}
run: |
if [[ ${{ github.ref }} == 'refs/heads/main' ]]; then
builder/build.sh --publish
else
builder/build.sh
fi
- name: Build update base
if: ${{ github.event_name == 'schedule'}}
run: builder/build.sh --publish --update-base
- name: Upload
uses: actions/upload-artifact@v4
with:
name: ${{ env.version }}
path: builds/*
- name: Release
if: ${{ github.ref == 'refs/heads/main' && env.latest_version != env.version }}
uses: softprops/action-gh-release@v2
with:
files: builds/*
tag_name: ${{ env.version }}
- name: Export cache
run: |
docker run --pull always --rm \
--volume "/media/saved-cache:/media/saved-cache" \
--volume "docker-builder-build-cache:/media/build-cache" \
madebytimo/scripts \
compress.sh --fast --output /media/saved-cache/build-cache /media/build-cache
sudo chown -R "$(whoami)" /media/saved-cache
- name: Delete old caches
env:
GH_TOKEN: ${{ github.token }}
run: |
for CACHE in $(gh cache list --key Factory-build-cache --ref ${{ github.ref}} \
| cut --fields 1); do
echo "Deleting cache \"$CACHE\"."
gh cache delete "$CACHE"
done
- name: Set cache
uses: actions/cache/save@v4
with:
path: /media/saved-cache
key: docker-builder-build-cache-${{ github.run_id }}
27 changes: 16 additions & 11 deletions .github/workflows/check-version-increment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,24 +10,29 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout new
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
path: new
- name: Checkout old
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
path: old
ref: refs/heads/main
- name: Setup Node.js
uses: actions/setup-node@v3
uses: actions/setup-node@v4
- name: Download script
run: |
curl --silent --output check-version-increment.sh \
https://gitlab.com/madebyTimo/scripts-development/-/raw/main/scripts/check-version-increment.sh && \
chmod +x check-version-increment.sh
curl --silent --output check-version-increment.sh \
https://gitlab.com/madebyTimo/scripts-development/-/raw/main/scripts/\
check-version-increment.sh \
&& chmod +x check-version-increment.sh
- name: Test version increment
if: startsWith(github.ref, '/refs/heads/feature/') || startsWith(github.ref, '/refs/heads/bugfix/')
run: ./check-version-increment.sh --file --new new/Version.txt --old old/Version.txt
- name: Test version same
if: ${{ !( startsWith(github.ref, '/refs/heads/feature/') || startsWith(github.ref, '/refs/heads/bugfix/') ) }}
run: "[[ $(cat new/Version.txt) == $(cat old/Version.txt) ]]"
run: |
BRANCH="${GITHUB_HEAD_REF#/ref/head}"
echo "Branch to check: $BRANCH"
if [[ "$BRANCH" == @(feature|bugfix)/* ]]; then
./check-version-increment.sh --file --new new/Version.txt \
--old old/Version.txt
else
[[ $(cat new/Version.txt) == $(cat old/Version.txt) ]]
fi
23 changes: 0 additions & 23 deletions .github/workflows/docker-build-and-push.yaml

This file was deleted.

20 changes: 0 additions & 20 deletions .github/workflows/docker-update-base-and-push.yaml

This file was deleted.

30 changes: 18 additions & 12 deletions .github/workflows/static-code-analysis.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,33 +9,30 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Get cache
uses: actions/cache/restore@v3
uses: actions/cache/restore@v4
with:
path: /media/saved-cache
key: docker-builder-build-cache-${{ github.run_id }}
restore-keys: docker-builder-build-cache
- name: Import cache
run: |
cat | \
if (ls /media/saved-cache/*.tar.zst); then
docker run --pull always --rm \
--volume "/media/saved-cache:/media/saved-cache" \
--volume "docker-builder-build-cache:/media/build-cache" \
--workdir /media \
madebytimo/scripts \
bash \
<< EOF
cd /media
if (ls /media/saved-cache/*.tar.zst); then
compress.sh --decompress /media/saved-cache/*.tar.zst
compress.sh --decompress /media/saved-cache/*.tar.zst
rm /media/saved-cache/*.tar.zst
fi
EOF
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Download script
run: |
curl --silent --output /usr/local/bin/static-code-analysis.sh \
https://gitlab.com/madebyTimo/scripts-development/-/raw/main/scripts/static-code-analysis.sh && \
chmod +x /usr/local/bin/static-code-analysis.sh
https://gitlab.com/madebyTimo/scripts-development/-/raw/main/scripts/\
static-code-analysis.sh \
&& chmod +x /usr/local/bin/static-code-analysis.sh
- name: Analyze
run: static-code-analysis.sh
env:
Expand All @@ -52,8 +49,17 @@ jobs:
madebytimo/scripts \
compress.sh --fast --output /media/saved-cache/build-cache /media/build-cache
sudo chown -R "$(whoami)" /media/saved-cache
- name: Delete old caches
env:
GH_TOKEN: ${{ github.token }}
run: |
for CACHE in $(gh cache list --key Factory-build-cache --ref ${{ github.ref}} \
| cut --fields 1); do
echo "Deleting cache \"$CACHE\"."
gh cache delete "$CACHE"
done
- name: Set cache
uses: actions/cache/save@v3
uses: actions/cache/save@v4
with:
path: /media/saved-cache
key: docker-builder-build-cache-${{ github.run_id }}
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# folders
.idea
.temp*
.vscode
builds
data-local
Expand All @@ -10,3 +11,6 @@ test-results
# files
.iml
package-lock.json

# exclude
!.vscode/launch.json
21 changes: 21 additions & 0 deletions builder/build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#!/usr/bin/env bash
set -e

SCRIPT_DIR="$(dirname "$(realpath "$0")")"

# help message
for ARGUMENT in "$@"; do
if [ "$ARGUMENT" == "-h" ] || [ "$ARGUMENT" == "--help" ]; then
echo "usage: $(basename "$0")"
echo "Run all scripts in the same folder."
echo "All arguments are passed to the build scripts."
exit
fi
done

mapfile -t BUILD_SCRIPTS -d '' < <(find "$SCRIPT_DIR" -name '*.sh' -not -name "$(basename "$0")")
for BUILD_SCRIPT in "${BUILD_SCRIPTS[@]}"; do
echo "Start \"$(basename "${BUILD_SCRIPT}")\""
"$BUILD_SCRIPT" "$@"
echo "Finished \"$(basename "${BUILD_SCRIPT}")\""
done
89 changes: 89 additions & 0 deletions builder/docker.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
#!/usr/bin/env bash
set -e

BUILD_ARGUMENTS=()
BUILDER_EXPORT_DOCKER_ARCHIVE="${BUILDER_EXPORT_DOCKER_ARCHIVE:-true}"
DEPENDENCIES=(docker zstd)
UPDATE_BASE=false
PLATFORMS=(amd64 arm64)
REGISTRY_USER="madebytimo"
APPLICATION_NAME="builder"

# help message
for ARGUMENT in "$@"; do
if [ "$ARGUMENT" == "-h" ] || [ "$ARGUMENT" == "--help" ]; then
echo "usage: $(basename "$0") [ARGUMENT]"
echo "Builds the docker image from the Dockerfile."
echo "ARGUMENT can be"
echo "--platform [amd64|arm64|arm] Build only for specified platform."
echo "--publish Push the build."
echo "--update-base Only build if newer base image is available."
exit
fi
done

# check dependencies
for CMD in "${DEPENDENCIES[@]}"; do
if [[ -z "$(which "$CMD")" ]]; then
echo "\"${CMD}\" is missing!"
exit 1
fi
done

# check arguments
while [[ -n "$1" ]]; do
if [[ "$1" = "--platform" ]]; then
shift
PLATFORMS=("$1")
elif [[ "$1" = "--update-base" ]]; then
UPDATE_BASE=true
elif [[ "$1" = "--publish" ]]; then
BUILD_ARGUMENTS+=("--push")
fi
shift
done

PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
VERSION="$(cat Version.txt)"

cd "$PROJECT_DIR"
mkdir --parents builds

BASE_IMAGE="$(tac Dockerfile | grep --max-count=1 "^FROM" | cut -d" " -f2)"
docker pull "$BASE_IMAGE"
BASE_IMAGE_DATE="$(docker image inspect --format="{{ .Created }}" "$BASE_IMAGE" | cut -d "T" -f1)"
echo "Base image is $BASE_IMAGE from $BASE_IMAGE_DATE"
IMAGE="${REGISTRY_USER}/${APPLICATION_NAME}"
if [[ "$UPDATE_BASE" == true ]]; then
docker pull "$IMAGE"
PUSHED_IMAGE_DATE="$(docker image inspect --format="{{ .Created }}" "$IMAGE" | cut -d "T" -f1)"
echo "Last pushed image is from $PUSHED_IMAGE_DATE"
if [[ "$BASE_IMAGE_DATE" < "$PUSHED_IMAGE_DATE" ]]; then
echo "Used base image is up to date"
exit;
fi
fi

PLATFORMS_STRING="${PLATFORMS[*]}"
BUILD_ARGUMENTS+=(--platform "${PLATFORMS_STRING// /,}")
OUTPUT_FILE="builds/${IMAGE//"/"/-}-${VERSION}-oci.tar"

docker buildx build "${BUILD_ARGUMENTS[@]}" --output \
"type=oci,dest=${OUTPUT_FILE},compression=zstd,compression-level=19,force-compression=true" \
--tag "${IMAGE}:latest" --tag "${IMAGE}:${VERSION}" \
--tag "${IMAGE}:${VERSION}-base-${BASE_IMAGE_DATE}" .

if [[ "$BUILDER_EXPORT_DOCKER_ARCHIVE" == true ]]; then
docker pull --quiet quay.io/skopeo/stable > /dev/null
rm -f builds/.temp-docker-archive.tar
for PLATFORM in "${PLATFORMS[@]}"; do
docker run --interactive --rm --volume "${PWD}/builds:/builds" \
quay.io/skopeo/stable copy --additional-tag "${IMAGE}:latest" --additional-tag \
"${IMAGE}:${VERSION}" --additional-tag "${IMAGE}:${VERSION}-base-${BASE_IMAGE_DATE}" \
--override-arch "$PLATFORM" --quiet "oci-archive:${OUTPUT_FILE}:latest" \
"docker-archive:builds/.temp-docker-archive.tar"
zstd -19 --force --quiet -T0 builds/.temp-docker-archive.tar \
-o "${OUTPUT_FILE%oci.tar}${PLATFORM}.tar.zst"
rm -f builds/.temp-docker-archive.tar
done
fi
22 changes: 0 additions & 22 deletions docker-bake.hcl

This file was deleted.

Loading

0 comments on commit 9fd966e

Please sign in to comment.