diff --git a/.github/actions/setup_cache_for_template/action.yml b/.github/actions/setup_cache_for_template/action.yml index 6632e1bce6ab..19c60d80779a 100644 --- a/.github/actions/setup_cache_for_template/action.yml +++ b/.github/actions/setup_cache_for_template/action.yml @@ -7,10 +7,6 @@ inputs: template: description: template yaml file required: true - detect-containerd: - description: detect containerd usage from template by using limactl validate - required: false - default: 'true' runs: using: "composite" steps: @@ -29,120 +25,35 @@ runs: id: cache-params-from-template run: | set -eux - arch="${{ inputs.arch }}" - template="${{ inputs.template }}" - detect_containerd="${{ inputs.detect-containerd }}" - if [[ $detect_containerd == "true" ]] && ! command -v limactl &>/dev/null; then - echo "containerd detection is disabled because limactl is not found" >&2 - detect_containerd="false" - fi - case "$template" in - https://*) - tmp_yaml=$(mktemp -d)/template.yaml - curl -sSLf "$template" > "$tmp_yaml" || exit 1 - template=$tmp_yaml - ;; - *) - test -f "$template" || exit 1 - ;; - esac - - # detect arch from template if not provided - arch="${arch:-$(yq '.arch // ""' "$template")}" - arch="${arch:-$(uname -m)}" - # normalize arch. amd64 -> x86_64, arm64 -> aarch64 - case "$arch" in - amd64) arch=x86_64 ;; - arm64) arch=aarch64 ;; - esac - - # extract digest, location and size by parsing template using arch - readonly yq_filter=" - [ - .images | map(select(.arch == \"${arch}\")) | [.[0,1].location, .[0,1].digest], - .containerd|[.system or .user], - .containerd.archives | map(select(.arch == \"${arch}\")) | [.[0].location, .[0].digest] - ]|flatten|.[] - " - if [[ $detect_containerd == "true" ]]; then - parsed=$(LIMA_HOME=$(mktemp -d) limactl validate "$template" --fill 2>/dev/null | yq eval "${yq_filter}") - else - parsed=$(yq eval "${yq_filter}" "$template") - fi - # macOS earlier than 15.0 uses bash 3.2.57, which does not support readarray -t - # readarray -t arr <<<"$parsed" - while IFS= read -r line; do arr+=("$line"); done <<<"${parsed}" - readonly locations=("${arr[@]:0:2}") digests=("${arr[@]:2:2}") - readonly containerd="${arr[4]}" containerd_location="${arr[5]}" containerd_digest="${arr[6]}" - for ((i = 0; i < ${#locations[@]}; i++)); do - [[ ${locations[i]} != "null" ]] || continue - http_code=$(curl -sIL -w "%{http_code}" "${locations[i]}" -o /dev/null) - if [[ ${http_code} -eq 200 ]]; then - location=${locations[i]} - digest=${digests[i]} - break - fi - done - if [[ -z ${location} ]]; then - echo "Failed to get the image location for ${template}" >&2 - exit 1 - fi - - function location_to_sha256() { - local location=$1 - if command -v sha256sum > /dev/null; then - sha256="$(echo -n "$location" | sha256sum | cut -d' ' -f1)" - elif command -v shasum > /dev/null; then - sha256="$(echo -n "$location" | shasum -a 256 | cut -d' ' -f1)" - else - echo "sha256sum or shasum not found" >&2 - exit 1 - fi - echo "$sha256" - } - - function location_to_cache_path() { - local location=$1 - sha256=$(location_to_sha256 "$location") && echo ".download/by-url-sha256/$sha256" - } - - # path to cache - image_cache_path=$(location_to_cache_path "$location") - echo "path=$image_cache_path" >> "$GITHUB_OUTPUT" + source hack/cache-common-inc.sh + print_cache_informations_from_template "${{ inputs.template }}" "${{ inputs.arch }}" >> "$GITHUB_OUTPUT" + shell: bash - # key for cache - image_basename=$(basename "$location") - if [[ "$digest" != "null" ]]; then - key="image:$image_basename-$digest" - else - # use sha256 of location as key if digest is not available - key="image:$image_basename-url-sha256:$(location_to_sha256 "$location")" - fi - echo "key=$key" >> "$GITHUB_OUTPUT" + - name: "Cache ${{ steps.cache-params-from-template.outputs.image-path }}" + if: ${{ steps.cache-params-from-template.outputs.image-key != '' }} + # avoid using `~` in path that will be expanded to platform specific home directory + uses: actions/cache@v4 + with: + path: ${{ steps.cache-params-from-template.outputs.image-path }} + key: ${{ steps.cache-params-from-template.outputs.image-key }} + enableCrossOsArchive: true - # containerd path and key for cache - if [[ $containerd == "true" && "$containerd_location" != "null" ]]; then - containerd_basename=$(basename "$containerd_location") - if [[ ${containerd_digest} != "null" ]]; then - containerd_key="containerd:$containerd_basename-$containerd_digest" - else - containerd_key="containerd:$containerd_basename-url-sha256:$(sha256 "$containerd_location")" - fi - echo "containerd-key=$containerd_key" >> "$GITHUB_OUTPUT" - containerd_cache_path=$(location_to_cache_path "$containerd_location") - echo "containerd-path=$containerd_cache_path" >> "$GITHUB_OUTPUT" - else - echo "containerd-key=" >> "$GITHUB_OUTPUT" - echo "containerd-path=" >> "$GITHUB_OUTPUT" - fi - shell: bash + - name: "Cache ${{ steps.cache-params-from-template.outputs.kernel-path }}" + if: ${{ steps.cache-params-from-template.outputs.kernel-key != '' }} + # avoid using `~` in path that will be expanded to platform specific home directory + uses: actions/cache@v4 + with: + path: ${{ steps.cache-params-from-template.outputs.kernel-path }} + key: ${{ steps.cache-params-from-template.outputs.kernel-key }} + enableCrossOsArchive: true - - name: "Cache ${{ steps.cache-params-from-template.outputs.path }}" + - name: "Cache ${{ steps.cache-params-from-template.outputs.initrd-path }}" + if: ${{ steps.cache-params-from-template.outputs.initrd-key != '' }} # avoid using `~` in path that will be expanded to platform specific home directory uses: actions/cache@v4 with: - path: ${{ steps.cache-params-from-template.outputs.path }} - key: ${{ steps.cache-params-from-template.outputs.key }} + path: ${{ steps.cache-params-from-template.outputs.initrd-path }} + key: ${{ steps.cache-params-from-template.outputs.initrd-key }} enableCrossOsArchive: true - name: "Cache ${{ steps.cache-params-from-template.outputs.containerd-key }}" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d0bb8a271fc7..a77056b4f9d6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -163,10 +163,6 @@ jobs: run: make - name: Install run: make install - - name: Cache image used by default.yaml - uses: ./.github/actions/setup_cache_for_template - with: - template: templates/default.yaml - name: Validate templates run: find -L templates -name '*.yaml' | xargs limactl validate - name: Install test dependencies @@ -180,11 +176,15 @@ jobs: # GitHub runners seem to have lima installed by brew already; we don't want/need it time brew uninstall --ignore-dependencies lima colima time brew install qemu bash coreutils curl jq - - name: "Show cache" - run: ./hack/debug-cache.sh - name: "Inject `no_timer_check` to kernel cmdline" # workaround to https://github.com/lima-vm/lima/issues/84 run: ./hack/inject-cmdline-to-template.sh templates/default.yaml no_timer_check + - name: Cache image used by default.yaml + uses: ./.github/actions/setup_cache_for_template + with: + template: templates/default.yaml + - name: "Show cache" + run: ./hack/debug-cache.sh - name: "Test default.yaml" uses: nick-invision/retry@v3 with: @@ -328,6 +328,9 @@ jobs: run: make - name: Install run: make install + - name: "Inject `no_timer_check` to kernel cmdline" + # workaround to https://github.com/lima-vm/lima/issues/84 + run: ./hack/inject-cmdline-to-template.sh templates/vmnet.yaml no_timer_check - name: Cache image used by vmnet.yaml uses: ./.github/actions/setup_cache_for_template with: @@ -350,9 +353,6 @@ jobs: - name: Unit test (pkg/networks) with socket_vmnet # Set -count=1 to disable cache run: go test -v -count=1 ./pkg/networks/... - - name: "Inject `no_timer_check` to kernel cmdline" - # workaround to https://github.com/lima-vm/lima/issues/84 - run: ./hack/inject-cmdline-to-template.sh templates/vmnet.yaml no_timer_check - name: Test socket_vmnet uses: nick-invision/retry@v3 with: @@ -381,7 +381,6 @@ jobs: uses: ./.github/actions/setup_cache_for_template with: template: https://raw.githubusercontent.com/lima-vm/lima/${{ matrix.oldver }}/examples/ubuntu-lts.yaml - detect-containerd: "false" - name: Install test dependencies run: brew install qemu bash coreutils - name: Test diff --git a/hack/cache-common-inc.sh b/hack/cache-common-inc.sh new file mode 100755 index 000000000000..f5bfbfc97968 --- /dev/null +++ b/hack/cache-common-inc.sh @@ -0,0 +1,323 @@ +#!/usr/bin/env bash + +# e.g. +# ```console +# $ download_template_if_needed templates/default.yaml +# templates/default.yaml +# $ download_template_if_needed https://raw.githubusercontent.com/lima-vm/lima/v0.15.1/examples/ubuntu-lts.yaml +# /tmp/tmp.1J9Q6Q/template.yaml +# ``` +function download_template_if_needed() { + local template="$1" + case "${template}" in + https://*) + tmp_yaml=$(mktemp -d)/template.yaml + curl -sSLf "${template}" >"${tmp_yaml}" || return + echo "${tmp_yaml}" + ;; + *) + test -f "${template}" || return + echo "${template}" + ;; + esac +} + +# e.g. +# ```console +# $ print_image_locations_for_arch_from_template templates/default.yaml +# https://cloud-images.ubuntu.com/releases/24.04/release-20240809/ubuntu-24.04-server-cloudimg-arm64.img sha256:2e0c90562af1970ffff220a5073a7830f4acc2aad55b31593003e8c363381e7a +# https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-arm64.img null +# ``` +function print_image_locations_for_arch_from_template() { + local template arch + template=$(download_template_if_needed "$1") || return + local -r template=${template} + arch=$(detect_arch "${template}" "${2:-}") || return + local -r arch=${arch} + + # extract digest, location and size by parsing template using arch + local -r yq_filter="[.images | map(select(.arch == \"${arch}\")) | .[].location] | .[]" + yq eval "${yq_filter}" "${template}" +} + +# e.g. +# ```console +# $ detect_arch templates/default.yaml +# x86_64 +# $ detect_arch templates/default.yaml arm64 +# aarch64 +# ``` +function detect_arch() { + local template arch + template=$(download_template_if_needed "$1") || return + local -r template=${template} + + arch="${2:-$(yq '.arch // ""' "${template}")}" + arch="${arch:-$(uname -m)}" + # normalize arch. amd64 -> x86_64, arm64 -> aarch64 + case "${arch}" in + amd64 | x86_64) arch=x86_64 ;; + aarch64 | arm64) arch=aarch64 ;; + *) ;; + esac + echo "${arch}" +} + +# e.g. +# ```console +# $ print_image_locations_for_arch_from_template templates/default.yaml|print_valid_image_index +# 0 +# ``` +function print_valid_image_index() { + local index=0 + while read -r location; do + [[ ${location} != "null" ]] || continue + http_code_and_size=$(check_location_with_cache "${location}") + read -r http_code _size <<<"${http_code_and_size}" + if [[ ${http_code} -eq 200 ]]; then + echo "${index}" + return + fi + index=$((index + 1)) + done + echo "Failed to get the valid image location" >&2 + return 1 +} + +# e.g. +# ```console +# $ size_from_location "https://cloud-images.ubuntu.com/releases/24.04/release-20240725/ubuntu-24.04-server-cloudimg-amd64.img" +# 585498624 +# ``` +function size_from_location() { + ( + set -o pipefail + local location=$1 + check_location "${location}" | cut -d' ' -f2 + ) +} + +# Check the remote location and return the http code and size. +# If GITHUB_ACTIONS is true, the result is not cached. +# e.g. +# ```console +# $ check_location "https://cloud-images.ubuntu.com/releases/24.04/release-20240725/ubuntu-24.04-server-cloudimg-amd64.img" +# 200 585498624 +# ``` +function check_location() { + # shellcheck disable=SC2154 + if [[ ${GITHUB_ACTIONS:-false} == true ]]; then + check_location_without_cache "$1" + else + check_location_with_cache "$1" + fi +} + +# Check the remote location and return the http code and size. +# The result is cached in .check_location-response-cache.yaml +# e.g. +# ```console +# $ check_location_with_cache "https://cloud-images.ubuntu.com/releases/24.04/release-20240725/ubuntu-24.04-server-cloudimg-amd64.img" +# 200 585498624 +# ``` +function check_location_with_cache() { + local -r location="$1" cache_file=".check_location-response-cache.yaml" + # check ${cache_file} for the cache + if [[ -f ${cache_file} ]]; then + cached=$(yq -e eval ".[\"${location}\"]" "${cache_file}" 2>/dev/null) && echo "${cached}" && return + else + touch "${cache_file}" + fi + http_code_and_size=$(check_location_without_cache "${location}") || return + yq eval ".[\"${location}\"] = \"${http_code_and_size}\"" -i "${cache_file}" || return + echo "${http_code_and_size}" +} + +# e.g. +# ```console +# $ check_location "https://cloud-images.ubuntu.com/releases/24.04/release-20240725/ubuntu-24.04-server-cloudimg-amd64.img" +# 200 585498624 +# ``` +function check_location_without_cache() { + local -r location="$1" + curl -sIL -w "%{http_code} %header{Content-Length}" "${location}" -o /dev/null +} + +# e.g. +# ```console +# $ print_image_kernel_initrd_locations_with_digest_for_arch_from_template_at_index templates/default.yaml 0 +# https://cloud-images.ubuntu.com/releases/24.04/release-20240809/ubuntu-24.04-server-cloudimg-arm64.img +# sha256:2e0c90562af1970ffff220a5073a7830f4acc2aad55b31593003e8c363381e7a +# null +# null +# null +# null +# ``` +function print_image_kernel_initrd_locations_with_digest_for_arch_from_template_at_index() { + local template index="${2:-}" arch + template=$(download_template_if_needed "$1") || return + local -r template=${template} + arch=$(detect_arch "${template}" "${3:-}") || return + local -r arch=${arch} + + local -r yq_filter="[(.images[] | select(.arch == \"${arch}\"))].[${index}]|[ + .location, + .digest, + .kernel.location, + .kernel.digest, + .initrd.location, + .initrd.digest + ]" + yq -o=t eval "${yq_filter}" "${template}" +} + +# e.g. +# ```console +# $ print_containerd_config_for_arch_from_template templates/default.yaml +# true +# https://github.com/containerd/nerdctl/releases/download/v1.7.6/nerdctl-full-1.7.6-linux-arm64.tar.gz +# sha256:77c747f09853ee3d229d77e8de0dd3c85622537d82be57433dc1fca4493bab95 +# ``` +function print_containerd_config_for_arch_from_template() { + local template arch + template=$(download_template_if_needed "$1") || return + local -r template=${template} + arch=$(detect_arch "${template}" "${2:-}") || return + local -r arch=${arch} + + local -r yq_filter=" + [.containerd|[.system or .user], + .containerd.archives | map(select(.arch == \"${arch}\")) | [.[0].location, .[0].digest]]|flatten + " + validated_template="$( + limactl validate "${template}" --fill 2>/dev/null || echo "{.containerd: {system: false, user: false, archives: []}}" + )" + yq -o=t eval "${yq_filter}" <<<"${validated_template}" +} + +# e.g. +# ```console +# $ location_to_sha256 "https://cloud-images.ubuntu.com/releases/24.04/release-20240809/ubuntu-24.04-server-cloudimg-arm64.img" +# ae988d797c6de06b9c8a81a2b814904151135ccfd4616c22948057f6280477e8 +# ``` +function location_to_sha256() { + ( + set -o pipefail + local -r location="$1" + if command -v sha256sum >/dev/null; then + sha256="$(echo -n "${location}" | sha256sum | cut -d' ' -f1)" + elif command -v shasum >/dev/null; then + sha256="$(echo -n "${location}" | shasum -a 256 | cut -d' ' -f1)" + else + echo "sha256sum or shasum not found" >&2 + exit 1 + fi + echo "${sha256}" + ) +} + +# e.g. +# ```console +# $ location_to_cache_path "https://cloud-images.ubuntu.com/releases/24.04/release-20240809/ubuntu-24.04-server-cloudimg-arm64.img" +# .download/by-url-sha256/ae988d797c6de06b9c8a81a2b814904151135ccfd4616c22948057f6280477e8 +# ``` +function location_to_cache_path() { + local location=$1 + [[ ${location} != "null" ]] || return + sha256=$(location_to_sha256 "${location}") && echo ".download/by-url-sha256/${sha256}" +} + +# e.g. +# ```console +# $ cache_key_from_prefix_location_and_digest image "https://cloud-images.ubuntu.com/releases/24.04/release-20240809/ubuntu-24.04-server-cloudimg-arm64.img" "sha256:2e0c90562af1970ffff220a5073a7830f4acc2aad55b31593003e8c363381e7a" +# image:ubuntu-24.04-server-cloudimg-arm64.img-sha256:2e0c90562af1970ffff220a5073a7830f4acc2aad55b31593003e8c363381e7a +# $ cache_key_from_prefix_location_and_digest image "https://cloud-images.ubuntu.com/releases/24.04/release-20240809/ubuntu-24.04-server-cloudimg-arm64.img" "null" +# image:ubuntu-24.04-server-cloudimg-arm64.img-url-sha256:ae988d797c6de06b9c8a81a2b814904151135ccfd4616c22948057f6280477e8 +# ``` +function cache_key_from_prefix_location_and_digest() { + local prefix=$1 location=$2 digest=$3 location_basename + [[ ${location} != "null" ]] || return + location_basename=$(basename "${location}") + if [[ ${digest} != "null" ]]; then + echo "${prefix}:${location_basename}-${digest}" + else + # use sha256 of location as key if digest is not available + echo "${prefix}:${location_basename}-url-sha256:$(location_to_sha256 "${location}")" + fi +} + +# e.g. +# ```console +# $ print_path_and_key_for_cache image "https://cloud-images.ubuntu.com/releases/24.04/release-20240809/ubuntu-24.04-server-cloudimg-arm64.img" "sha256:2e0c90562af1970ffff220a5073a7830f4acc2aad55b31593003e8c363381e7a" +# image-path=.download/by-url-sha256/ae988d797c6de06b9c8a81a2b814904151135ccfd4616c22948057f6280477e8 +# image-key=image:ubuntu-24.04-server-cloudimg-arm64.img-sha256:2e0c90562af1970ffff220a5073a7830f4acc2aad55b31593003e8c363381e7a +# ``` +function print_path_and_key_for_cache() { + local -r prefix=$1 location=$2 digest=$3 + cache_path=$(location_to_cache_path "${location}" || true) + cache_key=$(cache_key_from_prefix_location_and_digest "${prefix}" "${location}" "${digest}" || true) + echo "${prefix}-path=${cache_path}" + echo "${prefix}-key=${cache_key}" +} + +# e.g. +# ```console +# $ print_cache_informations_from_template templates/default.yaml +# image-path=.download/by-url-sha256/ae988d797c6de06b9c8a81a2b814904151135ccfd4616c22948057f6280477e8 +# image-key=image:ubuntu-24.04-server-cloudimg-arm64.img-sha256:2e0c90562af1970ffff220a5073a7830f4acc2aad55b31593003e8c363381e7a +# kernel-path= +# kernel-key= +# initrd-path= +# initrd-key= +# containerd-path=.download/by-url-sha256/21cc8dfa548ea8a678135bd6984c9feb9f8a01901d10b11bb491f6f4e7537158 +# containerd-key=containerd:nerdctl-full-1.7.6-linux-arm64.tar.gz-sha256:77c747f09853ee3d229d77e8de0dd3c85622537d82be57433dc1fca4493bab95 +# $ print_cache_informations_from_template templates/experimental/riscv64.yaml +# image-path=.download/by-url-sha256/760b6ec69c801177bdaea06d7ee25bcd6ab72a331b9d3bf38376578164eb8f01 +# image-key=image:ubuntu-24.04-server-cloudimg-riscv64.img-sha256:361d72c5ed9781b097ab2dfb1a489c64e51936be648bbc5badee762ebdc50c31 +# kernel-path=.download/by-url-sha256/4568026693dc0f31a551b6741839979c607ee6bb0bf7209c89f3348321c52c61 +# kernel-key=kernel:qemu-riscv64_smode_uboot.elf-sha256:d4b3a10c3ef04219641802a586dca905e768805f5a5164fb68520887df54f33c +# initrd-path= +# initrd-key= +# ``` +function print_cache_informations_from_template() { + ( + set -o pipefail + local template index image_kernel_initrd_info location digest containerd_info + template=$(download_template_if_needed "$1") || return + local -r template="${template}" + index=$(print_image_locations_for_arch_from_template "${template}" "${@:2}" | print_valid_image_index) || return + local -r index="${index}" + image_kernel_initrd_info=$(print_image_kernel_initrd_locations_with_digest_for_arch_from_template_at_index "${template}" "${index}" "${@:2}") || return + # shellcheck disable=SC2034 + read -r image_location image_digest kernel_location kernel_digest initrd_location initrd_digest <<<"${image_kernel_initrd_info}" + for prefix in image kernel initrd; do + location=${prefix}_location + digest=${prefix}_digest + print_path_and_key_for_cache "${prefix}" "${!location}" "${!digest}" + done + if command -v limactl >/dev/null; then + containerd_info=$(print_containerd_config_for_arch_from_template "${template}" "${@:2}") || return + read -r containerd_enabled containerd_location containerd_digest <<<"${containerd_info}" + if [[ ${containerd_enabled} == "true" ]]; then + print_path_and_key_for_cache "containerd" "${containerd_location}" "${containerd_digest}" + fi + fi + ) +} + +# Compatible with hashFile() in GitHub Actions +# e.g. +# ```console +# $ hash_file templates/default.yaml +# ceec5ba3dc8872c083b2eb7f44e3e3f295d5dcdeccf0961ee153be6586525e5e +# ``` +function hash_file() { + ( + set -o pipefail + local hash="" + for file in "$@"; do + hash="${hash}$(sha256sum "${file}" | cut -d' ' -f1)" || return + done + echo "${hash}" | xxd -r -p | sha256sum | cut -d' ' -f1 + ) +} diff --git a/hack/calculate-cache.sh b/hack/calculate-cache.sh index 7f7d51e23197..b0b72dfe3f0a 100755 --- a/hack/calculate-cache.sh +++ b/hack/calculate-cache.sh @@ -4,19 +4,23 @@ # # Answer to the question in https://github.com/lima-vm/lima/pull/2508#discussion_r1699798651 +scriptdir=$(dirname "${BASH_SOURCE[0]}") +# shellcheck source=./common.inc.sh +. "${scriptdir}/cache-common-inc.sh" + # usage: [DEBUG=1] ./hack/calculate-cache.sh # DEBUG=1 will save the collected information in .calculate-cache-collected-info-{before,after}.yaml # # This script does: # 1. extracts `runs_on` and `template` from workflow file (.github/workflows/test.yml) -# 2. check each template for image and nerdctl -# 3. detect size of image and nerdctl (responses from remote are cached for faster iteration) -# save the response in .calculate-cache-response-cache.yaml +# 2. check each template for image, kernel, initrd, and nerdctl +# 3. detect size of image, kernel, initrd, and nerdctl (responses from remote are cached for faster iteration) +# save the response in .check_location-response-cache.yaml # 4. print content size, actual cache size (if available), by cache key # # The major differences for reducing cache usage are as follows: # - it is now cached `~/.cache/lima/download/by-url-sha256/$sha256` instead of caching `~/.cache/lima/download` -# - the cache keys are now based on the image digest and nerdctl digest instead of the template file's hash +# - the cache keys are now based on the image, kernel, initrd, and nerdctl digest instead of the template file's hash # - enables the use of cache regardless of the operating system used to execute CI. # # The script requires the following commands: @@ -116,82 +120,6 @@ function runner_os_from_runner() { esac } -# check the remote location and return the http code and size. -# The result is cached in .calculate-cache-response-cache.yaml -# e.g. -# ```console -# $ check_location "https://cloud-images.ubuntu.com/releases/24.04/release-20240725/ubuntu-24.04-server-cloudimg-amd64.img" -# 200 585498624 -# ``` -function check_location() { - location="$1" - readonly cache_file="./.calculate-cache-response-cache.yaml" - # check response_cache.yaml for the cache - if [[ -f ${cache_file} ]]; then - cached=$(yq -e eval ".[\"${location}\"]" "${cache_file}" 2>/dev/null) && echo "${cached}" && return - else - touch "${cache_file}" - fi - http_code_and_size=$(curl -sIL -w "%{http_code} %header{Content-Length}" "${location}" -o /dev/null) - yq eval ".[\"${location}\"] = \"${http_code_and_size}\"" -i "${cache_file}" - echo "${http_code_and_size}" -} - -# print image location, digest, size, hash, containerd, containerd_location, containerd_digest, containerd_size from the template -# e.g. -# ```console -# $ print_location_digest_size_hash_from_template "templates/default.yaml" -# https://cloud-images.ubuntu.com/releases/24.04/release-20240725/ubuntu-24.04-server-cloudimg-amd64.img sha256:d2377667ea95222330ca2287817403c85178dad397e9fed768a9b4aec79d2a7f 585498624 49aa50a4872ded07ebf657c0eaf9e44ecc0c174d033a97c537ecd270f35b462f true https://github.com/containerd/nerdctl/releases/download/v1.7.6/nerdctl-full-1.7.6-linux-amd64.tar.gz sha256:2c841e097fcfb5a1760bd354b3778cb695b44cd01f9f271c17507dc4a0b25606 237465717 -# ``` -function print_location_digest_size_hash_from_template() { - readonly template=$1 - case "${template}" in - http*) - template_yaml=$(curl -sSL "${template}") - ;; - *) - template_yaml=$(<"${template}") - ;; - esac - readonly yq_filter=" - [ - .images | map(select(.arch == \"${arch}\")) | [.[0,1].location, .[0,1].digest], - .containerd|[.system or .user], - .containerd.archives | map(select(.arch == \"${arch}\")) | [.[0].location, .[0].digest] - ]|flatten|.[] - " - if command -v limactl &>/dev/null; then - parsed=$(limactl validate <(echo "${template_yaml}") --fill 2>/dev/null | yq eval "${yq_filter}") - else - parsed=$(yq eval "${yq_filter}" <<<"${template_yaml}") - fi - # macOS earlier than 15.0 uses bash 3.2.57, which does not support readarray -t - # readarray -t arr <<<"${parsed}" - while IFS= read -r line; do arr+=("${line}"); done <<<"${parsed}" - readonly locations=("${arr[@]:0:2}") digests=("${arr[@]:2:2}") - readonly containerd="${arr[4]}" containerd_location="${arr[5]}" containerd_digest="${arr[6]}" - declare location digest size hash - for ((i = 0; i < ${#locations[@]}; i++)); do - [[ ${locations[i]} != null ]] || continue - http_code_and_size=$(check_location "${locations[i]}") - read -r http_code size <<<"${http_code_and_size}" - if [[ ${http_code} -eq 200 ]]; then - location=${locations[i]} - digest=${digests[i]} - break - fi - done - if [[ -z ${location} ]]; then - echo "Failed to get the image location for ${template}" >&2 - return 1 - fi - hash=$(sha256sum <<<"${template_yaml}" | cut -d' ' -f1 | xxd -r -p | sha256sum | cut -d' ' -f1) - declare containerd_size - containerd_http_code_and_size=$(check_location "${containerd_location}") - read -r _containerd_http_code containerd_size <<<"${containerd_http_code_and_size}" - echo "${location} ${digest} ${size} ${hash} ${containerd} ${containerd_location} ${containerd_digest} ${containerd_size}" -} - # format first column to MiB # e.g. # ```console @@ -258,29 +186,39 @@ for cache_method in before after; do for workflow in "${workflows[@]}"; do print_runs_on_template_from_workflow "${workflow}" done | while IFS=$'\t' read -r runner template; do - runner_os=$(runner_os_from_runner "${runner}") - location_digest_size_hash=$(print_location_digest_size_hash_from_template "${template}") || continue - read -r location digest size hash containerd containerd_location containerd_digest containerd_size <<<"${location_digest_size_hash}" + template=$(download_template_if_needed "${template}") || continue + arch=$(detect_arch "${template}" "${arch}") || continue + index=$(print_image_locations_for_arch_from_template "${template}" "${arch}" | print_valid_image_index) || continue + image_kernel_initrd_info=$(print_image_kernel_initrd_locations_with_digest_for_arch_from_template_at_index "${template}" "${index}" "${arch}") || continue + # shellcheck disable=SC2034 # shellcheck does not detect dynamic variables usage + read -r image_location image_digest kernel_location kernel_digest initrd_location initrd_digest <<<"${image_kernel_initrd_info}" + containerd_info=$(print_containerd_config_for_arch_from_template "${template}" "${@:2}") || continue + # shellcheck disable=SC2034 # shellcheck does not detect dynamic variables usage + read -r _containerd_enabled containerd_location containerd_digest <<<"${containerd_info}" + if [[ ${cache_method} != after ]]; then - key=${runner_os}-${hash} - elif [[ ${digest} == null ]]; then - key=image:$(basename "${location}")-url-sha256:$(echo -n "${location}" | sha256sum | cut -d' ' -f1) + key=$(runner_os_from_runner "${runner}" || true)-$(hash_file "${template}") else - key=image:$(basename "${location}")-${digest} + key=$(cache_key_from_prefix_location_and_digest image "${image_location}" "${image_digest}") fi - if [[ ${containerd} == true ]]; then + size=$(size_from_location "${image_location}") + for prefix in containerd kernel initrd; do + location="${prefix}_location" + digest="${prefix}_digest" + [[ ${!location} != null ]] || continue if [[ ${cache_method} != after ]]; then - # previous caching method packages the containerd archive with the image - size=$((size + containerd_size)) + # previous caching method packages all files in download to a single cache key + size=$((size + $(size_from_location "${!location}"))) else - # new caching method packages the containerd archive separately - containerd_key=containerd:$(basename "${containerd_location}")-${containerd_digest} + # new caching method caches each file separately + key_for_prefix=$(cache_key_from_prefix_location_and_digest "${prefix}" "${!location}" "${!digest}") + size_for_prefix=$(size_from_location "${!location}") printf -- "- key: %s\n template: %s\n location: %s\n digest: %s\n size: %s\n" \ - "${containerd_key}" "${template}" "${containerd_location}" "${containerd_digest}" "${containerd_size}" + "${key_for_prefix}" "${template}" "${!location}" "${!digest}" "${size_for_prefix}" fi - fi + done printf -- "- key: %s\n template: %s\n location: %s\n digest: %s\n size: %s\n" \ - "${key}" "${template}" "${location}" "${digest}" "${size}" + "${key}" "${template}" "${image_location}" "${image_digest}" "${size}" done ) output_json=$(yq -o=j . <<<"${output_yaml}")