diff --git a/cosign/private/attest.bzl b/cosign/private/attest.bzl index fb71e53a..83172c7b 100644 --- a/cosign/private/attest.bzl +++ b/cosign/private/attest.bzl @@ -46,7 +46,7 @@ _attrs = { def _cosign_attest_impl(ctx): cosign = ctx.toolchains["@rules_oci//cosign:toolchain_type"] - yq = ctx.toolchains["@aspect_bazel_lib//lib:yq_toolchain_type"] + jq = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"] if ctx.attr.repository.find(":") != -1 or ctx.attr.repository.find("@") != -1: fail("repository attribute should not contain digest or tag.") @@ -67,7 +67,7 @@ def _cosign_attest_impl(ctx): is_executable = True, substitutions = { "{{cosign_path}}": cosign.cosign_info.binary.short_path, - "{{yq_path}}": yq.yqinfo.bin.short_path, + "{{jq_path}}": jq.jqinfo.bin.short_path, "{{image_dir}}": ctx.file.image.short_path, "{{fixed_args}}": " ".join(fixed_args), "{{type}}": ctx.attr.type, @@ -75,7 +75,7 @@ def _cosign_attest_impl(ctx): ) runfiles = ctx.runfiles(files = [ctx.file.image, ctx.file.predicate]) - runfiles = runfiles.merge(yq.default.default_runfiles) + runfiles = runfiles.merge(jq.default.default_runfiles) runfiles = runfiles.merge(cosign.default.default_runfiles) return DefaultInfo(executable = executable, runfiles = runfiles) @@ -87,6 +87,6 @@ cosign_attest = rule( executable = True, toolchains = [ "@rules_oci//cosign:toolchain_type", - "@aspect_bazel_lib//lib:yq_toolchain_type", + "@aspect_bazel_lib//lib:jq_toolchain_type", ], ) diff --git a/cosign/private/attest.sh.tpl b/cosign/private/attest.sh.tpl index a41eacf7..b812acdc 100644 --- a/cosign/private/attest.sh.tpl +++ b/cosign/private/attest.sh.tpl @@ -2,9 +2,9 @@ set -o pipefail -o errexit -o nounset readonly COSIGN="{{cosign_path}}" -readonly YQ="{{yq_path}}" +readonly JQ="{{jq_path}}" readonly IMAGE_DIR="{{image_dir}}" -readonly DIGEST=$("${YQ}" '.manifests[].digest' "${IMAGE_DIR}/index.json") +readonly DIGEST=$("${JQ}" -r '.manifests[].digest' "${IMAGE_DIR}/index.json") readonly FIXED_ARGS=({{fixed_args}}) diff --git a/cosign/private/sign.bzl b/cosign/private/sign.bzl index 55a5e8d8..5ff6d381 100644 --- a/cosign/private/sign.bzl +++ b/cosign/private/sign.bzl @@ -44,7 +44,7 @@ _attrs = { def _cosign_sign_impl(ctx): cosign = ctx.toolchains["@rules_oci//cosign:toolchain_type"] - yq = ctx.toolchains["@aspect_bazel_lib//lib:yq_toolchain_type"] + jq = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"] if ctx.attr.repository.find(":") != -1 or ctx.attr.repository.find("@") != -1: fail("repository attribute should not contain digest or tag.") @@ -56,14 +56,14 @@ def _cosign_sign_impl(ctx): is_executable = True, substitutions = { "{{cosign_path}}": cosign.cosign_info.binary.short_path, - "{{yq_path}}": yq.yqinfo.bin.short_path, + "{{jq_path}}": jq.jqinfo.bin.short_path, "{{image_dir}}": ctx.file.image.short_path, "{{fixed_args}}": " ".join(["--repository", ctx.attr.repository]), }, ) runfiles = ctx.runfiles(files = [ctx.file.image]) - runfiles = runfiles.merge(yq.default.default_runfiles) + runfiles = runfiles.merge(jq.default.default_runfiles) runfiles = runfiles.merge(cosign.default.default_runfiles) return DefaultInfo(executable = executable, runfiles = runfiles) @@ -75,6 +75,6 @@ cosign_sign = rule( executable = True, toolchains = [ "@rules_oci//cosign:toolchain_type", - "@aspect_bazel_lib//lib:yq_toolchain_type", + "@aspect_bazel_lib//lib:jq_toolchain_type", ], ) diff --git a/cosign/private/sign.sh.tpl b/cosign/private/sign.sh.tpl index 56f0992d..9ab1d8ca 100644 --- a/cosign/private/sign.sh.tpl +++ b/cosign/private/sign.sh.tpl @@ -2,9 +2,9 @@ set -o pipefail -o errexit -o nounset readonly COSIGN="{{cosign_path}}" -readonly YQ="{{yq_path}}" +readonly JQ="{{jq_path}}" readonly IMAGE_DIR="{{image_dir}}" -readonly DIGEST=$("${YQ}" '.manifests[].digest' "${IMAGE_DIR}/index.json") +readonly DIGEST=$("${JQ}" -r '.manifests[].digest' "${IMAGE_DIR}/index.json") readonly FIXED_ARGS=({{fixed_args}}) # set $@ to be FIXED_ARGS+$@ diff --git a/examples/multi_arch/BUILD.bazel b/examples/multi_arch/BUILD.bazel index c99cb725..5acf408e 100644 --- a/examples/multi_arch/BUILD.bazel +++ b/examples/multi_arch/BUILD.bazel @@ -43,5 +43,5 @@ genrule( assert_contains( name = "check_digest", actual = ":hash", - expected = "sha256:7dc510461a627051f9276ec21a26ed9a314e87e94eac1be996fcf48be6cc9aa2", + expected = "sha256:a5285d5827015227664ab5a2334a32b9750bd6ec8cfe3b655376b997f020eb43", ) diff --git a/oci/private/image.bzl b/oci/private/image.bzl index 1b8b6eaf..1dcf56de 100644 --- a/oci/private/image.bzl +++ b/oci/private/image.bzl @@ -84,11 +84,6 @@ If `group/gid` is not specified, the default group and supplementary groups of t "_empty_tar": attr.label(default = "empty.tar", allow_single_file = True), } -def _format_string_to_string_tuple(kv): - if type(kv) != "tuple": - fail("argument `kv` must be a tuple.") - return "%s=%s" % kv - def _platform_str(os, arch, variant = None): parts = [os, arch] if variant: diff --git a/oci/private/image_index.bzl b/oci/private/image_index.bzl index d93c39a8..a7dc1570 100644 --- a/oci/private/image_index.bzl +++ b/oci/private/image_index.bzl @@ -40,7 +40,7 @@ def _expand_image_to_args(image, expander): return args def _oci_image_index_impl(ctx): - yq = ctx.toolchains["@aspect_bazel_lib//lib:yq_toolchain_type"] + jq = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"] coreutils = ctx.toolchains["@aspect_bazel_lib//lib:coreutils_toolchain_type"] launcher = ctx.actions.declare_file("image_index_{}.sh".format(ctx.label.name)) @@ -49,7 +49,7 @@ def _oci_image_index_impl(ctx): output = launcher, is_executable = True, substitutions = { - "{{yq_path}}": yq.yqinfo.bin.path, + "{{jq_path}}": jq.jqinfo.bin.path, "{{coreutils_path}}": coreutils.coreutils_info.bin.path, }, ) @@ -65,7 +65,7 @@ def _oci_image_index_impl(ctx): arguments = [args], outputs = [output], executable = launcher, - tools = [yq.yqinfo.bin, coreutils.coreutils_info.bin], + tools = [jq.jqinfo.bin, coreutils.coreutils_info.bin], mnemonic = "OCIIndex", progress_message = "OCI Index %{label}", ) @@ -77,7 +77,7 @@ oci_image_index = rule( attrs = _attrs, doc = _DOC, toolchains = [ - "@aspect_bazel_lib//lib:yq_toolchain_type", + "@aspect_bazel_lib//lib:jq_toolchain_type", "@aspect_bazel_lib//lib:coreutils_toolchain_type", ], ) diff --git a/oci/private/image_index.sh.tpl b/oci/private/image_index.sh.tpl index e86b1046..3c901f86 100644 --- a/oci/private/image_index.sh.tpl +++ b/oci/private/image_index.sh.tpl @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -o pipefail -o errexit -o nounset -readonly YQ="{{yq_path}}" +readonly JQ="{{jq_path}}" readonly COREUTILS="{{coreutils_path}}" # Only crete the directory if it doesn't already exist. @@ -15,17 +15,18 @@ function add_image() { local image_path="$1" local output_path="$2" - local manifests=$("${YQ}" eval '.manifests[]' "${image_path}/index.json") + local manifests=$("${JQ}" -c '.manifests[]' "${image_path}/index.json") for manifest in "${manifests}"; do - local manifest_blob_path=$("${YQ}" '.digest | sub(":"; "/")' <<< ${manifest}) - local config_blob_path=$("${YQ}" '.config.digest | sub(":"; "/")' "${image_path}/blobs/${manifest_blob_path}") - - local platform=$("${YQ}" --output-format=json '{"os": .os, "architecture": .architecture, "variant": .variant, "os.version": .["os.version"], "os.features": .["os.features"]} | with_entries(select( .value != null ))' "${image_path}/blobs/${config_blob_path}") - - platform="${platform}" \ - manifest="${manifest}" \ - "${YQ}" --inplace --output-format=json '.manifests += [env(manifest) + {"platform": env(platform)}]' "${output_path}/manifest_list.json" + local manifest_blob_path=$("${JQ}" -r '.digest | sub(":"; "/")' <<< ${manifest}) + local config_blob_path=$("${JQ}" -r '.config.digest | sub(":"; "/")' "${image_path}/blobs/${manifest_blob_path}") + + local platform=$("${JQ}" -c '{"os": .os, "architecture": .architecture, "variant": .variant, "os.version": .["os.version"], "os.features": .["os.features"]} | with_entries(select( .value != null ))' "${image_path}/blobs/${config_blob_path}") + "${JQ}" --argjson platform "${platform}" \ + --argjson manifest "${manifest}" \ + '.manifests |= [$manifest + {"platform": $platform}]'\ + "${output_path}/manifest_list.json" > "${output_path}/manifest_list.new.json" + cat "${output_path}/manifest_list.new.json" > "${output_path}/manifest_list.json" done } @@ -60,9 +61,10 @@ for ARG in "$@"; do done -export checksum=$("${COREUTILS}" sha256sum "${OUTPUT}/manifest_list.json" | "${COREUTILS}" cut -f 1 -d " ") -export size=$("${COREUTILS}" wc -c < "${OUTPUT}/manifest_list.json") - -"${YQ}" --inplace --output-format=json '.manifests += [{"mediaType": "application/vnd.oci.image.index.v1+json", "size": env(size), "digest": "sha256:" + env(checksum)}]' "$OUTPUT/index.json" +checksum=$("${COREUTILS}" sha256sum "${OUTPUT}/manifest_list.json" | "${COREUTILS}" cut -f 1 -d " ") +size=$("${COREUTILS}" wc -c < "${OUTPUT}/manifest_list.json") +"${JQ}" -n --arg checksum "${checksum}" --argjson size "${size}" \ + '.manifests = [{"mediaType": "application/vnd.oci.image.index.v1+json", "size": $size, "digest": ("sha256:" + $checksum) }]' > "$OUTPUT/index.json" +cat "$OUTPUT/index.json" "${COREUTILS}" mv "${OUTPUT}/manifest_list.json" "$OUTPUT/blobs/sha256/${checksum}" diff --git a/oci/private/push.bzl b/oci/private/push.bzl index a6f68194..dcc7ba82 100644 --- a/oci/private/push.bzl +++ b/oci/private/push.bzl @@ -140,7 +140,7 @@ def _quote_args(args): def _impl(ctx): crane = ctx.toolchains["@rules_oci//oci:crane_toolchain_type"] - yq = ctx.toolchains["@aspect_bazel_lib//lib:yq_toolchain_type"] + jq = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"] if ctx.attr.repository and ctx.attr.repository_file: fail("must specify exactly one of 'repository_file' or 'repository'") @@ -156,7 +156,7 @@ def _impl(ctx): files = [ctx.file.image] substitutions = { "{{crane_path}}": crane.crane_info.binary.short_path, - "{{yq_path}}": yq.yqinfo.bin.short_path, + "{{jq_path}}": jq.jqinfo.bin.short_path, "{{image_dir}}": ctx.file.image.short_path, "{{fixed_args}}": "", } @@ -180,7 +180,7 @@ def _impl(ctx): substitutions = substitutions, ) runfiles = ctx.runfiles(files = files) - runfiles = runfiles.merge(yq.default.default_runfiles) + runfiles = runfiles.merge(jq.default.default_runfiles) runfiles = runfiles.merge(crane.default.default_runfiles) return DefaultInfo(executable = executable, runfiles = runfiles) @@ -190,7 +190,7 @@ oci_push_lib = struct( attrs = _attrs, toolchains = [ "@rules_oci//oci:crane_toolchain_type", - "@aspect_bazel_lib//lib:yq_toolchain_type", + "@aspect_bazel_lib//lib:jq_toolchain_type", ], ) diff --git a/oci/private/push.sh.tpl b/oci/private/push.sh.tpl index 096000af..8da0837b 100644 --- a/oci/private/push.sh.tpl +++ b/oci/private/push.sh.tpl @@ -2,7 +2,7 @@ set -o pipefail -o errexit -o nounset readonly CRANE="{{crane_path}}" -readonly YQ="{{yq_path}}" +readonly JQ="{{jq_path}}" readonly IMAGE_DIR="{{image_dir}}" readonly TAGS_FILE="{{tags}}" readonly FIXED_ARGS=({{fixed_args}}) @@ -14,7 +14,7 @@ if [ -f $REPOSITORY_FILE ] ; then fi # set $@ to be FIXED_ARGS+$@ -ALL_ARGS=(${FIXED_ARGS[@]} $@) +ALL_ARGS=(${FIXED_ARGS[@]+"${FIXED_ARGS[@]}"} $@) set -- ${ALL_ARGS[@]} TAGS=() @@ -42,7 +42,7 @@ while (( $# > 0 )); do esac done -DIGEST=$("${YQ}" eval '.manifests[0].digest' "${IMAGE_DIR}/index.json") +DIGEST=$("${JQ}" -r '.manifests[0].digest' "${IMAGE_DIR}/index.json") REFS=$(mktemp) "${CRANE}" push "${IMAGE_DIR}" "${REPOSITORY}@${DIGEST}" "${ARGS[@]+"${ARGS[@]}"}" --image-refs "${REFS}" diff --git a/oci/private/tarball.bzl b/oci/private/tarball.bzl index dc9d89c9..eac4e253 100644 --- a/oci/private/tarball.bzl +++ b/oci/private/tarball.bzl @@ -46,14 +46,15 @@ attrs = { } def _tarball_impl(ctx): + jq = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"].jqinfo + image = ctx.file.image tarball = ctx.actions.declare_file("{}/tarball.tar".format(ctx.label.name)) - yq_bin = ctx.toolchains["@aspect_bazel_lib//lib:yq_toolchain_type"].yqinfo.bin executable = ctx.actions.declare_file("{}/tarball.sh".format(ctx.label.name)) repo_tags = ctx.file.repo_tags substitutions = { - "{{yq}}": yq_bin.path, + "{{jq_path}}": jq.bin.path, "{{image_dir}}": image.path, "{{tarball_path}}": tarball.path, } @@ -72,7 +73,7 @@ def _tarball_impl(ctx): executable = util.maybe_wrap_launcher_for_windows(ctx, executable), inputs = [image, repo_tags, executable], outputs = [tarball], - tools = [yq_bin], + tools = [jq.bin], mnemonic = "OCITarball", progress_message = "OCI Tarball %{label}", ) @@ -98,7 +99,7 @@ oci_tarball = rule( doc = doc, toolchains = [ "@bazel_tools//tools/sh:toolchain_type", - "@aspect_bazel_lib//lib:yq_toolchain_type", + "@aspect_bazel_lib//lib:jq_toolchain_type", ], executable = True, ) diff --git a/oci/private/tarball.sh.tpl b/oci/private/tarball.sh.tpl index dff94b6d..b6e8a315 100644 --- a/oci/private/tarball.sh.tpl +++ b/oci/private/tarball.sh.tpl @@ -2,37 +2,35 @@ set -o pipefail -o errexit -o nounset readonly STAGING_DIR=$(mktemp -d) -readonly YQ="{{yq}}" +readonly JQ="{{jq_path}}" readonly IMAGE_DIR="{{image_dir}}" readonly BLOBS_DIR="${STAGING_DIR}/blobs" readonly TARBALL_PATH="{{tarball_path}}" readonly TAGS_FILE="{{tags}}" -MANIFEST_DIGEST=$(${YQ} eval '.manifests[0].digest | sub(":"; "/")' "${IMAGE_DIR}/index.json" | tr -d '"') +MANIFEST_DIGEST=$(${JQ} -r '.manifests[0].digest | sub(":"; "/")' "${IMAGE_DIR}/index.json" | tr -d '"') MANIFEST_BLOB_PATH="${IMAGE_DIR}/blobs/${MANIFEST_DIGEST}" -CONFIG_DIGEST=$(${YQ} eval '.config.digest | sub(":"; "/")' ${MANIFEST_BLOB_PATH}) +CONFIG_DIGEST=$(${JQ} -r '.config.digest | sub(":"; "/")' ${MANIFEST_BLOB_PATH}) CONFIG_BLOB_PATH="${IMAGE_DIR}/blobs/${CONFIG_DIGEST}" -LAYERS=$(${YQ} eval '.layers | map(.digest | sub(":"; "/"))' ${MANIFEST_BLOB_PATH}) +LAYERS=$(${JQ} -cr '.layers | map(.digest | sub(":"; "/"))' ${MANIFEST_BLOB_PATH}) mkdir -p $(dirname "${BLOBS_DIR}/${CONFIG_DIGEST}") cp "${CONFIG_BLOB_PATH}" "${BLOBS_DIR}/${CONFIG_DIGEST}" -for LAYER in $(${YQ} ".[]" <<< $LAYERS); do +for LAYER in $(${JQ} -r ".[]" <<< $LAYERS); do cp -f "${IMAGE_DIR}/blobs/${LAYER}" "${BLOBS_DIR}/${LAYER}.tar.gz" done -# Replace newlines (unix or windows line endings) with % character. -# We can't pass newlines to yq due to https://github.com/mikefarah/yq/issues/1430 and -# we can't update YQ at the moment because structure_test depends on a specific version: -# see https://github.com/bazel-contrib/rules_oci/issues/212 -repo_tags="$(tr -d '\r' < "${TAGS_FILE}" | tr '\n' '%')" \ -config="blobs/${CONFIG_DIGEST}" \ -layers="${LAYERS}" \ -"${YQ}" eval \ - --null-input '.[0] = {"Config": env(config), "RepoTags": "${repo_tags}" | envsubst | split("%") | map(select(. != "")) , "Layers": env(layers) | map( "blobs/" + . + ".tar.gz") }' \ - --output-format json > "${STAGING_DIR}/manifest.json" +repo_tags="$(cat "${TAGS_FILE}" | ${JQ} --null-input --raw-input 'reduce (inputs) as $line ([]; . + ([$line])) | map(select(. != ""))')" +config="blobs/${CONFIG_DIGEST}" +layers="${LAYERS}" + +"${JQ}" -n '.[0] = {"Config": $config, "RepoTags": $repo_tags, "Layers": $layers | map( "blobs/" + . + ".tar.gz") }' \ + --argjson repo_tags "${repo_tags}" \ + --arg config "${config}" \ + --argjson layers "${layers}" > "${STAGING_DIR}/manifest.json" # TODO: https://github.com/bazel-contrib/rules_oci/issues/217 tar -C "${STAGING_DIR}" -cf "${TARBALL_PATH}" manifest.json blobs diff --git a/oci/repositories.bzl b/oci/repositories.bzl index 6ce3e414..74ae42f2 100644 --- a/oci/repositories.bzl +++ b/oci/repositories.bzl @@ -1,6 +1,6 @@ """Repository rules for fetching external tools""" -load("@aspect_bazel_lib//lib:repositories.bzl", "register_copy_to_directory_toolchains", "register_coreutils_toolchains", "register_jq_toolchains", "register_yq_toolchains") +load("@aspect_bazel_lib//lib:repositories.bzl", "register_copy_to_directory_toolchains", "register_coreutils_toolchains", "register_jq_toolchains") load("//oci/private:auth_config_locator.bzl", "oci_auth_config_locator") load("//oci/private:toolchains_repo.bzl", "PLATFORMS", "toolchains_repo") load("//oci/private:versions.bzl", "CRANE_VERSIONS", "ZOT_VERSIONS") @@ -113,7 +113,6 @@ def oci_register_toolchains(name, crane_version, zot_version = None, register = # locate the config file on the host oci_auth_config_locator(name = "oci_auth_config") - register_yq_toolchains(register = register) register_jq_toolchains(register = register) register_coreutils_toolchains(register = register) register_copy_to_directory_toolchains(register = register)