From a56db7c0937a8c8f94106619c212152b72f39cbf Mon Sep 17 00:00:00 2001 From: thesayyn Date: Sun, 25 Sep 2022 15:11:46 +0300 Subject: [PATCH 1/2] feat: implement oci_index --- docs/BUILD.bazel | 5 ++ docs/index.md | 44 ++++++++++++++++++ example/BUILD.bazel | 4 ++ example/multi_arch/BUILD.bazel | 37 +++++++++++++++ example/multi_arch/test.bash | 1 + example/multi_arch/test.yaml | 7 +++ example/multi_arch/transition.bzl | 27 +++++++++++ oci/defs.bzl | 2 + oci/private/BUILD.bazel | 10 ++++ oci/private/index.bzl | 77 +++++++++++++++++++++++++++++++ oci/private/index.sh.tpl | 61 ++++++++++++++++++++++++ oci/private/toolchains_repo.bzl | 1 - 12 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 docs/index.md create mode 100644 example/multi_arch/BUILD.bazel create mode 100644 example/multi_arch/test.bash create mode 100644 example/multi_arch/test.yaml create mode 100644 example/multi_arch/transition.bzl create mode 100644 oci/private/index.bzl create mode 100644 oci/private/index.sh.tpl diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 7d457168..43b33814 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -12,4 +12,9 @@ stardoc_with_diff_test( bzl_library_target = "//oci/private:tarball", ) +stardoc_with_diff_test( + name = "index", + bzl_library_target = "//oci/private:index", +) + update_docs(name = "update") diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..e8816083 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,44 @@ + + +Implementation details for oci_index rule + + + +## oci_index + +
+oci_index(name, images)
+
+ +Build a multi-architecture OCI compatible container image. + +It takes number of `oci_image`s to create a fat multi-architecture image. + +```starlark +oci_image( + name = "app_linux_amd64" +) + +oci_image( + name = "app_linux_arm64" +) + +oci_index( + name = "app", + images = [ + ":app_linux_amd64", + ":app_linux_arm64" + ] +) +``` + + +**ATTRIBUTES** + + +| Name | Description | Type | Mandatory | Default | +| :------------- | :------------- | :------------- | :------------- | :------------- | +| name | A unique name for this target. | Name | required | | +| images | List of labels to oci_image targets. | List of labels | required | | + + diff --git a/example/BUILD.bazel b/example/BUILD.bazel index 54a8d091..16e16666 100644 --- a/example/BUILD.bazel +++ b/example/BUILD.bazel @@ -32,6 +32,10 @@ genrule( }, no_match_error = "Please build on a arm64 or amd64 host", ), + message = select({ + "@platforms//cpu:arm64": "Pulling base image for linux/arm64", + "@platforms//cpu:x86_64": "Pulling base image for linux/amd64", + }), output_to_bindir = True, toolchains = [ "@oci_crane_toolchains//:resolved_toolchain", diff --git a/example/multi_arch/BUILD.bazel b/example/multi_arch/BUILD.bazel new file mode 100644 index 00000000..347d747e --- /dev/null +++ b/example/multi_arch/BUILD.bazel @@ -0,0 +1,37 @@ +load("//oci:defs.bzl", "oci_image", "oci_index") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load(":transition.bzl", "multi_arch") + +pkg_tar( + name = "app", + srcs = ["test.bash"], +) + +oci_image( + name = "image", + architecture = select({ + "@platforms//cpu:arm64": "arm64", + "@platforms//cpu:x86_64": "amd64", + }), + base = "//example:base", + cmd = ["test.bash"], + entrypoint = ["bash"], + os = "linux", + tars = ["app.tar"], +) + +multi_arch( + name = "images", + image = ":image", + platforms = [ + "//example:linux_arm64", + "//example:linux_amd64", + ], +) + +oci_index( + name = "index", + images = [ + ":images", + ], +) diff --git a/example/multi_arch/test.bash b/example/multi_arch/test.bash new file mode 100644 index 00000000..4fda794d --- /dev/null +++ b/example/multi_arch/test.bash @@ -0,0 +1 @@ +echo "This is rules_oci!" \ No newline at end of file diff --git a/example/multi_arch/test.yaml b/example/multi_arch/test.yaml new file mode 100644 index 00000000..96d55ca3 --- /dev/null +++ b/example/multi_arch/test.yaml @@ -0,0 +1,7 @@ +schemaVersion: "2.0.0" + +commandTests: + - name: "echo hello" + command: "bash" + args: ["test.bash"] + expectedOutput: ["hello world!"] diff --git a/example/multi_arch/transition.bzl b/example/multi_arch/transition.bzl new file mode 100644 index 00000000..99300d82 --- /dev/null +++ b/example/multi_arch/transition.bzl @@ -0,0 +1,27 @@ +"a rule transitioning an oci_image to multiple platforms" + +def _multiarch_transition(settings, attr): + return [ + {"//command_line_option:platforms": str(platform)} + for platform in attr.platforms + ] + +multiarch_transition = transition( + implementation = _multiarch_transition, + inputs = [], + outputs = ["//command_line_option:platforms"], +) + +def _impl(ctx): + return DefaultInfo(files = depset(ctx.files.image)) + +multi_arch = rule( + implementation = _impl, + attrs = { + "image": attr.label(cfg = multiarch_transition), + "platforms": attr.label_list(), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + }, +) diff --git a/oci/defs.bzl b/oci/defs.bzl index 7d89d41d..5b388398 100644 --- a/oci/defs.bzl +++ b/oci/defs.bzl @@ -2,8 +2,10 @@ load("//oci/private:tarball.bzl", _oci_tarball = "oci_tarball") load("//oci/private:image.bzl", _oci_image = "oci_image") +load("//oci/private:index.bzl", _oci_index = "oci_index") load("//oci/private:structure_test.bzl", _structure_test = "structure_test") oci_tarball = _oci_tarball oci_image = _oci_image +oci_index = _oci_index structure_test = _structure_test diff --git a/oci/private/BUILD.bazel b/oci/private/BUILD.bazel index 79b26862..3ad18f9e 100644 --- a/oci/private/BUILD.bazel +++ b/oci/private/BUILD.bazel @@ -8,6 +8,7 @@ exports_files( exports_files([ "image.sh.tpl", "tarball.sh.tpl", + "index.sh.tpl", ]) filegroup( @@ -44,6 +45,15 @@ bzl_library( ], ) +bzl_library( + name = "index", + srcs = ["index.bzl"], + visibility = [ + "//docs:__pkg__", + "//oci:__subpackages__", + ], +) + bzl_library( name = "versions", srcs = ["versions.bzl"], diff --git a/oci/private/index.bzl b/oci/private/index.bzl new file mode 100644 index 00000000..03591d76 --- /dev/null +++ b/oci/private/index.bzl @@ -0,0 +1,77 @@ +"Implementation details for oci_index rule" + +_DOC = """Build a multi-architecture OCI compatible container image. + +It takes number of `oci_image`s to create a fat multi-architecture image. + +```starlark +oci_image( + name = "app_linux_amd64" +) + +oci_image( + name = "app_linux_arm64" +) + +oci_index( + name = "app", + images = [ + ":app_linux_amd64", + ":app_linux_arm64" + ] +) +``` +""" + +_attrs = { + "images": attr.label_list(mandatory = True, doc = "List of labels to oci_image targets."), + "_index_sh_tpl": attr.label(default = "index.sh.tpl", allow_single_file = True), +} + +def _expand_image_to_args(image, expander): + args = [ + "--image={}".format(image.path), + ] + for file in expander.expand(image): + if file.path.find("blobs") != -1: + args.append("--blob={}".format(file.tree_relative_path)) + return args + +def _oci_index_impl(ctx): + yq = ctx.toolchains["@aspect_bazel_lib//lib:yq_toolchain_type"] + + launcher = ctx.actions.declare_file("index_{}.sh".format(ctx.label.name)) + ctx.actions.expand_template( + template = ctx.file._index_sh_tpl, + output = launcher, + is_executable = True, + substitutions = { + "{{yq_path}}": yq.yqinfo.bin.path, + }, + ) + + output = ctx.actions.declare_directory(ctx.label.name) + + args = ctx.actions.args() + args.add(output.path, format = "--output=%s") + args.add_all(ctx.files.images, map_each = _expand_image_to_args, expand_directories = False) + + ctx.actions.run( + inputs = ctx.files.images, + arguments = [args], + outputs = [output], + executable = launcher, + tools = [yq.yqinfo.bin], + progress_message = "OCI Index %{label}", + ) + + return DefaultInfo(files = depset([output])) + +oci_index = rule( + implementation = _oci_index_impl, + attrs = _attrs, + doc = _DOC, + toolchains = [ + "@aspect_bazel_lib//lib:yq_toolchain_type", + ], +) diff --git a/oci/private/index.sh.tpl b/oci/private/index.sh.tpl new file mode 100644 index 00000000..7a4957db --- /dev/null +++ b/oci/private/index.sh.tpl @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -o pipefail -o errexit -o nounset + +readonly YQ="{{yq_path}}" + + +function add_image() { + local image_path="$1" + local output_path="$2" + + local manifests=$("${YQ}" eval '.manifests[]' "${image_path}/index.json") + + for manifest in "${manifests}"; do + local manifest_blob_path=$("${YQ}" '.digest | sub(":"; "/")' <<< ${manifest}) + local config_blob_path=$("${YQ}" '.config.digest | sub(":"; "/")' "${image_path}/blobs/${manifest_blob_path}") + + local platform=$("${YQ}" --output-format=json '{"os": .os, "architecture": .architecture, "variant": .variant, "os.version": .["os.version"], "os.features": .["os.features"]} | with_entries(select( .value != null ))' "${image_path}/blobs/${config_blob_path}") + + platform="${platform}" \ + manifest="${manifest}" \ + "${YQ}" --inplace --output-format=json '.manifests += [env(manifest) + {"platform": env(platform)}]' "${output_path}/manifest_list.json" + done +} + +function copy_blob() { + local image_path="$1" + local output_path="$2" + local blob_image_relative_path="$3" + local dest_path="${output_path}/${blob_image_relative_path}" + mkdir -p "$(dirname "${dest_path}")" + cat "${image_path}/${blob_image_relative_path}" > "${dest_path}" +} + +function create_oci_layout() { + local path="$1" + mkdir -p "${path}" + + echo '{"imageLayoutVersion": "1.0.0"}' > "${path}/oci-layout" + echo '{"schemaVersion": 2, "manifests": []}' > "${path}/index.json" + echo '{"schemaVersion": 2, "mediaType": "application/vnd.oci.image.index.v1+json", "manifests": []}' > "${path}/manifest_list.json" +} + +CURRENT_IMAGE="" +OUTPUT="" + +for ARG in "$@"; do + case "$ARG" in + (--output=*) OUTPUT="${ARG#--output=}"; create_oci_layout "$OUTPUT" ;; + (--image=*) CURRENT_IMAGE="${ARG#--image=}"; add_image "$CURRENT_IMAGE" "$OUTPUT" ;; + (--blob=*) copy_blob "${CURRENT_IMAGE}" "$OUTPUT" "${ARG#--blob=}" ;; + (*) echo "Unknown argument ${ARG}"; exit 1;; + esac +done + + +export checksum=$(shasum -a 256 "${OUTPUT}/manifest_list.json" | cut -f 1 -d " ") +export size=$(wc -c < "${OUTPUT}/manifest_list.json") + +"${YQ}" --inplace --output-format=json '.manifests += [{"mediaType": "application/vnd.oci.image.index.v1+json", "size": env(size), "digest": "sha256:" + env(checksum)}]' "$OUTPUT/index.json" + +mv "${OUTPUT}/manifest_list.json" "$OUTPUT/blobs/sha256/${checksum}" \ No newline at end of file diff --git a/oci/private/toolchains_repo.bzl b/oci/private/toolchains_repo.bzl index 1e966da0..7a8c590b 100644 --- a/oci/private/toolchains_repo.bzl +++ b/oci/private/toolchains_repo.bzl @@ -98,7 +98,6 @@ TOOLCHAIN_TMPL = """\ toolchain( name = "{platform}_toolchain", exec_compatible_with = {compatible_with}, - target_compatible_with = {compatible_with}, toolchain = "{toolchain}", toolchain_type = "{toolchain_type}", ) From fbc6a3951d40a8d618dc21cd79743cf6ce6969c6 Mon Sep 17 00:00:00 2001 From: thesayyn Date: Tue, 20 Dec 2022 20:08:12 +0300 Subject: [PATCH 2/2] rename to oci_image_index --- docs/BUILD.bazel | 4 +- docs/image_index.md | 46 +++++++++++++++++++ docs/index.md | 44 ------------------ example/multi_arch/BUILD.bazel | 4 +- example/multi_arch/test.yaml | 7 --- oci/defs.bzl | 4 +- oci/private/BUILD.bazel | 6 +-- oci/private/{index.bzl => image_index.bzl} | 18 ++++---- .../{index.sh.tpl => image_index.sh.tpl} | 0 9 files changed, 65 insertions(+), 68 deletions(-) create mode 100644 docs/image_index.md delete mode 100644 docs/index.md delete mode 100644 example/multi_arch/test.yaml rename oci/private/{index.bzl => image_index.bzl} (77%) rename oci/private/{index.sh.tpl => image_index.sh.tpl} (100%) diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 43b33814..3c0f639c 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -13,8 +13,8 @@ stardoc_with_diff_test( ) stardoc_with_diff_test( - name = "index", - bzl_library_target = "//oci/private:index", + name = "image_index", + bzl_library_target = "//oci/private:image_index", ) update_docs(name = "update") diff --git a/docs/image_index.md b/docs/image_index.md new file mode 100644 index 00000000..819484e1 --- /dev/null +++ b/docs/image_index.md @@ -0,0 +1,46 @@ + + +Implementation details for oci_image_index rule + + + +## oci_image_index + +
+oci_image_index(name, images)
+
+ +Build a multi-architecture OCI compatible container image. + +It takes number of `oci_image`s to create a fat multi-architecture image. + +Requires `wc` and `shasum` to be installed on the execution machine. + +```starlark +oci_image( + name = "app_linux_amd64" +) + +oci_image( + name = "app_linux_arm64" +) + +oci_image_index( + name = "app", + images = [ + ":app_linux_amd64", + ":app_linux_arm64" + ] +) +``` + + +**ATTRIBUTES** + + +| Name | Description | Type | Mandatory | Default | +| :------------- | :------------- | :------------- | :------------- | :------------- | +| name | A unique name for this target. | Name | required | | +| images | List of labels to oci_image targets. | List of labels | required | | + + diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index e8816083..00000000 --- a/docs/index.md +++ /dev/null @@ -1,44 +0,0 @@ - - -Implementation details for oci_index rule - - - -## oci_index - -
-oci_index(name, images)
-
- -Build a multi-architecture OCI compatible container image. - -It takes number of `oci_image`s to create a fat multi-architecture image. - -```starlark -oci_image( - name = "app_linux_amd64" -) - -oci_image( - name = "app_linux_arm64" -) - -oci_index( - name = "app", - images = [ - ":app_linux_amd64", - ":app_linux_arm64" - ] -) -``` - - -**ATTRIBUTES** - - -| Name | Description | Type | Mandatory | Default | -| :------------- | :------------- | :------------- | :------------- | :------------- | -| name | A unique name for this target. | Name | required | | -| images | List of labels to oci_image targets. | List of labels | required | | - - diff --git a/example/multi_arch/BUILD.bazel b/example/multi_arch/BUILD.bazel index 347d747e..94c82ace 100644 --- a/example/multi_arch/BUILD.bazel +++ b/example/multi_arch/BUILD.bazel @@ -1,4 +1,4 @@ -load("//oci:defs.bzl", "oci_image", "oci_index") +load("//oci:defs.bzl", "oci_image", "oci_image_index") load("@rules_pkg//:pkg.bzl", "pkg_tar") load(":transition.bzl", "multi_arch") @@ -29,7 +29,7 @@ multi_arch( ], ) -oci_index( +oci_image_index( name = "index", images = [ ":images", diff --git a/example/multi_arch/test.yaml b/example/multi_arch/test.yaml deleted file mode 100644 index 96d55ca3..00000000 --- a/example/multi_arch/test.yaml +++ /dev/null @@ -1,7 +0,0 @@ -schemaVersion: "2.0.0" - -commandTests: - - name: "echo hello" - command: "bash" - args: ["test.bash"] - expectedOutput: ["hello world!"] diff --git a/oci/defs.bzl b/oci/defs.bzl index 5b388398..aa1125ee 100644 --- a/oci/defs.bzl +++ b/oci/defs.bzl @@ -2,10 +2,10 @@ load("//oci/private:tarball.bzl", _oci_tarball = "oci_tarball") load("//oci/private:image.bzl", _oci_image = "oci_image") -load("//oci/private:index.bzl", _oci_index = "oci_index") +load("//oci/private:image_index.bzl", _oci_image_index = "oci_image_index") load("//oci/private:structure_test.bzl", _structure_test = "structure_test") oci_tarball = _oci_tarball oci_image = _oci_image -oci_index = _oci_index +oci_image_index = _oci_image_index structure_test = _structure_test diff --git a/oci/private/BUILD.bazel b/oci/private/BUILD.bazel index 3ad18f9e..4885e838 100644 --- a/oci/private/BUILD.bazel +++ b/oci/private/BUILD.bazel @@ -7,8 +7,8 @@ exports_files( exports_files([ "image.sh.tpl", + "image_index.sh.tpl", "tarball.sh.tpl", - "index.sh.tpl", ]) filegroup( @@ -46,8 +46,8 @@ bzl_library( ) bzl_library( - name = "index", - srcs = ["index.bzl"], + name = "image_index", + srcs = ["image_index.bzl"], visibility = [ "//docs:__pkg__", "//oci:__subpackages__", diff --git a/oci/private/index.bzl b/oci/private/image_index.bzl similarity index 77% rename from oci/private/index.bzl rename to oci/private/image_index.bzl index 03591d76..8908e3f5 100644 --- a/oci/private/index.bzl +++ b/oci/private/image_index.bzl @@ -1,9 +1,11 @@ -"Implementation details for oci_index rule" +"Implementation details for oci_image_index rule" _DOC = """Build a multi-architecture OCI compatible container image. It takes number of `oci_image`s to create a fat multi-architecture image. +Requires `wc` and `shasum` to be installed on the execution machine. + ```starlark oci_image( name = "app_linux_amd64" @@ -13,7 +15,7 @@ oci_image( name = "app_linux_arm64" ) -oci_index( +oci_image_index( name = "app", images = [ ":app_linux_amd64", @@ -25,7 +27,7 @@ oci_index( _attrs = { "images": attr.label_list(mandatory = True, doc = "List of labels to oci_image targets."), - "_index_sh_tpl": attr.label(default = "index.sh.tpl", allow_single_file = True), + "_image_index_sh_tpl": attr.label(default = "image_index.sh.tpl", allow_single_file = True), } def _expand_image_to_args(image, expander): @@ -37,12 +39,12 @@ def _expand_image_to_args(image, expander): args.append("--blob={}".format(file.tree_relative_path)) return args -def _oci_index_impl(ctx): +def _oci_image_index_impl(ctx): yq = ctx.toolchains["@aspect_bazel_lib//lib:yq_toolchain_type"] - launcher = ctx.actions.declare_file("index_{}.sh".format(ctx.label.name)) + launcher = ctx.actions.declare_file("image_index_{}.sh".format(ctx.label.name)) ctx.actions.expand_template( - template = ctx.file._index_sh_tpl, + template = ctx.file._image_index_sh_tpl, output = launcher, is_executable = True, substitutions = { @@ -67,8 +69,8 @@ def _oci_index_impl(ctx): return DefaultInfo(files = depset([output])) -oci_index = rule( - implementation = _oci_index_impl, +oci_image_index = rule( + implementation = _oci_image_index_impl, attrs = _attrs, doc = _DOC, toolchains = [ diff --git a/oci/private/index.sh.tpl b/oci/private/image_index.sh.tpl similarity index 100% rename from oci/private/index.sh.tpl rename to oci/private/image_index.sh.tpl