diff --git a/.bazelrc b/.bazelrc index 6ab87247..3474ab97 100644 --- a/.bazelrc +++ b/.bazelrc @@ -9,8 +9,11 @@ test --test_env=DOCKER_HOST # Disable bzlmod lockfile common --lockfile_mode=off -# TODO(2.0): enable once we drop support for Bazel 5. -# common --credential_helper=public.ecr.aws=%workspace%/examples/credential_helper/auth.sh +# On bazel 6.4.0 these are needed to successfully fetch images. +common:needs_credential_helpers --credential_helper=public.ecr.aws=%workspace%/examples/credential_helper/auth.sh +common:needs_credential_helpers --credential_helper=index.docker.io=%workspace%/examples/credential_helper/auth.sh +common:needs_credential_helpers --credential_helper=docker.elastic.co=%workspace%/examples/credential_helper/auth.sh +common:needs_credential_helpers --credential_helper_cache_duration=0 # Load any settings specific to the current user. # .bazelrc.user should appear in .gitignore so that settings are not shared with team members diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cffcb60c..e5d32559 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -103,6 +103,19 @@ jobs: if: matrix.bzlmodEnabled run: echo "bzlmod_flag=--enable_bzlmod" >> $GITHUB_OUTPUT + - name: Set credential helpers flag + # Add --config needs_credential_helpers to add additional credential helpers + # to fetch from registries with HTTP headers set by credential helpers. + id: set_credential_helper_flag + if: matrix.bazelversion == '6.4.0' && matrix.folder == '.' + run: echo "credential_helper_flag=--config=needs_credential_helpers" >> $GITHUB_OUTPUT + + - name: Setup crane for credential helpers to use + uses: imjasonh/setup-crane@v0.3 + if: matrix.bazelversion == '6.4.0' && matrix.folder == '.' + with: + version: "v0.19.1" + - name: Configure Bazel version working-directory: ${{ matrix.folder }} run: echo "${{ matrix.bazelversion }}" > .bazelversion @@ -146,4 +159,10 @@ jobs: env: # Bazelisk will download bazel to here, ensure it is cached between runs. XDG_CACHE_HOME: ~/.cache/bazel-repo - run: bazel --bazelrc=$GITHUB_WORKSPACE/.github/workflows/ci.bazelrc --bazelrc=.bazelrc test ${{ steps.set_bzlmod_flag.outputs.bzlmod_flag }} //... + run: | + bazel \ + --bazelrc=$GITHUB_WORKSPACE/.github/workflows/ci.bazelrc \ + --bazelrc=.bazelrc \ + test //... \ + ${{ steps.set_bzlmod_flag.outputs.bzlmod_flag }} \ + ${{ steps.set_credential_helper_flag.outputs.credential_helper_flag }} diff --git a/examples/credential_helper/auth.sh b/examples/credential_helper/auth.sh index 2bdd8f2b..531c1f0d 100755 --- a/examples/credential_helper/auth.sh +++ b/examples/credential_helper/auth.sh @@ -1,14 +1,15 @@ #!/usr/bin/env bash # Requirements -# - curl +# - crane # - jq # - awk # # ./examples/credential_helper/auth.sh <<< '{"uri":"https://public.ecr.aws/token/?scope\u003drepository:lambda/python:pull\u0026service\u003dpublic.ecr.aws"}' # ./examples/credential_helper/auth.sh <<< '{"uri":"https://public.ecr.aws/v2/lambda/python/manifests/3.11.2024.01.25.10"}' + function log () { - echo "$1" >> /tmp/oci_auth.log + echo $@ >> "/tmp/oci_auth.log" } log "" @@ -20,18 +21,39 @@ log "Payload: $input" uri=$(jq -r ".uri" <<< $input) log "URI: $uri" -host="$(echo $uri | awk -F[/:] '{print $4}')" +host="$(awk -F[/:] '{print $4}' <<< $uri)" log "Host: $host" + if [[ $input == *"/token"* ]]; then log "Auth: None" echo "{}" - exit 0 + exit 1 fi +repository=$(awk -F'^https?://|v2/|/manifests|/blobs' '{print $2 $3}' <<< "$uri") +log "Repository: $repository" + + +ACCEPTED_MEDIA_TYPES='[ + "application/vnd.docker.distribution.manifest.v2+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.oci.image.index.v1+json" +]' + + # This will write the response to stdout in a format that Bazels credential helper protocol understands. # Since this is called by Bazel, users won't bee seeing output of this. -curl -fsSL https://$host/token | jq '{headers:{"Authorization": [("Bearer " + .token)]}}' +crane auth token "$repository" | +jq --argjson accept "$ACCEPTED_MEDIA_TYPES" \ +'{headers: {Authorization: [("Bearer " + .token)], Accept: [($accept | join(", "))], "Docker-Distribution-API-Version": ["registry/2.0"] }}' + +if [[ $? != 0 ]]; then + log "Auth: Failed" + exit 1 +fi log "Auth: Complete" -# Alternatively you can call an external program such as `docker-credential-ecr-login` to perform the token exchange. \ No newline at end of file + +# Alternatively you can call an external program such as `docker-credential-ecr-login` to perform the token exchange. diff --git a/fetch.bzl b/fetch.bzl index e69d6dfc..c05e0204 100644 --- a/fetch.bzl +++ b/fetch.bzl @@ -31,9 +31,9 @@ def fetch_images(): ], ) - # Pull an image from public ECR. + # Pull an image from public ECR. # When --credential_helper is provided, see .bazelrc at workspace root, it will take precende over - # auth from oci_pull. However, pulling from public ECR works out of the box so this will never fail + # auth from oci_pull. However, pulling from public ECR works out of the box so this will never fail # unless oci_pull's authentication mechanism breaks and --credential_helper is absent. oci_pull( name = "ecr_lambda_python", @@ -41,8 +41,8 @@ def fetch_images(): tag = "3.11.2024.01.25.10", platforms = [ "linux/amd64", - "linux/arm64/v8" - ] + "linux/arm64/v8", + ], ) # Show that the digest is optional. @@ -172,7 +172,7 @@ def fetch_images(): digest = "sha256:9a83bce5d337e7e19d789ee7f952d36d0d514c80987c3d76d90fd1afd2411a9a", platforms = [ "linux/amd64", - "linux/arm64" + "linux/arm64", ], ) @@ -183,7 +183,7 @@ def fetch_images(): digest = "sha256:8d38ffa8fad72f4bc2647644284c16491cc2d375602519a1f963f96ccc916276", platforms = [ "linux/amd64", - "linux/arm64" + "linux/arm64", ], ) @@ -195,9 +195,12 @@ def fetch_images(): ) _DEB_TO_LAYER = """\ -alias( +genrule( name = "layer", - actual = ":data.tar.xz", + srcs = [":data.tar.xz"], + outs = ["data.tar.zst"], + cmd = "$(BSDTAR_BIN) --zstd -cf $@ @$<", + toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], visibility = ["//visibility:public"], ) """ diff --git a/oci/private/BUILD.bazel b/oci/private/BUILD.bazel index 91ccb4af..c88efce2 100644 --- a/oci/private/BUILD.bazel +++ b/oci/private/BUILD.bazel @@ -51,7 +51,6 @@ bzl_library( ], deps = [ "//oci/private:authn", - "//oci/private:download", "//oci/private:util", "@bazel_skylib//lib:dicts", ], @@ -84,13 +83,6 @@ bzl_library( visibility = ["//oci:__subpackages__"], ) -bzl_library( - name = "download", - srcs = ["download.bzl"], - visibility = ["//oci:__subpackages__"], - deps = ["@bazel_skylib//lib:versions"], -) - bzl_library( name = "authn", srcs = ["authn.bzl"], diff --git a/oci/private/download.bzl b/oci/private/download.bzl deleted file mode 100644 index d62e7373..00000000 --- a/oci/private/download.bzl +++ /dev/null @@ -1,125 +0,0 @@ -"Downloader functions " - -load("@aspect_bazel_lib//lib:base64.bzl", "base64") -load("@bazel_skylib//lib:versions.bzl", "versions") -load(":util.bzl", "util") - -def _auth_to_header(url, auth): - for auth_url in auth: - if auth_url == url: - auth_val = auth[auth_url] - - if "type" not in auth_val: - continue - - if auth_val["type"] == "basic": - credentials = base64.encode("{}:{}".format(auth_val["login"], auth_val["password"])) - return [ - "--header", - "Authorization: Basic {}".format(credentials), - ] - elif auth_val["type"] == "pattern": - token = auth_val["pattern"].replace("", auth_val["password"]) - return [ - "--header", - "Authorization: {}".format(token), - ] - return [] - -def _debug(message): - # Change to true when debugging - if False: - # buildifier: disable=print - print(message) - -# TODO(2.0): remove curl downloader -def _download( - rctx, - url, - output, - sha256 = "", - allow_fail = False, - auth = {}, - # ignored - # buildifier: disable=unused-variable - canonical_id = "", - # unsupported - executable = False, - integrity = "", - # custom features - headers = {}): - if executable or integrity: - fail("executable and integrity attributes are unsupported.") - - version_result = rctx.execute(["curl", "--version"]) - if version_result.return_code != 0: - fail("Failed to execute curl --version:\n{}".format(version_result.stderr)) - - # parse from - # curl 8.1.2 (x86_64-apple-darwin22.0) libcurl/8.1.2 (SecureTransport) LibreSSL/3.3.6 zlib/1.2.11 nghttp2/1.51.0 - # Release-Date: 2023-05-30 - # ... - curl_version = version_result.stdout.split(" ")[1] - - command = [ - "curl", - url, - "--write-out", - "%{http_code}", - "--location", - "--request", - "GET", - "--create-dirs", - "--output", - output, - ] - - # Detect more flags which may be supported based on changelog: - # https://curl.se/changes.html - if versions.is_at_least("7.67.0", curl_version): - command.append("--no-progress-meter") - - for (name, value) in headers.items(): - command.append("--header") - command.append("{}: {}".format(name, value)) - - command.extend(_auth_to_header(url, auth)) - - result = rctx.execute(command) - _debug("""\nSTDOUT\n{}\nSTDERR\n{}""".format(result.stdout, result.stderr)) - - if result.return_code != 0: - if allow_fail: - return struct(success = False) - else: - fail("Failed to execute curl {} (version {}): {}".format(url, curl_version, result.stderr)) - - status_code = int(result.stdout.strip()) - if status_code >= 400: - if allow_fail: - return struct(success = False) - else: - fail("curl {} returned non-success status code {}".format(url, status_code)) - checksum = util.sha256(rctx, output) - if sha256 and checksum != sha256: - fail("Checksum for url {} was {} but expected {}".format(url, checksum, sha256)) - return struct( - success = True, - sha256 = checksum, - ) - -# A dummy function that uses bazel downloader. -def _bazel_download( - rctx, - headers = {}, - **kwargs): - # Passing headers to the downloader is only available as of 7.1 - if versions.is_at_least("7.1.0", versions.get()): - return rctx.download(headers = headers, **kwargs) - else: - return rctx.download(**kwargs) - -download = struct( - curl = _download, - bazel = _bazel_download, -) diff --git a/oci/private/pull.bzl b/oci/private/pull.bzl index 8e216a29..cc826dfd 100644 --- a/oci/private/pull.bzl +++ b/oci/private/pull.bzl @@ -1,8 +1,8 @@ "Implementation details for oci_pull repository rules" load("@bazel_skylib//lib:dicts.bzl", "dicts") +load("@bazel_skylib//lib:versions.bzl", "versions") load("//oci/private:authn.bzl", "authn") -load("//oci/private:download.bzl", "download") load("//oci/private:util.bzl", "util") # attributes that are specific to image reference url. shared between multiple targets @@ -38,15 +38,16 @@ _IMAGE_REFERENCE_ATTRS = { SCHEMA1_ERROR = """\ The registry sent a manifest with schemaVersion=1. This commonly occurs when fetching from a registry that needs the Docker-Distribution-API-Version header to be set. +See: https://github.com/bazel-contrib/rules_oci/blob/main/docs/pull.md#authentication-using-credential-helpers """ OCI_MEDIA_TYPE_OR_AUTHN_ERROR = """\ -Unable to retrieve the manifest. This could be due to authentication problems or an attempt to fetch an image with OCI image media types. +Unable to retrieve the image manifest. This could be due to authentication problems or an attempt to fetch an image with OCI image media types. +See: https://github.com/bazel-contrib/rules_oci/blob/main/docs/pull.md#authentication-using-credential-helpers """ -CURL_FALLBACK_WARNING = """\ -The use of Curl fallback is deprecated and is set to be removed in version 2.0. -For more details, refer to: https://github.com/bazel-contrib/rules_oci/issues/456 +OCI_MEDIA_TYPE_OR_AUTHN_ERROR_BAZEL7 = """\ +Unable to retrieve the image manifest. This could be due to authentication problems. """ # Supported media types @@ -85,7 +86,7 @@ def _digest_into_blob_path(digest): digest_path = digest.replace(":", "/", 1) return "blobs/{}".format(digest_path) -def _download(rctx, authn, identifier, output, resource, download_fn = download.bazel, headers = {}, allow_fail = False): +def _download(rctx, authn, identifier, output, resource, headers = {}, allow_fail = False): "Use the Bazel Downloader to fetch from the remote registry" if resource != "blobs" and resource != "manifests": @@ -108,17 +109,19 @@ def _download(rctx, authn, identifier, output, resource, download_fn = download. if identifier.startswith("sha256:"): sha256 = identifier[len("sha256:"):] else: - util.warning(rctx, "Fetching from {}@{} without an integrity hash. The result will not be cached.".format(rctx.attr.repository, identifier)) + util.warning(rctx, "Fetching from {}@{} without an integrity hash, result will not be cached.".format(rctx.attr.repository, identifier)) - return download_fn( - rctx, + kwargs = dict( output = output, sha256 = sha256, url = registry_url, auth = {registry_url: auth}, - headers = headers, allow_fail = allow_fail, ) + if versions.is_at_least("7.1.0", versions.get()): + return rctx.download(headers = headers, **kwargs) + else: + return rctx.download(**kwargs) def _download_manifest(rctx, authn, identifier, output): bytes = None @@ -135,35 +138,19 @@ def _download_manifest(rctx, authn, identifier, output): headers = _DOWNLOAD_HEADERS, ) - fallback_to_curl = False if result.success: bytes = rctx.read(output) manifest = json.decode(bytes) digest = "sha256:{}".format(result.sha256) if manifest["schemaVersion"] == 1: - fallback_to_curl = True - util.warning(rctx, SCHEMA1_ERROR) + fail(SCHEMA1_ERROR) else: - fallback_to_curl = True - util.warning(rctx, OCI_MEDIA_TYPE_OR_AUTHN_ERROR) explanation = authn.explain() if explanation: util.warning(rctx, explanation) - - if fallback_to_curl: - util.warning(rctx, CURL_FALLBACK_WARNING) - _download( - rctx, - authn, - identifier, - output, - "manifests", - download.curl, - headers = _DOWNLOAD_HEADERS, + fail( + OCI_MEDIA_TYPE_OR_AUTHN_ERROR_BAZEL7 if versions.is_at_least("7.1.0", versions.get()) else OCI_MEDIA_TYPE_OR_AUTHN_ERROR, ) - bytes = rctx.read(output) - manifest = json.decode(bytes) - digest = "sha256:{}".format(util.sha256(rctx, output)) return manifest, len(bytes), digest