Skip to content

Commit

Permalink
Add new registry.Lookup wrapper to handle generic Reference lookups
Browse files Browse the repository at this point in the history
This allows us to update `cmd/lookup` to use this new wrapper and thus let us easily/correctly test more edge cases / lookups. 🚀
  • Loading branch information
tianon committed Mar 25, 2024
1 parent 073cf83 commit 87afcd2
Show file tree
Hide file tree
Showing 5 changed files with 232 additions and 29 deletions.
55 changes: 54 additions & 1 deletion .test/lookup-test.json
Original file line number Diff line number Diff line change
Expand Up @@ -75,5 +75,58 @@
"annotations": {
"org.opencontainers.image.ref.name": "tianon/test@sha256:347290ddd775c1b85a3e381b09edde95242478eb65153e9b17225356f4c072ac"
}
}
},
{
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.index.v1+json",
"manifests": [
{
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"digest": "sha256:e2fc4e5012d16e7fe466f5291c476431beaa1f9b90a5c2125b493ed28e2aba57",
"size": 861,
"annotations": {
"com.docker.official-images.bashbrew.arch": "amd64",
"org.opencontainers.image.ref.name": "hello-world@sha256:e2fc4e5012d16e7fe466f5291c476431beaa1f9b90a5c2125b493ed28e2aba57",
"org.opencontainers.image.revision": "3fb6ebca4163bf5b9cc496ac3e8f11cb1e754aee",
"org.opencontainers.image.source": "https://github.com/docker-library/hello-world.git#3fb6ebca4163bf5b9cc496ac3e8f11cb1e754aee:amd64/hello-world",
"org.opencontainers.image.url": "https://hub.docker.com/_/hello-world",
"org.opencontainers.image.version": "linux"
}
},
{
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"digest": "sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23",
"size": 946,
"annotations": {
"com.docker.official-images.bashbrew.arch": "windows-amd64",
"org.opencontainers.image.ref.name": "hello-world@sha256:2f19ce27632e6baf4ebb1b582960d68948e52902c8cfac10133da0058f1dab23"
}
},
{
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"digest": "sha256:3a0bd0fb5ad6dd6528dc78726b3df78e980b39b379e99c5a508904ec17cfafe5",
"size": 946,
"annotations": {
"com.docker.official-images.bashbrew.arch": "windows-amd64",
"org.opencontainers.image.ref.name": "hello-world@sha256:3a0bd0fb5ad6dd6528dc78726b3df78e980b39b379e99c5a508904ec17cfafe5"
}
}
]
},
"eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiQ21kIjpbIi9oZWxsbyJdLCJXb3JraW5nRGlyIjoiLyIsIkFyZ3NFc2NhcGVkIjp0cnVlLCJPbkJ1aWxkIjpudWxsfSwiY3JlYXRlZCI6IjIwMjMtMDUtMDJUMTY6NDk6MjdaIiwiaGlzdG9yeSI6W3siY3JlYXRlZCI6IjIwMjMtMDUtMDJUMTY6NDk6MjdaIiwiY3JlYXRlZF9ieSI6IkNPUFkgaGVsbG8gLyAjIGJ1aWxka2l0IiwiY29tbWVudCI6ImJ1aWxka2l0LmRvY2tlcmZpbGUudjAifSx7ImNyZWF0ZWQiOiIyMDIzLTA1LTAyVDE2OjQ5OjI3WiIsImNyZWF0ZWRfYnkiOiJDTUQgW1wiL2hlbGxvXCJdIiwiY29tbWVudCI6ImJ1aWxka2l0LmRvY2tlcmZpbGUudjAiLCJlbXB0eV9sYXllciI6dHJ1ZX1dLCJvcyI6ImxpbnV4Iiwicm9vdGZzIjp7InR5cGUiOiJsYXllcnMiLCJkaWZmX2lkcyI6WyJzaGEyNTY6YWMyODgwMGVjOGJiMzhkNWMzNWI0OWQ0NWE2YWM0Nzc3NTQ0OTQxMTk5MDc1ZGZmOGM0ZWI2M2UwOTNhYTgxZSJdfX0=",
{
"mediaType": "application/vnd.oci.image.index.v1+json",
"digest": "sha256:347290ddd775c1b85a3e381b09edde95242478eb65153e9b17225356f4c072ac",
"size": 1649,
"data": "ewoJInNjaGVtYVZlcnNpb24iOiAyLAoJIm1lZGlhVHlwZSI6ICJhcHBsaWNhdGlvbi92bmQub2NpLmltYWdlLmluZGV4LnYxK2pzb24iLAoJIm1hbmlmZXN0cyI6IFsKCQl7CgkJCSJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLm9jaS5pbWFnZS5tYW5pZmVzdC52MStqc29uIiwKCQkJImRpZ2VzdCI6ICJzaGEyNTY6ZTJmYzRlNTAxMmQxNmU3ZmU0NjZmNTI5MWM0NzY0MzFiZWFhMWY5YjkwYTVjMjEyNWI0OTNlZDI4ZTJhYmE1NyIsCgkJCSJzaXplIjogODYxLAoJCQkiYW5ub3RhdGlvbnMiOiB7CgkJCQkiY29tLmRvY2tlci5vZmZpY2lhbC1pbWFnZXMuYmFzaGJyZXcuYXJjaCI6ICJhbWQ2NCIsCgkJCQkib3JnLm9wZW5jb250YWluZXJzLmltYWdlLnJlZi5uYW1lIjogImhlbGxvLXdvcmxkQHNoYTI1NjplMmZjNGU1MDEyZDE2ZTdmZTQ2NmY1MjkxYzQ3NjQzMWJlYWExZjliOTBhNWMyMTI1YjQ5M2VkMjhlMmFiYTU3IiwKCQkJCSJvcmcub3BlbmNvbnRhaW5lcnMuaW1hZ2UucmV2aXNpb24iOiAiM2ZiNmViY2E0MTYzYmY1YjljYzQ5NmFjM2U4ZjExY2IxZTc1NGFlZSIsCgkJCQkib3JnLm9wZW5jb250YWluZXJzLmltYWdlLnNvdXJjZSI6ICJodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyLWxpYnJhcnkvaGVsbG8td29ybGQuZ2l0IzNmYjZlYmNhNDE2M2JmNWI5Y2M0OTZhYzNlOGYxMWNiMWU3NTRhZWU6YW1kNjQvaGVsbG8td29ybGQiLAoJCQkJIm9yZy5vcGVuY29udGFpbmVycy5pbWFnZS51cmwiOiAiaHR0cHM6Ly9odWIuZG9ja2VyLmNvbS9fL2hlbGxvLXdvcmxkIiwKCQkJCSJvcmcub3BlbmNvbnRhaW5lcnMuaW1hZ2UudmVyc2lvbiI6ICJsaW51eCIKCQkJfQoJCX0sCgkJewoJCQkibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuZGlzdHJpYnV0aW9uLm1hbmlmZXN0LnYyK2pzb24iLAoJCQkiZGlnZXN0IjogInNoYTI1NjoyZjE5Y2UyNzYzMmU2YmFmNGViYjFiNTgyOTYwZDY4OTQ4ZTUyOTAyYzhjZmFjMTAxMzNkYTAwNThmMWRhYjIzIiwKCQkJInNpemUiOiA5NDYsCgkJCSJhbm5vdGF0aW9ucyI6IHsKCQkJCSJjb20uZG9ja2VyLm9mZmljaWFsLWltYWdlcy5iYXNoYnJldy5hcmNoIjogIndpbmRvd3MtYW1kNjQiLAoJCQkJIm9yZy5vcGVuY29udGFpbmVycy5pbWFnZS5yZWYubmFtZSI6ICJoZWxsby13b3JsZEBzaGEyNTY6MmYxOWNlMjc2MzJlNmJhZjRlYmIxYjU4Mjk2MGQ2ODk0OGU1MjkwMmM4Y2ZhYzEwMTMzZGEwMDU4ZjFkYWIyMyIKCQkJfQoJCX0sCgkJewoJCQkibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuZGlzdHJpYnV0aW9uLm1hbmlmZXN0LnYyK2pzb24iLAoJCQkiZGlnZXN0IjogInNoYTI1NjozYTBiZDBmYjVhZDZkZDY1MjhkYzc4NzI2YjNkZjc4ZTk4MGIzOWIzNzllOTljNWE1MDg5MDRlYzE3Y2ZhZmU1IiwKCQkJInNpemUiOiA5NDYsCgkJCSJhbm5vdGF0aW9ucyI6IHsKCQkJCSJjb20uZG9ja2VyLm9mZmljaWFsLWltYWdlcy5iYXNoYnJldy5hcmNoIjogIndpbmRvd3MtYW1kNjQiLAoJCQkJIm9yZy5vcGVuY29udGFpbmVycy5pbWFnZS5yZWYubmFtZSI6ICJoZWxsby13b3JsZEBzaGEyNTY6M2EwYmQwZmI1YWQ2ZGQ2NTI4ZGM3ODcyNmIzZGY3OGU5ODBiMzliMzc5ZTk5YzVhNTA4OTA0ZWMxN2NmYWZlNSIKCQkJfQoJCX0KCV0KfQo="
},
{
"mediaType": "application/octet-stream",
"digest": "sha256:d2c94e258dcb3c5ac2798d32e1249e42ef01cba4841c2234249495f87264ac5a",
"size": 581,
"data": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiQ21kIjpbIi9oZWxsbyJdLCJXb3JraW5nRGlyIjoiLyIsIkFyZ3NFc2NhcGVkIjp0cnVlLCJPbkJ1aWxkIjpudWxsfSwiY3JlYXRlZCI6IjIwMjMtMDUtMDJUMTY6NDk6MjdaIiwiaGlzdG9yeSI6W3siY3JlYXRlZCI6IjIwMjMtMDUtMDJUMTY6NDk6MjdaIiwiY3JlYXRlZF9ieSI6IkNPUFkgaGVsbG8gLyAjIGJ1aWxka2l0IiwiY29tbWVudCI6ImJ1aWxka2l0LmRvY2tlcmZpbGUudjAifSx7ImNyZWF0ZWQiOiIyMDIzLTA1LTAyVDE2OjQ5OjI3WiIsImNyZWF0ZWRfYnkiOiJDTUQgW1wiL2hlbGxvXCJdIiwiY29tbWVudCI6ImJ1aWxka2l0LmRvY2tlcmZpbGUudjAiLCJlbXB0eV9sYXllciI6dHJ1ZX1dLCJvcyI6ImxpbnV4Iiwicm9vdGZzIjp7InR5cGUiOiJsYXllcnMiLCJkaWZmX2lkcyI6WyJzaGEyNTY6YWMyODgwMGVjOGJiMzhkNWMzNWI0OWQ0NWE2YWM0Nzc3NTQ0OTQxMTk5MDc1ZGZmOGM0ZWI2M2UwOTNhYTgxZSJdfX0="
},
null,
null,
null
]
11 changes: 11 additions & 0 deletions .test/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,17 @@ lookup=(
# tianon/test:index-no-platform-smaller - a "broken" index with *zero* platform objects in it (so every manifest requires a platform lookup)
'tianon/test@sha256:347290ddd775c1b85a3e381b09edde95242478eb65153e9b17225356f4c072ac'
# (doing these in the same run means the manifest from above should be cached and exercise more codepaths for better coverage)

--type manifest 'tianon/test@sha256:347290ddd775c1b85a3e381b09edde95242478eb65153e9b17225356f4c072ac' # same manifest again, but without SynthesizeIndex
--type blob 'tianon/test@sha256:d2c94e258dcb3c5ac2798d32e1249e42ef01cba4841c2234249495f87264ac5a' # first config blob from the above
# and again, but this time HEADs
--head --type manifest 'tianon/test@sha256:347290ddd775c1b85a3e381b09edde95242478eb65153e9b17225356f4c072ac'
--head --type blob 'tianon/test@sha256:d2c94e258dcb3c5ac2798d32e1249e42ef01cba4841c2234249495f87264ac5a'

# exercise 404 codepaths
"tianon/this-is-a-repository-that-will-never-ever-exist-$RANDOM-$RANDOM:$RANDOM-$RANDOM"
--head "tianon/this-is-a-repository-that-will-never-ever-exist-$RANDOM-$RANDOM:$RANDOM-$RANDOM"
'tianon/test@sha256:0000000000000000000000000000000000000000000000000000000000000000'
)
"$dir/../bin/lookup" "${lookup[@]}" | jq -s > "$dir/lookup-test.json"

Expand Down
69 changes: 64 additions & 5 deletions cmd/lookup/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package main
import (
"context"
"encoding/json"
"io"
"os"
"os/signal"

Expand All @@ -15,21 +16,79 @@ func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
defer stop()

for _, img := range os.Args[1:] {
var (
zeroOpts registry.LookupOptions
opts = zeroOpts
)

args := os.Args[1:]
for len(args) > 0 {
img := args[0]
args = args[1:]
switch img {
case "--type":
opts.Type = registry.LookupType(args[0])
args = args[1:]
continue
case "--head":
opts.Head = true
continue
}

ref, err := registry.ParseRef(img)
if err != nil {
panic(err)
}

index, err := registry.SynthesizeIndex(ctx, ref)
if err != nil {
panic(err)
var obj any
if opts == zeroOpts {
// if we have no explicit type and didn't request a HEAD, invoke SynthesizeIndex instead of Lookup
obj, err = registry.SynthesizeIndex(ctx, ref)
if err != nil {
panic(err)
}
} else {
r, err := registry.Lookup(ctx, ref, &opts)
if err != nil {
panic(err)
}
if r != nil {
desc := r.Descriptor()
if opts.Head {
obj = desc
} else {
b, err := io.ReadAll(r)
if err != nil {
r.Close()
panic(err)
}
if opts.Type == registry.LookupTypeManifest {
// if it was a manifest lookup, cast the byte slice to json.RawMessage so we get the actual JSON (not base64)
obj = json.RawMessage(b)
} else {
obj = b
}
}
err = r.Close()
if err != nil {
panic(err)
}
} else {
obj = nil
}
}

e := json.NewEncoder(os.Stdout)
e.SetIndent("", "\t")
if err := e.Encode(index); err != nil {
if err := e.Encode(obj); err != nil {
panic(err)
}

// reset state
opts = zeroOpts
}

if opts != zeroOpts {
panic("dangling --type, --head, etc (without a following reference for it to apply to)")
}
}
99 changes: 99 additions & 0 deletions registry/lookup.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package registry

import (
"context"
"errors"
"fmt"
"strings"

"cuelabs.dev/go/oci/ociregistry"
"cuelabs.dev/go/oci/ociregistry/ocimem"
)

// see `LookupType*` consts for possible values for this type
type LookupType string

const (
LookupTypeManifest LookupType = "manifest"
LookupTypeBlob LookupType = "blob"
)

type LookupOptions struct {
// unspecified implies [LookupTypeManifest]
Type LookupType

// whether or not to do a HEAD instead of a GET (will still return an [ociregistry.BlobReader], but with an empty body / zero bytes)
Head bool
}

// a wrapper around [ociregistry.Interface.GetManifest] (and `GetTag`, `GetBlob`, and the `Resolve*` versions of the above) that accepts a [Reference] and always returns a [ociregistry.BlobReader] (in the case of a HEAD request, it will be a zero-length reader with just a valid descriptor)
func Lookup(ctx context.Context, ref Reference, opts *LookupOptions) (ociregistry.BlobReader, error) {
client, err := Client(ref.Host, nil)
if err != nil {
return nil, fmt.Errorf("%s: failed getting client: %w", ref, err)
}

var o LookupOptions
if opts != nil {
o = *opts
}

var (
r ociregistry.BlobReader
desc ociregistry.Descriptor
)
switch o.Type {
case LookupTypeManifest, "":
if ref.Digest != "" {
if o.Head {
desc, err = client.ResolveManifest(ctx, ref.Repository, ref.Digest)
} else {
r, err = client.GetManifest(ctx, ref.Repository, ref.Digest)
}
} else {
tag := ref.Tag
if tag == "" {
tag = "latest"
}
if o.Head {
desc, err = client.ResolveTag(ctx, ref.Repository, tag)
} else {
r, err = client.GetTag(ctx, ref.Repository, tag)
}
}

case LookupTypeBlob:
// TODO error if Digest == "" ? (ociclient already does for us, so we can probably just pass it through here without much worry)
if o.Head {
desc, err = client.ResolveBlob(ctx, ref.Repository, ref.Digest)
} else {
r, err = client.GetBlob(ctx, ref.Repository, ref.Digest)
}

default:
return nil, fmt.Errorf("unknown LookupType: %q", o.Type)
}

// normalize 404 and 404-like to nil return (so it's easier to detect)
if err != nil {
if errors.Is(err, ociregistry.ErrBlobUnknown) ||
errors.Is(err, ociregistry.ErrManifestUnknown) ||
errors.Is(err, ociregistry.ErrNameUnknown) {
// obvious 404 cases
return nil, nil
}
// https://github.com/cue-labs/oci/issues/26
if errStr := strings.TrimPrefix(err.Error(), "error response: "); strings.HasPrefix(errStr, "404 ") ||
// 401 often means "repository not found" (due to the nature of public/private mixing on Hub and the fact that ociauth definitely handled any possible authentication for us, so if we're still getting 401 it's unavoidable and might as well be 404)
strings.HasPrefix(errStr, "401 ") {
return nil, nil
}
return r, err
}

if o.Head {
r = ocimem.NewBytesReader(nil, desc)
}

return r, err
}
27 changes: 4 additions & 23 deletions registry/synthesize-index.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,7 @@ package registry

import (
"context"
"errors"
"fmt"
"strings"

"github.com/docker-library/bashbrew/architecture"

Expand All @@ -15,35 +13,18 @@ import (

// returns a synthesized [ocispec.Index] object for the given reference that includes automatically pulling up [ocispec.Platform] objects for entries missing them plus annotations for bashbrew architecture ([AnnotationBashbrewArch]) and where to find the "upstream" object if it needs to be copied/pulled ([ocispec.AnnotationRefName])
func SynthesizeIndex(ctx context.Context, ref Reference) (*ocispec.Index, error) {
// consider making this a full ociregistry.Interface object? GetManifest(digest) not returning an object with that digest would certainly be Weird though so maybe that's a misguided idea (with very minimal actual benefit, at least right now)

client, err := Client(ref.Host, nil)
if err != nil {
return nil, fmt.Errorf("%s: failed getting client: %w", ref, err)
}

var r ociregistry.BlobReader = nil
if ref.Digest != "" {
r, err = client.GetManifest(ctx, ref.Repository, ref.Digest)
} else {
tag := ref.Tag
if tag == "" {
tag = "latest"
}
r, err = client.GetTag(ctx, ref.Repository, tag)
}
r, err := Lookup(ctx, ref, nil)
if err != nil {
// https://github.com/cue-labs/oci/issues/26
if errors.Is(err, ociregistry.ErrBlobUnknown) ||
errors.Is(err, ociregistry.ErrManifestUnknown) ||
errors.Is(err, ociregistry.ErrNameUnknown) ||
strings.HasPrefix(err.Error(), "404 ") ||
// 401 often means "repository not found" (due to the nature of public/private mixing on Hub and the fact that ociauth definitely handled any possible authentication for us, so if we're still getting 401 it's unavoidable and might as well be 404)
strings.HasPrefix(err.Error(), "401 ") {
return nil, nil
}
return nil, fmt.Errorf("%s: failed GET: %w", ref, err)
}
if r == nil {
return nil, nil
}
defer r.Close()

desc := r.Descriptor()
Expand Down

0 comments on commit 87afcd2

Please sign in to comment.