From 313c40bba81258feca7f154c93528857ee050e0d Mon Sep 17 00:00:00 2001 From: Josh Wolf Date: Mon, 13 Dec 2021 11:30:57 -0700 Subject: [PATCH] standardize content naming for unnamed content --- cmd/hauler/cli/download/download.go | 10 ++-- cmd/hauler/cli/store/add.go | 35 ++++++++----- cmd/hauler/cli/store/extract.go | 51 +++++++++--------- cmd/hauler/cli/store/info.go | 8 ++- pkg/apis/hauler.cattle.io/v1alpha1/file.go | 8 ++- pkg/apis/hauler.cattle.io/v1alpha1/image.go | 3 +- pkg/collection/chart/chart.go | 4 +- pkg/collection/chart/dependents.go | 2 +- pkg/content/file/file.go | 18 +++---- pkg/content/image/image.go | 12 ++--- pkg/reference/reference.go | 45 ++++++++++++++++ pkg/reference/reference_test.go | 57 +++++++++++++++++++++ pkg/store/store.go | 9 ++-- testdata/contents.yaml | 16 +++--- 14 files changed, 200 insertions(+), 78 deletions(-) create mode 100644 pkg/reference/reference.go create mode 100644 pkg/reference/reference_test.go diff --git a/cmd/hauler/cli/download/download.go b/cmd/hauler/cli/download/download.go index b00f8677..ac438c9d 100644 --- a/cmd/hauler/cli/download/download.go +++ b/cmd/hauler/cli/download/download.go @@ -5,7 +5,6 @@ import ( "encoding/json" "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" @@ -15,6 +14,7 @@ import ( "github.com/rancherfederal/hauler/internal/mapper" "github.com/rancherfederal/hauler/pkg/consts" "github.com/rancherfederal/hauler/pkg/log" + "github.com/rancherfederal/hauler/pkg/reference" ) type Opts struct { @@ -36,7 +36,7 @@ func (o *Opts) AddArgs(cmd *cobra.Command) { f.BoolVar(&o.PlainHTTP, "plain-http", false, "Toggle allowing plain http connections when copying to a remote registry") } -func Cmd(ctx context.Context, o *Opts, reference string) error { +func Cmd(ctx context.Context, o *Opts, ref string) error { l := log.FromContext(ctx) rs, err := content.NewRegistry(content.RegistryOptions{ @@ -49,12 +49,12 @@ func Cmd(ctx context.Context, o *Opts, reference string) error { return err } - ref, err := name.ParseReference(reference) + r, err := reference.Parse(ref) if err != nil { return err } - desc, err := remote.Get(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain), remote.WithContext(ctx)) + desc, err := remote.Get(r, remote.WithAuthFromKeychain(authn.DefaultKeychain), remote.WithContext(ctx)) if err != nil { return err } @@ -74,7 +74,7 @@ func Cmd(ctx context.Context, o *Opts, reference string) error { return err } - pushedDesc, err := oras.Copy(ctx, rs, ref.Name(), mapperStore, "", + pushedDesc, err := oras.Copy(ctx, rs, r.Name(), mapperStore, "", oras.WithAdditionalCachedMediaTypes(consts.DockerManifestSchema2)) if err != nil { return err diff --git a/cmd/hauler/cli/store/add.go b/cmd/hauler/cli/store/add.go index 7a53a141..4bdba13d 100644 --- a/cmd/hauler/cli/store/add.go +++ b/cmd/hauler/cli/store/add.go @@ -2,7 +2,6 @@ package store import ( "context" - "fmt" "os" "github.com/google/go-containerregistry/pkg/name" @@ -14,6 +13,7 @@ import ( "github.com/rancherfederal/hauler/pkg/content/file" "github.com/rancherfederal/hauler/pkg/content/image" "github.com/rancherfederal/hauler/pkg/log" + "github.com/rancherfederal/hauler/pkg/reference" "github.com/rancherfederal/hauler/pkg/store" ) @@ -87,7 +87,7 @@ func (o *AddFileOpts) AddFlags(cmd *cobra.Command) { func AddFileCmd(ctx context.Context, o *AddFileOpts, s *store.Store, reference string) error { cfg := v1alpha1.File{ - Ref: reference, + Path: reference, } return storeFile(ctx, s, cfg) @@ -96,9 +96,13 @@ func AddFileCmd(ctx context.Context, o *AddFileOpts, s *store.Store, reference s func storeFile(ctx context.Context, s *store.Store, fi v1alpha1.File) error { l := log.FromContext(ctx) - f := file.NewFile(fi.Ref) + f := file.NewFile(fi.Path) + ref, err := reference.NewTagged(f.Name(fi.Path), reference.DefaultTag) + if err != nil { + return err + } - desc, err := s.AddArtifact(ctx, f, f.Name(fi.Ref)) + desc, err := s.AddArtifact(ctx, f, ref.Name()) if err != nil { return err } @@ -118,7 +122,7 @@ func (o *AddImageOpts) AddFlags(cmd *cobra.Command) { func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Store, reference string) error { cfg := v1alpha1.Image{ - Ref: reference, + Name: reference, } return storeImage(ctx, s, cfg) @@ -127,17 +131,22 @@ func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Store, reference func storeImage(ctx context.Context, s *store.Store, i v1alpha1.Image) error { l := log.FromContext(ctx) - oci, err := image.NewImage(i.Ref) + oci, err := image.NewImage(i.Name) + if err != nil { + return err + } + + r, err := name.ParseReference(i.Name) if err != nil { return err } - desc, err := s.AddArtifact(ctx, oci, i.Ref) + desc, err := s.AddArtifact(ctx, oci, r.Name()) if err != nil { return err } - l.With(log.Fields{"type": s.Identify(ctx, desc)}).Infof("added [%s] to store", i.Ref) + l.With(log.Fields{"type": s.Identify(ctx, desc)}).Infof("added [%s] to store", i.Name) return nil } @@ -178,13 +187,11 @@ func storeChart(ctx context.Context, s *store.Store, cfg v1alpha1.Chart) error { return err } - tag := cfg.Version - if tag == "" { - tag = name.DefaultTag + ref, err := reference.NewTagged(cfg.Name, cfg.Version) + if err != nil { + return err } - - ref := fmt.Sprintf("%s:%s", cfg.Name, tag) - desc, err := s.AddArtifact(ctx, oci, ref) + desc, err := s.AddArtifact(ctx, oci, ref.Name()) if err != nil { return err } diff --git a/cmd/hauler/cli/store/extract.go b/cmd/hauler/cli/store/extract.go index 2e1dda0a..7ebf7ab3 100644 --- a/cmd/hauler/cli/store/extract.go +++ b/cmd/hauler/cli/store/extract.go @@ -3,14 +3,14 @@ package store import ( "context" "encoding/json" + "fmt" - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/layout" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" "github.com/rancherfederal/hauler/internal/mapper" "github.com/rancherfederal/hauler/pkg/log" + "github.com/rancherfederal/hauler/pkg/reference" "github.com/rancherfederal/hauler/pkg/store" ) @@ -24,53 +24,52 @@ func (o *ExtractOpts) AddArgs(cmd *cobra.Command) { f.StringVar(&o.DestinationDir, "dir", "", "Directory to save contents to (defaults to current directory)") } -func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Store, reference string) error { +func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Store, ref string) error { l := log.FromContext(ctx) - ref, err := name.ParseReference(reference, name.WithDefaultRegistry(""), name.WithDefaultTag("latest")) + r, err := reference.Parse(ref) if err != nil { return err } - p, err := layout.FromPath("store") - if err != nil { - return err - } - - ii, _ := p.ImageIndex() - im, _ := ii.IndexManifest() - var manifest ocispec.Manifest - for _, m := range im.Manifests { - if r, ok := m.Annotations[ocispec.AnnotationRefName]; !ok || r != ref.Name() { - continue + found := false + if err := s.Content.Walk(func(reference string, desc ocispec.Descriptor) error { + if reference != r.Name() { + return nil } + found = true - desc, err := p.Image(m.Digest) + rc, err := s.Content.Fetch(ctx, desc) if err != nil { return err } - l.Infof(m.Annotations[ocispec.AnnotationRefName]) + defer rc.Close() + + var m ocispec.Manifest + if err := json.NewDecoder(rc).Decode(&m); err != nil { + return err + } - manifestData, err := desc.RawManifest() + mapperStore, err := mapper.FromManifest(m, o.DestinationDir) if err != nil { return err } - if err := json.Unmarshal(manifestData, &manifest); err != nil { + pushedDesc, err := s.Copy(ctx, r.Name(), mapperStore, "") + if err != nil { return err } - } - mapperStore, err := mapper.FromManifest(manifest, o.DestinationDir) - if err != nil { + l.Infof("downloaded [%s] with digest [%s]", pushedDesc.MediaType, pushedDesc.Digest.String()) + + return nil + }); err != nil { return err } - desc, err := s.Copy(ctx, ref.Name(), mapperStore, "") - if err != nil { - return err + if !found { + return fmt.Errorf("reference [%s] not found in store (hint: use `hauler store info` to list store contents)", ref) } - l.Infof("downloaded [%s] with digest [%s]", desc.MediaType, desc.Digest.String()) return nil } diff --git a/cmd/hauler/cli/store/info.go b/cmd/hauler/cli/store/info.go index a211ebf4..07e2e7e2 100644 --- a/cmd/hauler/cli/store/info.go +++ b/cmd/hauler/cli/store/info.go @@ -11,6 +11,7 @@ import ( "github.com/spf13/cobra" "github.com/rancherfederal/hauler/pkg/consts" + "github.com/rancherfederal/hauler/pkg/reference" "github.com/rancherfederal/hauler/pkg/store" ) @@ -115,8 +116,13 @@ func newItem(s *store.Store, desc ocispec.Descriptor, m ocispec.Manifest) item { ctype = "unknown" } + ref, err := reference.Parse(desc.Annotations[ocispec.AnnotationRefName]) + if err != nil { + return item{} + } + return item{ - Reference: desc.Annotations[ocispec.AnnotationRefName], + Reference: ref.Context().RepositoryStr(), Type: ctype, Layers: len(m.Layers), Size: byteCountSI(size), diff --git a/pkg/apis/hauler.cattle.io/v1alpha1/file.go b/pkg/apis/hauler.cattle.io/v1alpha1/file.go index 9856da9b..b28e8214 100644 --- a/pkg/apis/hauler.cattle.io/v1alpha1/file.go +++ b/pkg/apis/hauler.cattle.io/v1alpha1/file.go @@ -18,5 +18,11 @@ type FileSpec struct { } type File struct { - Ref string `json:"ref"` + // Path is the path to the file contents, can be a local or remote path + Path string `json:"path"` + + // Reference is an optionally defined reference to the contents within the store + // If not specified, this will be generated as follows: + // hauler/:latest + Reference string `json:"reference,omitempty"` } diff --git a/pkg/apis/hauler.cattle.io/v1alpha1/image.go b/pkg/apis/hauler.cattle.io/v1alpha1/image.go index 338a975a..cde4d2c0 100644 --- a/pkg/apis/hauler.cattle.io/v1alpha1/image.go +++ b/pkg/apis/hauler.cattle.io/v1alpha1/image.go @@ -18,5 +18,6 @@ type ImageSpec struct { } type Image struct { - Ref string `json:"ref"` + // Name is the full location for the image, can be referenced by tags or digests + Name string `json:"name"` } diff --git a/pkg/collection/chart/chart.go b/pkg/collection/chart/chart.go index 14612320..700dd59d 100644 --- a/pkg/collection/chart/chart.go +++ b/pkg/collection/chart/chart.go @@ -74,11 +74,11 @@ func (c *tchart) dependentImages() error { } for _, img := range imgs.Spec.Images { - i, err := image.NewImage(img.Ref) + i, err := image.NewImage(img.Name) if err != nil { return err } - c.contents[img.Ref] = i + c.contents[img.Name] = i } return nil } diff --git a/pkg/collection/chart/dependents.go b/pkg/collection/chart/dependents.go index f790f44d..d74f6f38 100644 --- a/pkg/collection/chart/dependents.go +++ b/pkg/collection/chart/dependents.go @@ -48,7 +48,7 @@ func ImagesInChart(c *helmchart.Chart) (v1alpha1.Images, error) { found := find(raw, defaultKnownImagePaths...) for _, f := range found { - images = append(images, v1alpha1.Image{Ref: f}) + images = append(images, v1alpha1.Image{Name: f}) } } diff --git a/pkg/content/file/file.go b/pkg/content/file/file.go index 10aea79e..373ba25e 100644 --- a/pkg/content/file/file.go +++ b/pkg/content/file/file.go @@ -16,9 +16,9 @@ import ( var _ artifact.OCI = (*File)(nil) // File implements the OCI interface for File API objects. API spec information is -// stored into the Ref field. +// stored into the Path field. type File struct { - Ref string + Path string client *getter.Client @@ -29,12 +29,12 @@ type File struct { annotations map[string]string } -func NewFile(ref string, opts ...Option) *File { +func NewFile(path string, opts ...Option) *File { client := getter.NewClient(getter.ClientOptions{}) f := &File{ client: client, - Ref: ref, + Path: path, } for _, opt := range opts { @@ -43,8 +43,8 @@ func NewFile(ref string, opts ...Option) *File { return f } -func (f *File) Name(ref string) string { - return f.client.Name(ref) +func (f *File) Name(path string) string { + return f.client.Name(path) } func (f *File) MediaType() string { @@ -80,7 +80,7 @@ func (f *File) compute() error { } ctx := context.Background() - blob, err := f.client.LayerFrom(ctx, f.Ref) + blob, err := f.client.LayerFrom(ctx, f.Path) if err != nil { return err } @@ -90,9 +90,9 @@ func (f *File) compute() error { return err } - cfg := f.client.Config(f.Ref) + cfg := f.client.Config(f.Path) if cfg == nil { - cfg = f.client.Config(f.Ref) + cfg = f.client.Config(f.Path) } cfgDesc, err := partial.Descriptor(cfg) diff --git a/pkg/content/image/image.go b/pkg/content/image/image.go index 18ca9e6f..fd37cc91 100644 --- a/pkg/content/image/image.go +++ b/pkg/content/image/image.go @@ -2,7 +2,7 @@ package image import ( "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" + gname "github.com/google/go-containerregistry/pkg/name" gv1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" @@ -24,14 +24,14 @@ func (i *Image) RawConfig() ([]byte, error) { } // Image implements the OCI interface for Image API objects. API spec information -// is stored into the Ref field. +// is stored into the Name field. type Image struct { - Ref string + Name string gv1.Image } -func NewImage(ref string, opts ...remote.Option) (*Image, error) { - r, err := name.ParseReference(ref) +func NewImage(name string, opts ...remote.Option) (*Image, error) { + r, err := gname.ParseReference(name) if err != nil { return nil, err } @@ -47,7 +47,7 @@ func NewImage(ref string, opts ...remote.Option) (*Image, error) { } return &Image{ - Ref: ref, + Name: name, Image: img, }, nil } diff --git a/pkg/reference/reference.go b/pkg/reference/reference.go new file mode 100644 index 00000000..d79d71a1 --- /dev/null +++ b/pkg/reference/reference.go @@ -0,0 +1,45 @@ +package reference + +import ( + "strings" + + gname "github.com/google/go-containerregistry/pkg/name" +) + +const ( + DefaultNamespace = "hauler" + DefaultTag = "latest" +) + +type Reference interface { + Name() string +} + +// NewTagged will create a new docker.NamedTagged given a path-component +func NewTagged(n string, tag string) (gname.Reference, error) { + repo, err := Parse(n) + if err != nil { + return nil, err + } + + return repo.Context().Tag(tag), nil +} + +// Parse will parse a reference and return a name.Reference namespaced with DefaultNamespace if necessary +func Parse(ref string) (gname.Reference, error) { + repo, err := gname.ParseReference(ref, gname.WithDefaultRegistry("")) + if err != nil { + return nil, err + } + + if !strings.ContainsRune(repo.String(), '/') { + ref = DefaultNamespace + "/" + repo.String() + } + + r, err := gname.ParseReference(ref) + if err != nil { + return nil, err + } + + return r, nil +} diff --git a/pkg/reference/reference_test.go b/pkg/reference/reference_test.go new file mode 100644 index 00000000..f3fea2a8 --- /dev/null +++ b/pkg/reference/reference_test.go @@ -0,0 +1,57 @@ +package reference_test + +import ( + "reflect" + "testing" + + "github.com/rancherfederal/hauler/pkg/reference" +) + +func TestParse(t *testing.T) { + type args struct { + ref string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "Should add hauler namespace when doesn't exist", + args: args{ + ref: "myfile", + }, + want: "index.docker.io/hauler/myfile:latest", + wantErr: false, + }, + { + name: "Shouldn't modify fully qualified reference", + args: args{ + ref: "index.docker.io/library/registry@sha256:42043edfae481178f07aa077fa872fcc242e276d302f4ac2026d9d2eb65b955f", + }, + want: "index.docker.io/library/registry@sha256:42043edfae481178f07aa077fa872fcc242e276d302f4ac2026d9d2eb65b955f", + wantErr: false, + }, + { + name: "Shouldn't modify library", + args: args{ + ref: "library/alpine", + }, + want: "index.docker.io/library/alpine:latest", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := reference.Parse(tt.args.ref) + if (err != nil) != tt.wantErr { + t.Errorf("Parse() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got.Name(), tt.want) { + t.Errorf("Parse() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/store/store.go b/pkg/store/store.go index ffb77953..ea191612 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -15,6 +15,7 @@ import ( "github.com/rancherfederal/hauler/internal/cache" "github.com/rancherfederal/hauler/pkg/artifact" "github.com/rancherfederal/hauler/pkg/consts" + "github.com/rancherfederal/hauler/pkg/reference" ) type Store struct { @@ -46,7 +47,7 @@ func NewStore(rootdir string, opts ...Options) (*Store, error) { // saved, the entirety of the layout is copied to the store (which is just a registry). This allows us to not only use // strict types to define generic content, but provides a processing pipeline suitable for extensibility. In the // future we'll allow users to define their own content that must adhere either by artifact.OCI or simply an OCI layout. -func (s *Store) AddArtifact(ctx context.Context, oci artifact.OCI, reference string) (ocispec.Descriptor, error) { +func (s *Store) AddArtifact(ctx context.Context, oci artifact.OCI, r string) (ocispec.Descriptor, error) { stage, err := newLayout() if err != nil { return ocispec.Descriptor{}, err @@ -57,10 +58,10 @@ func (s *Store) AddArtifact(ctx context.Context, oci artifact.OCI, reference str oci = cached } - // Ensure that index.docker.io isn't prepended - ref, err := name.ParseReference(reference, name.WithDefaultRegistry(""), name.WithDefaultTag("latest")) + // Validate we have a locatable reference + ref, err := reference.Parse(r) if err != nil { - return ocispec.Descriptor{}, fmt.Errorf("%w", ErrInvalidReference) + return ocispec.Descriptor{}, err } if err := stage.add(ctx, oci, ref); err != nil { diff --git a/testdata/contents.yaml b/testdata/contents.yaml index b68b3768..25eb4136 100644 --- a/testdata/contents.yaml +++ b/testdata/contents.yaml @@ -5,17 +5,17 @@ metadata: spec: files: # hauler can save/redistribute files on disk (be careful! paths are relative) - - ref: testdata/contents.yaml + - path: testdata/contents.yaml # TODO: when directories are specified, they will be archived and stored as a file -# - ref: testdata/ +# - path: testdata/ # hauler can also fetch remote content, and will "smartly" identify filenames _when possible_ # filename below = "k3s-images.txt" - - ref: "https://github.com/k3s-io/k3s/releases/download/v1.22.2%2Bk3s2/k3s-images.txt" + - path: "https://github.com/k3s-io/k3s/releases/download/v1.22.2%2Bk3s2/k3s-images.txt" # when filenames are not appropriate, a name should be specified - - ref: https://get.k3s.io?filename=get-k3s.sh + - path: https://get.k3s.io?filename=get-k3s.sh --- apiVersion: content.hauler.cattle.io/v1alpha1 @@ -25,16 +25,16 @@ metadata: spec: images: # images can be referenced shorthanded without a tag - - ref: hello-world + - name: hello-world # or namespaced with a tag - - ref: rancher/cowsay:latest + - name: rancher/cowsay:latest # or by their digest: -# - ref: registry@sha256:42043edfae481178f07aa077fa872fcc242e276d302f4ac2026d9d2eb65b955f + - name: registry@sha256:42043edfae481178f07aa077fa872fcc242e276d302f4ac2026d9d2eb65b955f # or fully qualified from any OCI compliant registry registry - - ref: ghcr.io/fluxcd/flux-cli:v0.22.0 + - name: ghcr.io/fluxcd/flux-cli:v0.22.0 --- apiVersion: content.hauler.cattle.io/v1alpha1