From 8b372d8a2026f0659bf48cf1d2a95e77a638d905 Mon Sep 17 00:00:00 2001 From: Josh Wolf Date: Wed, 12 Jan 2022 09:47:09 -0700 Subject: [PATCH] factor out core oci logic into independent library (rancherfederal/ocil) --- cmd/hauler/cli/cli.go | 13 +- cmd/hauler/cli/download/download.go | 3 +- cmd/hauler/cli/store/add.go | 26 +-- cmd/hauler/cli/store/copy.go | 12 +- cmd/hauler/cli/store/extract.go | 9 +- cmd/hauler/cli/store/info.go | 14 +- cmd/hauler/cli/store/serve.go | 5 +- cmd/hauler/cli/store/sync.go | 9 +- go.mod | 14 +- go.sum | 12 +- internal/cache/cache.go | 102 ------------ internal/cache/doc.go | 5 - internal/cache/filesystem.go | 120 -------------- internal/getter/directory.go | 165 ------------------- internal/getter/file.go | 53 ------- internal/getter/getter.go | 114 -------------- internal/getter/getter_test.go | 129 --------------- internal/getter/https.go | 67 -------- internal/layer/layer.go | 127 --------------- internal/mapper/mappers.go | 2 +- pkg/artifact/config.go | 92 ----------- pkg/artifact/oci.go | 23 --- pkg/collection/chart/chart.go | 15 +- pkg/collection/k3s/k3s.go | 20 +-- pkg/consts/consts.go | 43 ----- pkg/content/chart/chart.go | 10 +- pkg/content/chart/chart_test.go | 3 +- pkg/content/file/file.go | 116 -------------- pkg/content/file/file_test.go | 166 ------------------- pkg/content/file/options.go | 26 --- pkg/content/image/image.go | 53 ------- pkg/content/image/image_test.go | 1 - pkg/reference/reference.go | 18 +++ pkg/store/errors.go | 7 - pkg/store/oci.go | 236 ---------------------------- pkg/store/options.go | 13 -- pkg/store/stager.go | 192 ---------------------- pkg/store/store.go | 184 ---------------------- pkg/store/store_test.go | 213 ------------------------- 39 files changed, 112 insertions(+), 2320 deletions(-) delete mode 100644 internal/cache/cache.go delete mode 100644 internal/cache/doc.go delete mode 100644 internal/cache/filesystem.go delete mode 100644 internal/getter/directory.go delete mode 100644 internal/getter/file.go delete mode 100644 internal/getter/getter.go delete mode 100644 internal/getter/getter_test.go delete mode 100644 internal/getter/https.go delete mode 100644 internal/layer/layer.go delete mode 100644 pkg/artifact/config.go delete mode 100644 pkg/artifact/oci.go delete mode 100644 pkg/consts/consts.go delete mode 100644 pkg/content/file/file.go delete mode 100644 pkg/content/file/file_test.go delete mode 100644 pkg/content/file/options.go delete mode 100644 pkg/content/image/image.go delete mode 100644 pkg/content/image/image_test.go delete mode 100644 pkg/store/errors.go delete mode 100644 pkg/store/oci.go delete mode 100644 pkg/store/options.go delete mode 100644 pkg/store/stager.go delete mode 100644 pkg/store/store.go delete mode 100644 pkg/store/store_test.go diff --git a/cmd/hauler/cli/cli.go b/cmd/hauler/cli/cli.go index 5d2d45ff..96deb062 100644 --- a/cmd/hauler/cli/cli.go +++ b/cmd/hauler/cli/cli.go @@ -6,11 +6,12 @@ import ( "os" "path/filepath" + "github.com/rancherfederal/ocil/pkg/layer" "github.com/spf13/cobra" - cache2 "github.com/rancherfederal/hauler/internal/cache" + "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/log" - "github.com/rancherfederal/hauler/pkg/store" ) type rootOpts struct { @@ -55,7 +56,7 @@ func New() *cobra.Command { return cmd } -func (o *rootOpts) getStore(ctx context.Context) (*store.Store, error) { +func (o *rootOpts) getStore(ctx context.Context) (*store.Layout, error) { l := log.FromContext(ctx) dir := o.storeDir @@ -80,14 +81,14 @@ func (o *rootOpts) getStore(ctx context.Context) (*store.Store, error) { return nil, err } - s, err := store.NewStore(abs, store.WithCache(c)) + s, err := store.NewLayout(abs, store.WithCache(c)) if err != nil { return nil, err } return s, nil } -func (o *rootOpts) getCache(ctx context.Context) (cache2.Cache, error) { +func (o *rootOpts) getCache(ctx context.Context) (layer.Cache, error) { dir := o.cacheDir if dir == "" { @@ -105,6 +106,6 @@ func (o *rootOpts) getCache(ctx context.Context) (cache2.Cache, error) { dir = abs } - c := cache2.NewFilesystem(dir) + c := layer.NewFilesystemCache(dir) return c, nil } diff --git a/cmd/hauler/cli/download/download.go b/cmd/hauler/cli/download/download.go index ac438c9d..de812dce 100644 --- a/cmd/hauler/cli/download/download.go +++ b/cmd/hauler/cli/download/download.go @@ -11,8 +11,9 @@ import ( "oras.land/oras-go/pkg/content" "oras.land/oras-go/pkg/oras" + "github.com/rancherfederal/ocil/pkg/consts" + "github.com/rancherfederal/hauler/internal/mapper" - "github.com/rancherfederal/hauler/pkg/consts" "github.com/rancherfederal/hauler/pkg/log" "github.com/rancherfederal/hauler/pkg/reference" ) diff --git a/cmd/hauler/cli/store/add.go b/cmd/hauler/cli/store/add.go index 8c2ebab9..1b3ef839 100644 --- a/cmd/hauler/cli/store/add.go +++ b/cmd/hauler/cli/store/add.go @@ -7,13 +7,15 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/spf13/cobra" + "github.com/rancherfederal/ocil/pkg/artifacts/file" + "github.com/rancherfederal/ocil/pkg/artifacts/image" + + "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" "github.com/rancherfederal/hauler/pkg/content/chart" - "github.com/rancherfederal/hauler/pkg/content/file" - "github.com/rancherfederal/hauler/pkg/content/image" "github.com/rancherfederal/hauler/pkg/log" "github.com/rancherfederal/hauler/pkg/reference" - "github.com/rancherfederal/hauler/pkg/store" ) type AddFileOpts struct { @@ -25,7 +27,7 @@ func (o *AddFileOpts) AddFlags(cmd *cobra.Command) { f.StringVarP(&o.Name, "name", "n", "", "(Optional) Name to assign to file in store") } -func AddFileCmd(ctx context.Context, o *AddFileOpts, s *store.Store, reference string) error { +func AddFileCmd(ctx context.Context, o *AddFileOpts, s *store.Layout, reference string) error { cfg := v1alpha1.File{ Path: reference, } @@ -33,7 +35,7 @@ func AddFileCmd(ctx context.Context, o *AddFileOpts, s *store.Store, reference s return storeFile(ctx, s, cfg) } -func storeFile(ctx context.Context, s *store.Store, fi v1alpha1.File) error { +func storeFile(ctx context.Context, s *store.Layout, fi v1alpha1.File) error { l := log.FromContext(ctx) f := file.NewFile(fi.Path) @@ -42,7 +44,7 @@ func storeFile(ctx context.Context, s *store.Store, fi v1alpha1.File) error { return err } - desc, err := s.AddArtifact(ctx, f, ref.Name()) + desc, err := s.AddOCI(ctx, f, ref.Name()) if err != nil { return err } @@ -60,7 +62,7 @@ func (o *AddImageOpts) AddFlags(cmd *cobra.Command) { _ = f } -func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Store, reference string) error { +func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Layout, reference string) error { cfg := v1alpha1.Image{ Name: reference, } @@ -68,7 +70,7 @@ func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Store, reference return storeImage(ctx, s, cfg) } -func storeImage(ctx context.Context, s *store.Store, i v1alpha1.Image) error { +func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image) error { l := log.FromContext(ctx) oci, err := image.NewImage(i.Name) @@ -81,7 +83,7 @@ func storeImage(ctx context.Context, s *store.Store, i v1alpha1.Image) error { return err } - desc, err := s.AddArtifact(ctx, oci, r.Name()) + desc, err := s.AddOCI(ctx, oci, r.Name()) if err != nil { return err } @@ -104,7 +106,7 @@ func (o *AddChartOpts) AddFlags(cmd *cobra.Command) { f.StringVar(&o.Version, "version", "", "(Optional) Version of the chart to download, defaults to latest if not specified") } -func AddChartCmd(ctx context.Context, o *AddChartOpts, s *store.Store, chartName string) error { +func AddChartCmd(ctx context.Context, o *AddChartOpts, s *store.Layout, chartName string) error { path := "" if _, err := os.Stat(chartName); err == nil { path = chartName @@ -119,7 +121,7 @@ func AddChartCmd(ctx context.Context, o *AddChartOpts, s *store.Store, chartName return storeChart(ctx, s, cfg) } -func storeChart(ctx context.Context, s *store.Store, cfg v1alpha1.Chart) error { +func storeChart(ctx context.Context, s *store.Layout, cfg v1alpha1.Chart) error { l := log.FromContext(ctx) oci, err := chart.NewChart(cfg) @@ -131,7 +133,7 @@ func storeChart(ctx context.Context, s *store.Store, cfg v1alpha1.Chart) error { if err != nil { return err } - desc, err := s.AddArtifact(ctx, oci, ref.Name()) + desc, err := s.AddOCI(ctx, oci, ref.Name()) if err != nil { return err } diff --git a/cmd/hauler/cli/store/copy.go b/cmd/hauler/cli/store/copy.go index bb59a484..51e844ba 100644 --- a/cmd/hauler/cli/store/copy.go +++ b/cmd/hauler/cli/store/copy.go @@ -9,8 +9,10 @@ import ( "github.com/spf13/cobra" "oras.land/oras-go/pkg/content" + "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/log" - "github.com/rancherfederal/hauler/pkg/store" + "github.com/rancherfederal/hauler/pkg/reference" ) type CopyOpts struct { @@ -31,7 +33,7 @@ func (o *CopyOpts) AddFlags(cmd *cobra.Command) { f.BoolVar(&o.PlainHTTP, "plain-http", false, "Toggle allowing plain http connections when copying to a remote registry") } -func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Store, targetRef string) error { +func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string) error { l := log.FromContext(ctx) var descs []ocispec.Descriptor @@ -60,12 +62,12 @@ func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Store, targetRef string) return err } - mapperFn := func(reference string) (string, error) { - ref, err := store.RelocateReference(reference, components[1]) + mapperFn := func(ref string) (string, error) { + r, err := reference.Relocate(ref, components[1]) if err != nil { return "", err } - return ref.Name(), nil + return r.Name(), nil } ds, err := s.CopyAll(ctx, r, mapperFn) diff --git a/cmd/hauler/cli/store/extract.go b/cmd/hauler/cli/store/extract.go index fa7b4f5d..9d24469d 100644 --- a/cmd/hauler/cli/store/extract.go +++ b/cmd/hauler/cli/store/extract.go @@ -8,10 +8,11 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" + "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/internal/mapper" "github.com/rancherfederal/hauler/pkg/log" "github.com/rancherfederal/hauler/pkg/reference" - "github.com/rancherfederal/hauler/pkg/store" ) type ExtractOpts struct { @@ -24,7 +25,7 @@ func (o *ExtractOpts) AddArgs(cmd *cobra.Command) { f.StringVarP(&o.DestinationDir, "output", "o", "", "Directory to save contents to (defaults to current directory)") } -func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Store, ref string) error { +func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Layout, ref string) error { l := log.FromContext(ctx) r, err := reference.Parse(ref) @@ -33,13 +34,13 @@ func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Store, ref string) } found := false - if err := s.Content.Walk(func(reference string, desc ocispec.Descriptor) error { + if err := s.Walk(func(reference string, desc ocispec.Descriptor) error { if reference != r.Name() { return nil } found = true - rc, err := s.Content.Fetch(ctx, desc) + rc, err := s.Fetch(ctx, desc) if err != nil { return err } diff --git a/cmd/hauler/cli/store/info.go b/cmd/hauler/cli/store/info.go index b8d83659..439a0ef9 100644 --- a/cmd/hauler/cli/store/info.go +++ b/cmd/hauler/cli/store/info.go @@ -10,9 +10,11 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" - "github.com/rancherfederal/hauler/pkg/consts" + "github.com/rancherfederal/ocil/pkg/consts" + + "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/reference" - "github.com/rancherfederal/hauler/pkg/store" ) type InfoOpts struct { @@ -28,14 +30,14 @@ func (o *InfoOpts) AddFlags(cmd *cobra.Command) { // TODO: Regex/globbing } -func InfoCmd(ctx context.Context, o *InfoOpts, s *store.Store) error { +func InfoCmd(ctx context.Context, o *InfoOpts, s *store.Layout) error { var items []item - if err := s.Content.Walk(func(ref string, desc ocispec.Descriptor) error { + if err := s.Walk(func(ref string, desc ocispec.Descriptor) error { if _, ok := desc.Annotations[ocispec.AnnotationRefName]; !ok { return nil } - rc, err := s.Content.Fetch(ctx, desc) + rc, err := s.Fetch(ctx, desc) if err != nil { return err } @@ -97,7 +99,7 @@ type item struct { Size string } -func newItem(s *store.Store, desc ocispec.Descriptor, m ocispec.Manifest) item { +func newItem(s *store.Layout, desc ocispec.Descriptor, m ocispec.Manifest) item { var size int64 = 0 for _, l := range m.Layers { size = +l.Size diff --git a/cmd/hauler/cli/store/serve.go b/cmd/hauler/cli/store/serve.go index e1d344f3..37b378f8 100644 --- a/cmd/hauler/cli/store/serve.go +++ b/cmd/hauler/cli/store/serve.go @@ -14,8 +14,9 @@ import ( "github.com/distribution/distribution/v3/version" "github.com/spf13/cobra" + "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/internal/server" - "github.com/rancherfederal/hauler/pkg/store" ) type ServeOpts struct { @@ -37,7 +38,7 @@ func (o *ServeOpts) AddFlags(cmd *cobra.Command) { } // ServeCmd serves the embedded registry almost identically to how distribution/v3 does it -func ServeCmd(ctx context.Context, o *ServeOpts, s *store.Store) error { +func ServeCmd(ctx context.Context, o *ServeOpts, s *store.Layout) error { ctx = dcontext.WithVersion(ctx, version.Version) tr := server.NewTempRegistry(ctx, o.RootDir) diff --git a/cmd/hauler/cli/store/sync.go b/cmd/hauler/cli/store/sync.go index 67a72c84..4d254f9c 100644 --- a/cmd/hauler/cli/store/sync.go +++ b/cmd/hauler/cli/store/sync.go @@ -10,12 +10,13 @@ import ( "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/util/yaml" + "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" tchart "github.com/rancherfederal/hauler/pkg/collection/chart" "github.com/rancherfederal/hauler/pkg/collection/k3s" "github.com/rancherfederal/hauler/pkg/content" "github.com/rancherfederal/hauler/pkg/log" - "github.com/rancherfederal/hauler/pkg/store" ) type SyncOpts struct { @@ -28,7 +29,7 @@ func (o *SyncOpts) AddFlags(cmd *cobra.Command) { f.StringSliceVarP(&o.ContentFiles, "files", "f", []string{}, "Path to content files") } -func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error { +func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error { l := log.FromContext(ctx) // Start from an empty store (contents are cached elsewhere) @@ -120,7 +121,7 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error { return err } - if _, err := s.AddCollection(ctx, k); err != nil { + if _, err := s.AddOCICollection(ctx, k); err != nil { return err } @@ -136,7 +137,7 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error { return err } - if _, err := s.AddCollection(ctx, tc); err != nil { + if _, err := s.AddOCICollection(ctx, tc); err != nil { return err } } diff --git a/go.mod b/go.mod index 6f25421c..a48109d9 100644 --- a/go.mod +++ b/go.mod @@ -10,14 +10,12 @@ require ( github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/mholt/archiver/v3 v3.5.1 - github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.0.2 github.com/pkg/errors v0.9.1 + github.com/rancherfederal/ocil v0.1.6 github.com/rs/zerolog v1.26.0 github.com/sirupsen/logrus v1.8.1 - github.com/spf13/afero v1.6.0 github.com/spf13/cobra v1.2.1 - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c helm.sh/helm/v3 v3.6.1-0.20211203140009-0e6e936ab84e k8s.io/apimachinery v0.22.4 k8s.io/client-go v0.22.4 @@ -48,7 +46,6 @@ require ( github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b // indirect github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect - github.com/containerd/stargz-snapshotter/estargz v0.10.0 // indirect github.com/cyphar/filepath-securejoin v0.2.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/cli v20.10.11+incompatible // indirect @@ -112,13 +109,14 @@ require ( github.com/nwaples/rardecode v1.1.0 // indirect github.com/onsi/ginkgo v1.16.4 // indirect github.com/onsi/gomega v1.15.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.11.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.26.0 // indirect - github.com/prometheus/procfs v0.6.0 // indirect + github.com/prometheus/common v0.30.0 // indirect + github.com/prometheus/procfs v0.7.3 // indirect github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc // indirect github.com/russross/blackfriday v1.5.2 // indirect github.com/shopspring/decimal v1.2.0 // indirect @@ -126,7 +124,6 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.7.0 // indirect github.com/ulikunitz/xz v0.5.9 // indirect - github.com/vbatts/tar-split v0.11.2 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect @@ -139,9 +136,10 @@ require ( golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 // indirect golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 // indirect golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect - golang.org/x/text v0.3.6 // indirect + golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20211111162719-482062a4217b // indirect diff --git a/go.sum b/go.sum index cbd068b7..fe1e85e7 100644 --- a/go.sum +++ b/go.sum @@ -852,8 +852,9 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.30.0 h1:JEkYlQnpzrzQFxi6gnukFPdQ+ac82oRhzMcIduJu/Ug= +github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -864,9 +865,12 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rancherfederal/ocil v0.1.6 h1:PncaQJkny10WHr3bYOuM6uXVUUdMki/xooNF0XiMGvo= +github.com/rancherfederal/ocil v0.1.6/go.mod h1:l4d1cHHfdXDGtio32AYDjG6n1i1JxQK+kAom0cVf0SY= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -1153,6 +1157,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211111160137-58aab5ef257a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= @@ -1296,8 +1301,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/internal/cache/cache.go b/internal/cache/cache.go deleted file mode 100644 index 781bcd92..00000000 --- a/internal/cache/cache.go +++ /dev/null @@ -1,102 +0,0 @@ -package cache - -import ( - "errors" - "io" - - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/types" - - "github.com/rancherfederal/hauler/pkg/artifact" -) - -type Cache interface { - Put(v1.Layer) (v1.Layer, error) - - Get(v1.Hash) (v1.Layer, error) -} - -var ErrLayerNotFound = errors.New("layer not found") - -type oci struct { - artifact.OCI - - c Cache -} - -func Oci(o artifact.OCI, c Cache) artifact.OCI { - return &oci{ - OCI: o, - c: c, - } -} - -func (o *oci) Layers() ([]v1.Layer, error) { - ls, err := o.OCI.Layers() - if err != nil { - return nil, err - } - - var out []v1.Layer - for _, l := range ls { - out = append(out, &lazyLayer{inner: l, c: o.c}) - } - return out, nil -} - -type lazyLayer struct { - inner v1.Layer - c Cache -} - -func (l *lazyLayer) Compressed() (io.ReadCloser, error) { - digest, err := l.inner.Digest() - if err != nil { - return nil, err - } - - layer, err := l.getOrPut(digest) - if err != nil { - return nil, err - } - - return layer.Compressed() -} - -func (l *lazyLayer) Uncompressed() (io.ReadCloser, error) { - diffID, err := l.inner.DiffID() - if err != nil { - return nil, err - } - - layer, err := l.getOrPut(diffID) - if err != nil { - return nil, err - } - - return layer.Uncompressed() -} - -func (l *lazyLayer) getOrPut(h v1.Hash) (v1.Layer, error) { - var layer v1.Layer - if cl, err := l.c.Get(h); err == nil { - layer = cl - - } else if err == ErrLayerNotFound { - rl, err := l.c.Put(l.inner) - if err != nil { - return nil, err - } - layer = rl - - } else { - return nil, err - } - - return layer, nil -} - -func (l *lazyLayer) Size() (int64, error) { return l.inner.Size() } -func (l *lazyLayer) DiffID() (v1.Hash, error) { return l.inner.Digest() } -func (l *lazyLayer) Digest() (v1.Hash, error) { return l.inner.Digest() } -func (l *lazyLayer) MediaType() (types.MediaType, error) { return l.inner.MediaType() } diff --git a/internal/cache/doc.go b/internal/cache/doc.go deleted file mode 100644 index 9a0a2b8a..00000000 --- a/internal/cache/doc.go +++ /dev/null @@ -1,5 +0,0 @@ -package cache - -/* -This package is _heavily_ influenced by go-containerregistry and it's cache implementation: https://github.com/google/go-containerregistry/tree/main/pkg/v1/cache -*/ diff --git a/internal/cache/filesystem.go b/internal/cache/filesystem.go deleted file mode 100644 index dc6e147c..00000000 --- a/internal/cache/filesystem.go +++ /dev/null @@ -1,120 +0,0 @@ -package cache - -import ( - "io" - "os" - "path/filepath" - - v1 "github.com/google/go-containerregistry/pkg/v1" - - "github.com/rancherfederal/hauler/internal/layer" -) - -type fs struct { - root string -} - -func NewFilesystem(root string) Cache { - return &fs{root: root} -} - -func (f *fs) Put(l v1.Layer) (v1.Layer, error) { - digest, err := l.Digest() - if err != nil { - return nil, err - } - diffID, err := l.DiffID() - if err != nil { - return nil, err - } - return &cachedLayer{ - Layer: l, - root: f.root, - digest: digest, - diffID: diffID, - }, nil -} - -func (f *fs) Get(h v1.Hash) (v1.Layer, error) { - opener := f.open(h) - l, err := layer.FromOpener(opener) - if os.IsNotExist(err) { - return nil, ErrLayerNotFound - } - return l, err -} - -func (f *fs) open(h v1.Hash) layer.Opener { - return func() (io.ReadCloser, error) { - return os.Open(layerpath(f.root, h)) - } -} - -type cachedLayer struct { - v1.Layer - - root string - digest, diffID v1.Hash -} - -func (l *cachedLayer) create(h v1.Hash) (io.WriteCloser, error) { - lp := layerpath(l.root, h) - if err := os.MkdirAll(filepath.Dir(lp), os.ModePerm); err != nil { - return nil, err - } - return os.Create(lp) -} - -func (l *cachedLayer) Compressed() (io.ReadCloser, error) { - f, err := l.create(l.digest) - if err != nil { - return nil, nil - } - rc, err := l.Layer.Compressed() - if err != nil { - return nil, err - } - return &readcloser{ - t: io.TeeReader(rc, f), - closes: []func() error{rc.Close, f.Close}, - }, nil -} - -func (l *cachedLayer) Uncompressed() (io.ReadCloser, error) { - f, err := l.create(l.diffID) - if err != nil { - return nil, err - } - rc, err := l.Layer.Uncompressed() - if err != nil { - return nil, err - } - return &readcloser{ - t: io.TeeReader(rc, f), - closes: []func() error{rc.Close, f.Close}, - }, nil -} - -func layerpath(root string, h v1.Hash) string { - return filepath.Join(root, h.Algorithm, h.Hex) -} - -type readcloser struct { - t io.Reader - closes []func() error -} - -func (rc *readcloser) Read(b []byte) (int, error) { - return rc.t.Read(b) -} - -func (rc *readcloser) Close() error { - var err error - for _, c := range rc.closes { - lastErr := c() - if err == nil { - err = lastErr - } - } - return err -} diff --git a/internal/getter/directory.go b/internal/getter/directory.go deleted file mode 100644 index 4a27e70b..00000000 --- a/internal/getter/directory.go +++ /dev/null @@ -1,165 +0,0 @@ -package getter - -import ( - "archive/tar" - "compress/gzip" - "context" - "io" - "net/url" - "os" - "path/filepath" - "time" - - "github.com/opencontainers/go-digest" - "github.com/pkg/errors" - - "github.com/rancherfederal/hauler/pkg/artifact" - "github.com/rancherfederal/hauler/pkg/consts" -) - -type directory struct { - *File -} - -func NewDirectory() *directory { - return &directory{File: NewFile()} -} - -func (d directory) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) { - tmpfile, err := os.CreateTemp("", "hauler") - if err != nil { - return nil, err - } - - digester := digest.Canonical.Digester() - zw := gzip.NewWriter(io.MultiWriter(tmpfile, digester.Hash())) - defer zw.Close() - - tarDigester := digest.Canonical.Digester() - if err := tarDir(d.path(u), d.Name(u), io.MultiWriter(zw, tarDigester.Hash()), false); err != nil { - return nil, err - } - - if err := zw.Close(); err != nil { - return nil, err - } - if err := tmpfile.Sync(); err != nil { - return nil, err - } - - fi, err := os.Open(tmpfile.Name()) - if err != nil { - return nil, err - } - - // rc := &closer{ - // t: io.TeeReader(tmpfile, fi), - // closes: []func() error{fi.Close, tmpfile.Close, zw.Close}, - // } - return fi, nil -} - -func (d directory) Detect(u *url.URL) bool { - if len(d.path(u)) == 0 { - return false - } - - fi, err := os.Stat(d.path(u)) - if err != nil { - return false - } - return fi.IsDir() -} - -func (d directory) Config(u *url.URL) artifact.Config { - c := &directoryConfig{ - config{Reference: u.String()}, - } - return artifact.ToConfig(c, artifact.WithConfigMediaType(consts.FileDirectoryConfigMediaType)) -} - -type directoryConfig struct { - config `json:",inline,omitempty"` -} - -func tarDir(root string, prefix string, w io.Writer, stripTimes bool) error { - tw := tar.NewWriter(w) - defer tw.Close() - if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - // Rename path - name, err := filepath.Rel(root, path) - if err != nil { - return err - } - name = filepath.Join(prefix, name) - name = filepath.ToSlash(name) - - // Generate header - var link string - mode := info.Mode() - if mode&os.ModeSymlink != 0 { - if link, err = os.Readlink(path); err != nil { - return err - } - } - header, err := tar.FileInfoHeader(info, link) - if err != nil { - return errors.Wrap(err, path) - } - header.Name = name - header.Uid = 0 - header.Gid = 0 - header.Uname = "" - header.Gname = "" - - if stripTimes { - header.ModTime = time.Time{} - header.AccessTime = time.Time{} - header.ChangeTime = time.Time{} - } - - // Write file - if err := tw.WriteHeader(header); err != nil { - return errors.Wrap(err, "tar") - } - if mode.IsRegular() { - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - if _, err := io.Copy(tw, file); err != nil { - return errors.Wrap(err, path) - } - } - - return nil - }); err != nil { - return err - } - return nil -} - -type closer struct { - t io.Reader - closes []func() error -} - -func (c *closer) Read(p []byte) (n int, err error) { - return c.t.Read(p) -} - -func (c *closer) Close() error { - var err error - for _, c := range c.closes { - lastErr := c() - if err == nil { - err = lastErr - } - } - return err -} diff --git a/internal/getter/file.go b/internal/getter/file.go deleted file mode 100644 index 60284950..00000000 --- a/internal/getter/file.go +++ /dev/null @@ -1,53 +0,0 @@ -package getter - -import ( - "context" - "io" - "net/url" - "os" - "path/filepath" - - "github.com/rancherfederal/hauler/pkg/artifact" - "github.com/rancherfederal/hauler/pkg/consts" -) - -type File struct{} - -func NewFile() *File { - return &File{} -} - -func (f File) Name(u *url.URL) string { - return filepath.Base(f.path(u)) -} - -func (f File) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) { - return os.Open(f.path(u)) -} - -func (f File) Detect(u *url.URL) bool { - if len(f.path(u)) == 0 { - return false - } - - fi, err := os.Stat(f.path(u)) - if err != nil { - return false - } - return !fi.IsDir() -} - -func (f File) path(u *url.URL) string { - return filepath.Join(u.Host, u.Path) -} - -func (f File) Config(u *url.URL) artifact.Config { - c := &fileConfig{ - config{Reference: u.String()}, - } - return artifact.ToConfig(c, artifact.WithConfigMediaType(consts.FileLocalConfigMediaType)) -} - -type fileConfig struct { - config `json:",inline,omitempty"` -} diff --git a/internal/getter/getter.go b/internal/getter/getter.go deleted file mode 100644 index 029d167f..00000000 --- a/internal/getter/getter.go +++ /dev/null @@ -1,114 +0,0 @@ -package getter - -import ( - "context" - "io" - "net/url" - - v1 "github.com/google/go-containerregistry/pkg/v1" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" - "oras.land/oras-go/pkg/content" - - "github.com/rancherfederal/hauler/internal/layer" - "github.com/rancherfederal/hauler/pkg/artifact" - "github.com/rancherfederal/hauler/pkg/consts" -) - -type Client struct { - Getters map[string]Getter - Options ClientOptions -} - -// TODO: Make some valid ClientOptions -type ClientOptions struct{} - -var ( - ErrGetterTypeUnknown = errors.New("no getter type found matching reference") -) - -type Getter interface { - Open(context.Context, *url.URL) (io.ReadCloser, error) - - Detect(*url.URL) bool - - Name(*url.URL) string - - Config(*url.URL) artifact.Config -} - -func NewClient(opts ClientOptions) *Client { - defaults := map[string]Getter{ - "file": NewFile(), - "directory": NewDirectory(), - "http": NewHttp(), - } - - c := &Client{ - Getters: defaults, - Options: opts, - } - return c -} - -func (c *Client) LayerFrom(ctx context.Context, source string) (v1.Layer, error) { - u, err := url.Parse(source) - if err != nil { - return nil, err - } - for _, g := range c.Getters { - if g.Detect(u) { - opener := func() (io.ReadCloser, error) { - return g.Open(ctx, u) - } - - annotations := make(map[string]string) - annotations[ocispec.AnnotationTitle] = g.Name(u) - - switch g.(type) { - case *directory: - annotations[content.AnnotationUnpack] = "true" - } - - l, err := layer.FromOpener(opener, - layer.WithMediaType(consts.FileLayerMediaType), - layer.WithAnnotations(annotations)) - if err != nil { - return nil, err - } - return l, nil - } - } - return nil, errors.Wrapf(ErrGetterTypeUnknown, "%s", source) -} - -func (c *Client) Name(source string) string { - u, err := url.Parse(source) - if err != nil { - return source - } - for _, g := range c.Getters { - if g.Detect(u) { - return g.Name(u) - } - } - return source -} - -func (c *Client) Config(source string) artifact.Config { - u, err := url.Parse(source) - if err != nil { - return nil - } - for _, g := range c.Getters { - if g.Detect(u) { - return g.Config(u) - } - } - return nil -} - -type config struct { - Reference string `json:"reference"` - Annotations map[string]string `json:"annotations,omitempty"` -} diff --git a/internal/getter/getter_test.go b/internal/getter/getter_test.go deleted file mode 100644 index d98ca5dd..00000000 --- a/internal/getter/getter_test.go +++ /dev/null @@ -1,129 +0,0 @@ -package getter_test - -import ( - "net/url" - "os" - "path/filepath" - "testing" - - "github.com/rancherfederal/hauler/internal/getter" -) - -func TestClient_Detect(t *testing.T) { - teardown := setup(t) - defer teardown() - - c := getter.NewClient(getter.ClientOptions{}) - - type args struct { - source string - } - tests := []struct { - name string - args args - want string - }{ - { - name: "should identify a file", - args: args{ - source: fileWithExt, - }, - want: "file", - }, - { - name: "should identify a directory", - args: args{ - source: rootDir, - }, - want: "directory", - }, - { - name: "should identify a http", - args: args{ - source: "http://my.cool.website", - }, - want: "http", - }, - { - name: "should identify a http", - args: args{ - source: "https://my.cool.website", - }, - want: "http", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := identify(c, tt.args.source); got != tt.want { - t.Errorf("identify() = %v, want %v", got, tt.want) - } - }) - } -} - -func identify(c *getter.Client, source string) string { - u, _ := url.Parse(source) - for t, g := range c.Getters { - if g.Detect(u) { - return t - } - } - return "" -} - -func TestClient_Name(t *testing.T) { - teardown := setup(t) - defer teardown() - - c := getter.NewClient(getter.ClientOptions{}) - - type args struct { - source string - } - tests := []struct { - name string - args args - want string - }{ - { - name: "should correctly name a file with an extension", - args: args{ - source: fileWithExt, - }, - want: "file.yaml", - }, - { - name: "should correctly name a directory", - args: args{ - source: rootDir, - }, - want: rootDir, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := c.Name(tt.args.source); got != tt.want { - t.Errorf("Name() = %v, want %v", got, tt.want) - } - }) - } -} - -var ( - rootDir = "gettertests" - fileWithExt = filepath.Join(rootDir, "file.yaml") -) - -func setup(t *testing.T) func() { - if err := os.MkdirAll(rootDir, os.ModePerm); err != nil { - t.Fatal(err) - } - - if err := os.WriteFile(fileWithExt, []byte(""), 0644); err != nil { - t.Fatal(err) - } - - return func() { - os.RemoveAll(rootDir) - } -} diff --git a/internal/getter/https.go b/internal/getter/https.go deleted file mode 100644 index 61c561a8..00000000 --- a/internal/getter/https.go +++ /dev/null @@ -1,67 +0,0 @@ -package getter - -import ( - "context" - "io" - "mime" - "net/http" - "net/url" - "path/filepath" - "strings" - - "github.com/rancherfederal/hauler/pkg/artifact" - "github.com/rancherfederal/hauler/pkg/consts" -) - -type Http struct{} - -func NewHttp() *Http { - return &Http{} -} - -func (h Http) Name(u *url.URL) string { - resp, err := http.Head(u.String()) - if err != nil { - return "" - } - - contentType := resp.Header.Get("Content-Type") - for _, v := range strings.Split(contentType, ",") { - t, _, err := mime.ParseMediaType(v) - if err != nil { - break - } - // TODO: Identify known mimetypes for hints at a filename - _ = t - } - - // TODO: Not this - return filepath.Base(u.String()) -} - -func (h Http) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) { - resp, err := http.Get(u.String()) - if err != nil { - return nil, err - } - return resp.Body, nil -} - -func (h Http) Detect(u *url.URL) bool { - switch u.Scheme { - case "http", "https": - return true - } - return false -} - -func (h *Http) Config(u *url.URL) artifact.Config { - c := &httpConfig{ - config{Reference: u.String()}, - } - return artifact.ToConfig(c, artifact.WithConfigMediaType(consts.FileHttpConfigMediaType)) -} - -type httpConfig struct { - config `json:",inline,omitempty"` -} diff --git a/internal/layer/layer.go b/internal/layer/layer.go deleted file mode 100644 index fdfbf6a8..00000000 --- a/internal/layer/layer.go +++ /dev/null @@ -1,127 +0,0 @@ -package layer - -import ( - "io" - - v1 "github.com/google/go-containerregistry/pkg/v1" - gtypes "github.com/google/go-containerregistry/pkg/v1/types" - - "github.com/rancherfederal/hauler/pkg/consts" -) - -type Opener func() (io.ReadCloser, error) - -func FromOpener(opener Opener, opts ...Option) (v1.Layer, error) { - var err error - - layer := &layer{ - mediaType: consts.UnknownLayer, - annotations: make(map[string]string, 1), - } - - layer.uncompressedOpener = opener - layer.compressedOpener = func() (io.ReadCloser, error) { - rc, err := opener() - if err != nil { - return nil, err - } - - return rc, nil - } - - for _, opt := range opts { - opt(layer) - } - - if layer.digest, layer.size, err = compute(layer.uncompressedOpener); err != nil { - return nil, err - } - - if layer.diffID, _, err = compute(layer.compressedOpener); err != nil { - return nil, err - } - - return layer, nil -} - -func compute(opener Opener) (v1.Hash, int64, error) { - rc, err := opener() - if err != nil { - return v1.Hash{}, 0, err - } - defer rc.Close() - return v1.SHA256(rc) -} - -type Option func(*layer) - -func WithMediaType(mt string) Option { - return func(l *layer) { - l.mediaType = mt - } -} - -func WithAnnotations(annotations map[string]string) Option { - return func(l *layer) { - if l.annotations == nil { - l.annotations = make(map[string]string) - } - l.annotations = annotations - } -} - -type layer struct { - digest v1.Hash - diffID v1.Hash - size int64 - compressedOpener Opener - uncompressedOpener Opener - mediaType string - annotations map[string]string - urls []string -} - -func (l layer) Descriptor() (*v1.Descriptor, error) { - digest, err := l.Digest() - if err != nil { - return nil, err - } - mt, err := l.MediaType() - if err != nil { - return nil, err - } - return &v1.Descriptor{ - MediaType: mt, - Size: l.size, - Digest: digest, - Annotations: l.annotations, - URLs: l.urls, - - // TODO: Allow platforms - Platform: nil, - }, nil -} - -func (l layer) Digest() (v1.Hash, error) { - return l.digest, nil -} - -func (l layer) DiffID() (v1.Hash, error) { - return l.diffID, nil -} - -func (l layer) Compressed() (io.ReadCloser, error) { - return l.compressedOpener() -} - -func (l layer) Uncompressed() (io.ReadCloser, error) { - return l.uncompressedOpener() -} - -func (l layer) Size() (int64, error) { - return l.size, nil -} - -func (l layer) MediaType() (gtypes.MediaType, error) { - return gtypes.MediaType(l.mediaType), nil -} diff --git a/internal/mapper/mappers.go b/internal/mapper/mappers.go index 05cf89bb..1cc60d48 100644 --- a/internal/mapper/mappers.go +++ b/internal/mapper/mappers.go @@ -6,7 +6,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/pkg/target" - "github.com/rancherfederal/hauler/pkg/consts" + "github.com/rancherfederal/ocil/pkg/consts" ) type Fn func(desc ocispec.Descriptor) (string, error) diff --git a/pkg/artifact/config.go b/pkg/artifact/config.go deleted file mode 100644 index 94eb69e3..00000000 --- a/pkg/artifact/config.go +++ /dev/null @@ -1,92 +0,0 @@ -package artifact - -import ( - "bytes" - - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/partial" - "github.com/google/go-containerregistry/pkg/v1/types" - "k8s.io/apimachinery/pkg/util/json" - - "github.com/rancherfederal/hauler/pkg/consts" -) - -var _ partial.Describable = (*marshallableConfig)(nil) - -type Config interface { - // Raw returns the config bytes - Raw() ([]byte, error) - - Digest() (v1.Hash, error) - - MediaType() (types.MediaType, error) - - Size() (int64, error) -} - -type Marshallable interface{} - -type ConfigOption func(*marshallableConfig) - -// ToConfig takes anything that is marshallabe and converts it into a Config -func ToConfig(i Marshallable, opts ...ConfigOption) Config { - mc := &marshallableConfig{Marshallable: i} - for _, o := range opts { - o(mc) - } - return mc -} - -func WithConfigMediaType(mediaType string) ConfigOption { - return func(config *marshallableConfig) { - config.mediaType = mediaType - } -} - -// marshallableConfig implements Config using helper methods -type marshallableConfig struct { - Marshallable - - mediaType string -} - -func (c *marshallableConfig) MediaType() (types.MediaType, error) { - mt := c.mediaType - if mt == "" { - mt = consts.UnknownManifest - } - return types.MediaType(mt), nil -} - -func (c *marshallableConfig) Raw() ([]byte, error) { - return json.Marshal(c.Marshallable) -} - -func (c *marshallableConfig) Digest() (v1.Hash, error) { - return Digest(c) -} - -func (c *marshallableConfig) Size() (int64, error) { - return Size(c) -} - -type WithRawConfig interface { - Raw() ([]byte, error) -} - -func Digest(c WithRawConfig) (v1.Hash, error) { - b, err := c.Raw() - if err != nil { - return v1.Hash{}, err - } - digest, _, err := v1.SHA256(bytes.NewReader(b)) - return digest, err -} - -func Size(c WithRawConfig) (int64, error) { - b, err := c.Raw() - if err != nil { - return -1, err - } - return int64(len(b)), nil -} diff --git a/pkg/artifact/oci.go b/pkg/artifact/oci.go deleted file mode 100644 index 8faf43df..00000000 --- a/pkg/artifact/oci.go +++ /dev/null @@ -1,23 +0,0 @@ -package artifact - -import ( - "github.com/google/go-containerregistry/pkg/v1" -) - -// OCI is the bare minimum we need to represent an artifact in an oci layout -// At a high level, it is not constrained by an Image's config, manifests, and layer ordinality -// This specific implementation fully encapsulates v1.Layer's within a more generic form -type OCI interface { - MediaType() string - - Manifest() (*v1.Manifest, error) - - RawConfig() ([]byte, error) - - Layers() ([]v1.Layer, error) -} - -type Collection interface { - // Contents returns the list of contents in the collection - Contents() (map[string]OCI, error) -} diff --git a/pkg/collection/chart/chart.go b/pkg/collection/chart/chart.go index 700dd59d..846b71ee 100644 --- a/pkg/collection/chart/chart.go +++ b/pkg/collection/chart/chart.go @@ -1,13 +1,14 @@ package chart import ( + "github.com/rancherfederal/ocil/pkg/artifacts" + "github.com/rancherfederal/ocil/pkg/artifacts/image" + "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" - "github.com/rancherfederal/hauler/pkg/artifact" "github.com/rancherfederal/hauler/pkg/content/chart" - "github.com/rancherfederal/hauler/pkg/content/image" ) -var _ artifact.Collection = (*tchart)(nil) +var _ artifacts.OCICollection = (*tchart)(nil) // tchart is a thick chart that includes all the dependent images as well as the chart itself type tchart struct { @@ -15,10 +16,10 @@ type tchart struct { config v1alpha1.ThickChart computed bool - contents map[string]artifact.OCI + contents map[string]artifacts.OCI } -func NewThickChart(cfg v1alpha1.ThickChart) (artifact.Collection, error) { +func NewThickChart(cfg v1alpha1.ThickChart) (artifacts.OCICollection, error) { o, err := chart.NewChart(cfg.Chart) if err != nil { return nil, err @@ -27,11 +28,11 @@ func NewThickChart(cfg v1alpha1.ThickChart) (artifact.Collection, error) { return &tchart{ chart: o, config: cfg, - contents: make(map[string]artifact.OCI), + contents: make(map[string]artifacts.OCI), }, nil } -func (c *tchart) Contents() (map[string]artifact.OCI, error) { +func (c *tchart) Contents() (map[string]artifacts.OCI, error) { if err := c.compute(); err != nil { return nil, err } diff --git a/pkg/collection/k3s/k3s.go b/pkg/collection/k3s/k3s.go index fbe648be..b1cef80a 100644 --- a/pkg/collection/k3s/k3s.go +++ b/pkg/collection/k3s/k3s.go @@ -10,13 +10,15 @@ import ( "path" "strings" - "github.com/rancherfederal/hauler/internal/getter" - "github.com/rancherfederal/hauler/pkg/artifact" - "github.com/rancherfederal/hauler/pkg/content/file" - "github.com/rancherfederal/hauler/pkg/content/image" + "github.com/rancherfederal/ocil/pkg/artifacts" + "github.com/rancherfederal/ocil/pkg/artifacts/image" + + "github.com/rancherfederal/ocil/pkg/artifacts/file" + + "github.com/rancherfederal/ocil/pkg/artifacts/file/getter" ) -var _ artifact.Collection = (*k3s)(nil) +var _ artifacts.OCICollection = (*k3s)(nil) const ( releaseUrl = "https://github.com/k3s-io/k3s/releases/download" @@ -36,19 +38,19 @@ type k3s struct { arch string computed bool - contents map[string]artifact.OCI + contents map[string]artifacts.OCI channels map[string]string client *getter.Client } -func NewK3s(version string) (artifact.Collection, error) { +func NewK3s(version string) (artifacts.OCICollection, error) { return &k3s{ version: version, - contents: make(map[string]artifact.OCI), + contents: make(map[string]artifacts.OCI), }, nil } -func (k *k3s) Contents() (map[string]artifact.OCI, error) { +func (k *k3s) Contents() (map[string]artifacts.OCI, error) { if err := k.compute(); err != nil { return nil, err } diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go deleted file mode 100644 index b92d7906..00000000 --- a/pkg/consts/consts.go +++ /dev/null @@ -1,43 +0,0 @@ -package consts - -const ( - OCIManifestSchema1 = "application/vnd.oci.image.manifest.v1+json" - DockerManifestSchema2 = "application/vnd.docker.distribution.manifest.v2+json" - - DockerConfigJSON = "application/vnd.docker.container.image.v1+json" - DockerLayer = "application/vnd.docker.image.rootfs.diff.tar.gzip" - DockerForeignLayer = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip" - DockerUncompressedLayer = "application/vnd.docker.image.rootfs.diff.tar" - OCILayer = "application/vnd.oci.image.layer.v1.tar+gzip" - - // ChartConfigMediaType is the reserved media type for the Helm chart manifest config - ChartConfigMediaType = "application/vnd.cncf.helm.config.v1+json" - - // ChartLayerMediaType is the reserved media type for Helm chart package content - ChartLayerMediaType = "application/vnd.cncf.helm.chart.content.v1.tar+gzip" - - // ProvLayerMediaType is the reserved media type for Helm chart provenance files - ProvLayerMediaType = "application/vnd.cncf.helm.chart.provenance.v1.prov" - - // FileLayerMediaType is the reserved media type for File content layers - FileLayerMediaType = "application/vnd.content.hauler.file.layer.v1" - - // FileLocalConfigMediaType is the reserved media type for File config - FileLocalConfigMediaType = "application/vnd.content.hauler.file.local.config.v1+json" - FileDirectoryConfigMediaType = "application/vnd.content.hauler.file.directory.config.v1+json" - FileHttpConfigMediaType = "application/vnd.content.hauler.file.http.config.v1+json" - - // WasmArtifactLayerMediaType is the reserved media type for WASM artifact layers - WasmArtifactLayerMediaType = "application/vnd.wasm.content.layer.v1+wasm" - - // WasmConfigMediaType is the reserved media type for WASM configs - WasmConfigMediaType = "application/vnd.wasm.config.v1+json" - - UnknownManifest = "application/vnd.hauler.cattle.io.unknown.v1+json" - UnknownLayer = "application/vnd.content.hauler.unknown.layer" - - OCIVendorPrefix = "vnd.oci" - DockerVendorPrefix = "vnd.docker" - HaulerVendorPrefix = "vnd.hauler" - OCIImageIndexFile = "index.json" -) diff --git a/pkg/content/chart/chart.go b/pkg/content/chart/chart.go index 023fbd79..a8adb827 100644 --- a/pkg/content/chart/chart.go +++ b/pkg/content/chart/chart.go @@ -14,18 +14,20 @@ import ( "github.com/google/go-containerregistry/pkg/v1/partial" gtypes "github.com/google/go-containerregistry/pkg/v1/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/rancherfederal/ocil/pkg/artifacts" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/cli" - "github.com/rancherfederal/hauler/internal/layer" + "github.com/rancherfederal/ocil/pkg/layer" + + "github.com/rancherfederal/ocil/pkg/consts" + "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" - "github.com/rancherfederal/hauler/pkg/artifact" - "github.com/rancherfederal/hauler/pkg/consts" ) -var _ artifact.OCI = (*Chart)(nil) +var _ artifacts.OCI = (*Chart)(nil) // Chart implements the OCI interface for Chart API objects. API spec values are // stored into the Repo, Name, and Version fields. diff --git a/pkg/content/chart/chart_test.go b/pkg/content/chart/chart_test.go index 03370e30..caa7445e 100644 --- a/pkg/content/chart/chart_test.go +++ b/pkg/content/chart/chart_test.go @@ -10,7 +10,8 @@ import ( "github.com/mholt/archiver/v3" ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/rancherfederal/hauler/pkg/consts" + "github.com/rancherfederal/ocil/pkg/consts" + "github.com/rancherfederal/hauler/pkg/content/chart" ) diff --git a/pkg/content/file/file.go b/pkg/content/file/file.go deleted file mode 100644 index 373ba25e..00000000 --- a/pkg/content/file/file.go +++ /dev/null @@ -1,116 +0,0 @@ -package file - -import ( - "context" - - gv1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/partial" - gtypes "github.com/google/go-containerregistry/pkg/v1/types" - - "github.com/rancherfederal/hauler/internal/getter" - "github.com/rancherfederal/hauler/pkg/artifact" - "github.com/rancherfederal/hauler/pkg/consts" -) - -// interface guard -var _ artifact.OCI = (*File)(nil) - -// File implements the OCI interface for File API objects. API spec information is -// stored into the Path field. -type File struct { - Path string - - client *getter.Client - - computed bool - config artifact.Config - blob gv1.Layer - manifest *gv1.Manifest - annotations map[string]string -} - -func NewFile(path string, opts ...Option) *File { - client := getter.NewClient(getter.ClientOptions{}) - - f := &File{ - client: client, - Path: path, - } - - for _, opt := range opts { - opt(f) - } - return f -} - -func (f *File) Name(path string) string { - return f.client.Name(path) -} - -func (f *File) MediaType() string { - return consts.OCIManifestSchema1 -} - -func (f *File) RawConfig() ([]byte, error) { - if err := f.compute(); err != nil { - return nil, err - } - return f.config.Raw() -} - -func (f *File) Layers() ([]gv1.Layer, error) { - if err := f.compute(); err != nil { - return nil, err - } - var layers []gv1.Layer - layers = append(layers, f.blob) - return layers, nil -} - -func (f *File) Manifest() (*gv1.Manifest, error) { - if err := f.compute(); err != nil { - return nil, err - } - return f.manifest, nil -} - -func (f *File) compute() error { - if f.computed { - return nil - } - - ctx := context.Background() - blob, err := f.client.LayerFrom(ctx, f.Path) - if err != nil { - return err - } - - layer, err := partial.Descriptor(blob) - if err != nil { - return err - } - - cfg := f.client.Config(f.Path) - if cfg == nil { - cfg = f.client.Config(f.Path) - } - - cfgDesc, err := partial.Descriptor(cfg) - if err != nil { - return err - } - - m := &gv1.Manifest{ - SchemaVersion: 2, - MediaType: gtypes.MediaType(f.MediaType()), - Config: *cfgDesc, - Layers: []gv1.Descriptor{*layer}, - Annotations: f.annotations, - } - - f.manifest = m - f.config = cfg - f.blob = blob - f.computed = true - return nil -} diff --git a/pkg/content/file/file_test.go b/pkg/content/file/file_test.go deleted file mode 100644 index abfa5aae..00000000 --- a/pkg/content/file/file_test.go +++ /dev/null @@ -1,166 +0,0 @@ -package file_test - -import ( - "bytes" - "context" - "io" - "net/http" - "net/http/httptest" - "net/url" - "os" - "path/filepath" - "testing" - - "github.com/spf13/afero" - - "github.com/rancherfederal/hauler/internal/getter" - "github.com/rancherfederal/hauler/pkg/consts" - "github.com/rancherfederal/hauler/pkg/content/file" -) - -var ( - filename = "myfile.yaml" - data = []byte(`data`) - - ts *httptest.Server - tfs afero.Fs - mc *getter.Client -) - -func TestMain(m *testing.M) { - teardown := setup() - defer teardown() - code := m.Run() - os.Exit(code) -} - -func Test_file_Config(t *testing.T) { - tests := []struct { - name string - ref string - want string - wantErr bool - }{ - { - name: "should properly type local file", - ref: filename, - want: consts.FileLocalConfigMediaType, - wantErr: false, - }, - { - name: "should properly type remote file", - ref: ts.URL + "/" + filename, - want: consts.FileHttpConfigMediaType, - wantErr: false, - }, - // TODO: Add directory test - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - f := file.NewFile(tt.ref, file.WithClient(mc)) - - f.MediaType() - - m, err := f.Manifest() - if err != nil { - t.Fatal(err) - } - - got := string(m.Config.MediaType) - if got != tt.want { - t.Errorf("unxpected mediatype; got %s, want %s", got, tt.want) - } - }) - } -} - -func Test_file_Layers(t *testing.T) { - tests := []struct { - name string - ref string - want []byte - wantErr bool - }{ - { - name: "should load a local file and preserve contents", - ref: filename, - want: data, - wantErr: false, - }, - { - name: "should load a remote file and preserve contents", - ref: ts.URL + "/" + filename, - want: data, - wantErr: false, - }, - // TODO: Add directory test - } - for _, tt := range tests { - t.Run(tt.name, func(it *testing.T) { - f := file.NewFile(tt.ref, file.WithClient(mc)) - - layers, err := f.Layers() - if (err != nil) != tt.wantErr { - it.Fatalf("unexpected Layers() error: got %v, want %v", err, tt.wantErr) - } - - rc, err := layers[0].Compressed() - if err != nil { - it.Fatal(err) - } - - got, err := io.ReadAll(rc) - if err != nil { - it.Fatal(err) - } - - if !bytes.Equal(got, tt.want) { - it.Fatalf("unexpected Layers(): got %v, want %v", layers, tt.want) - } - }) - } -} - -func setup() func() { - tfs = afero.NewMemMapFs() - afero.WriteFile(tfs, filename, data, 0644) - - mf := &mockFile{File: getter.NewFile(), fs: tfs} - - mockHttp := getter.NewHttp() - mhttp := afero.NewHttpFs(tfs) - fileserver := http.FileServer(mhttp.Dir(".")) - http.Handle("/", fileserver) - ts = httptest.NewServer(fileserver) - - mc = &getter.Client{ - Options: getter.ClientOptions{}, - Getters: map[string]getter.Getter{ - "file": mf, - "http": mockHttp, - }, - } - - teardown := func() { - defer ts.Close() - } - - return teardown -} - -type mockFile struct { - *getter.File - fs afero.Fs -} - -func (m mockFile) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) { - return m.fs.Open(filepath.Join(u.Host, u.Path)) -} - -func (m mockFile) Detect(u *url.URL) bool { - fi, err := m.fs.Stat(filepath.Join(u.Host, u.Path)) - if err != nil { - return false - } - return !fi.IsDir() -} diff --git a/pkg/content/file/options.go b/pkg/content/file/options.go deleted file mode 100644 index c0365e02..00000000 --- a/pkg/content/file/options.go +++ /dev/null @@ -1,26 +0,0 @@ -package file - -import ( - "github.com/rancherfederal/hauler/internal/getter" - "github.com/rancherfederal/hauler/pkg/artifact" -) - -type Option func(*File) - -func WithClient(c *getter.Client) Option { - return func(f *File) { - f.client = c - } -} - -func WithConfig(obj interface{}, mediaType string) Option { - return func(f *File) { - f.config = artifact.ToConfig(obj, artifact.WithConfigMediaType(mediaType)) - } -} - -func WithAnnotations(m map[string]string) Option { - return func(f *File) { - f.annotations = m - } -} diff --git a/pkg/content/image/image.go b/pkg/content/image/image.go deleted file mode 100644 index fd37cc91..00000000 --- a/pkg/content/image/image.go +++ /dev/null @@ -1,53 +0,0 @@ -package image - -import ( - "github.com/google/go-containerregistry/pkg/authn" - gname "github.com/google/go-containerregistry/pkg/name" - gv1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/remote" - - "github.com/rancherfederal/hauler/pkg/artifact" -) - -var _ artifact.OCI = (*Image)(nil) - -func (i *Image) MediaType() string { - mt, err := i.Image.MediaType() - if err != nil { - return "" - } - return string(mt) -} - -func (i *Image) RawConfig() ([]byte, error) { - return i.RawConfigFile() -} - -// Image implements the OCI interface for Image API objects. API spec information -// is stored into the Name field. -type Image struct { - Name string - gv1.Image -} - -func NewImage(name string, opts ...remote.Option) (*Image, error) { - r, err := gname.ParseReference(name) - if err != nil { - return nil, err - } - - defaultOpts := []remote.Option{ - remote.WithAuthFromKeychain(authn.DefaultKeychain), - } - opts = append(opts, defaultOpts...) - - img, err := remote.Image(r, opts...) - if err != nil { - return nil, err - } - - return &Image{ - Name: name, - Image: img, - }, nil -} diff --git a/pkg/content/image/image_test.go b/pkg/content/image/image_test.go deleted file mode 100644 index aa66c334..00000000 --- a/pkg/content/image/image_test.go +++ /dev/null @@ -1 +0,0 @@ -package image_test diff --git a/pkg/reference/reference.go b/pkg/reference/reference.go index 578f281c..7ebbf564 100644 --- a/pkg/reference/reference.go +++ b/pkg/reference/reference.go @@ -47,3 +47,21 @@ func Parse(ref string) (gname.Reference, error) { return r, nil } + +// Relocate returns a name.Reference given a reference and registry +func Relocate(reference string, registry string) (gname.Reference, error) { + ref, err := gname.ParseReference(reference) + if err != nil { + return nil, err + } + + relocated, err := gname.ParseReference(ref.Context().RepositoryStr(), gname.WithDefaultRegistry(registry)) + if err != nil { + return nil, err + } + + if _, err := gname.NewDigest(ref.Name()); err == nil { + return relocated.Context().Digest(ref.Identifier()), nil + } + return relocated.Context().Tag(ref.Identifier()), nil +} diff --git a/pkg/store/errors.go b/pkg/store/errors.go deleted file mode 100644 index ffcba827..00000000 --- a/pkg/store/errors.go +++ /dev/null @@ -1,7 +0,0 @@ -package store - -import "github.com/pkg/errors" - -var ( - ErrInvalidReference = errors.New("invalid reference") -) diff --git a/pkg/store/oci.go b/pkg/store/oci.go deleted file mode 100644 index e1efaedc..00000000 --- a/pkg/store/oci.go +++ /dev/null @@ -1,236 +0,0 @@ -package store - -import ( - "context" - "encoding/json" - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" - "sync" - - ccontent "github.com/containerd/containerd/content" - "github.com/containerd/containerd/remotes" - "github.com/opencontainers/image-spec/specs-go" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "oras.land/oras-go/pkg/content" - "oras.land/oras-go/pkg/target" - - "github.com/rancherfederal/hauler/pkg/consts" -) - -var _ target.Target = (*OCI)(nil) - -type OCI struct { - root string - index *ocispec.Index - nameMap *sync.Map // map[string]ocispec.Descriptor -} - -func NewOCI(root string) (*OCI, error) { - o := &OCI{ - root: root, - nameMap: &sync.Map{}, - } - return o, nil -} - -func (o *OCI) LoadIndex() error { - path := o.path(consts.OCIImageIndexFile) - idx, err := os.Open(path) - if err != nil { - if !os.IsNotExist(err) { - return err - } - o.index = &ocispec.Index{ - Versioned: specs.Versioned{ - SchemaVersion: 2, - }, - } - return nil - } - defer idx.Close() - - if err := json.NewDecoder(idx).Decode(&o.index); err != nil { - return err - } - - for _, desc := range o.index.Manifests { - if name := desc.Annotations[ocispec.AnnotationRefName]; name != "" { - o.nameMap.Store(name, desc) - } - } - return nil -} - -func (o *OCI) SaveIndex() error { - var descs []ocispec.Descriptor - o.nameMap.Range(func(name, desc interface{}) bool { - n := name.(string) - d := desc.(ocispec.Descriptor) - - if d.Annotations == nil { - d.Annotations = make(map[string]string) - } - d.Annotations[ocispec.AnnotationRefName] = n - descs = append(descs, d) - return true - }) - o.index.Manifests = descs - data, err := json.Marshal(o.index) - if err != nil { - return err - } - return os.WriteFile(o.path(consts.OCIImageIndexFile), data, 0644) -} - -// Resolve attempts to resolve the reference into a name and descriptor. -// -// The argument `ref` should be a scheme-less URI representing the remote. -// Structurally, it has a host and path. The "host" can be used to directly -// reference a specific host or be matched against a specific handler. -// -// The returned name should be used to identify the referenced entity. -// Dependending on the remote namespace, this may be immutable or mutable. -// While the name may differ from ref, it should itself be a valid ref. -// -// If the resolution fails, an error will be returned. -func (o *OCI) Resolve(ctx context.Context, ref string) (name string, desc ocispec.Descriptor, err error) { - if err := o.LoadIndex(); err != nil { - return "", ocispec.Descriptor{}, err - } - d, ok := o.nameMap.Load(ref) - if !ok { - return "", ocispec.Descriptor{}, err - } - desc = d.(ocispec.Descriptor) - return ref, desc, nil -} - -// Fetcher returns a new fetcher for the provided reference. -// All content fetched from the returned fetcher will be -// from the namespace referred to by ref. -func (o *OCI) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, error) { - if err := o.LoadIndex(); err != nil { - return nil, err - } - if _, ok := o.nameMap.Load(ref); !ok { - return nil, nil - } - return o, nil -} - -func (o *OCI) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) { - readerAt, err := o.blobReaderAt(desc) - if err != nil { - return nil, err - } - return readerAt, nil -} - -// Pusher returns a new pusher for the provided reference -// The returned Pusher should satisfy content.Ingester and concurrent attempts -// to push the same blob using the Ingester API should result in ErrUnavailable. -func (o *OCI) Pusher(ctx context.Context, ref string) (remotes.Pusher, error) { - if err := o.LoadIndex(); err != nil { - return nil, err - } - - var baseRef, hash string - parts := strings.SplitN(ref, "@", 2) - baseRef = parts[0] - if len(parts) > 1 { - hash = parts[1] - } - return &ociPusher{ - oci: o, - ref: baseRef, - digest: hash, - }, nil -} - -func (o *OCI) Walk(fn func(reference string, desc ocispec.Descriptor) error) error { - if err := o.LoadIndex(); err != nil { - return err - } - o.nameMap.Range(func(key, value interface{}) bool { - if err := fn(key.(string), value.(ocispec.Descriptor)); err != nil { - return false - } - - return true - }) - return nil -} - -func (o *OCI) blobReaderAt(desc ocispec.Descriptor) (*os.File, error) { - blobPath, err := o.ensureBlob(desc) - if err != nil { - return nil, err - } - return os.Open(blobPath) -} - -func (o *OCI) blobWriterAt(desc ocispec.Descriptor) (*os.File, error) { - blobPath, err := o.ensureBlob(desc) - if err != nil { - return nil, err - } - return os.OpenFile(blobPath, os.O_WRONLY|os.O_CREATE, 0644) -} - -func (o *OCI) ensureBlob(desc ocispec.Descriptor) (string, error) { - dir := o.path("blobs", desc.Digest.Algorithm().String()) - if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) { - return "", err - } - return filepath.Join(dir, desc.Digest.Hex()), nil -} - -func (o *OCI) path(elem ...string) string { - complete := []string{string(o.root)} - return filepath.Join(append(complete, elem...)...) -} - -type ociPusher struct { - oci *OCI - ref string - digest string -} - -// Push returns a content writer for the given resource identified -// by the descriptor. -func (p *ociPusher) Push(ctx context.Context, d ocispec.Descriptor) (ccontent.Writer, error) { - switch d.MediaType { - case ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex, consts.DockerManifestSchema2: - // if the hash of the content matches that which was provided as the hash for the root, mark it - if p.digest != "" && p.digest == d.Digest.String() { - if err := p.oci.LoadIndex(); err != nil { - return nil, err - } - p.oci.nameMap.Store(p.ref, d) - if err := p.oci.SaveIndex(); err != nil { - return nil, err - } - } - } - - blobPath, err := p.oci.ensureBlob(d) - if err != nil { - return nil, err - } - - if _, err := os.Stat(blobPath); err == nil { - // file already exists, discard (but validate digest) - return content.NewIoContentWriter(ioutil.Discard, content.WithOutputHash(d.Digest)), nil - } - - f, err := os.Create(blobPath) - if err != nil { - return nil, err - } - - w := content.NewIoContentWriter(f, content.WithInputHash(d.Digest), content.WithOutputHash(d.Digest)) - return w, nil -} diff --git a/pkg/store/options.go b/pkg/store/options.go deleted file mode 100644 index f1144036..00000000 --- a/pkg/store/options.go +++ /dev/null @@ -1,13 +0,0 @@ -package store - -import ( - "github.com/rancherfederal/hauler/internal/cache" -) - -type Options func(*Store) - -func WithCache(c cache.Cache) Options { - return func(b *Store) { - b.cache = c - } -} diff --git a/pkg/store/stager.go b/pkg/store/stager.go deleted file mode 100644 index 049d4d8c..00000000 --- a/pkg/store/stager.go +++ /dev/null @@ -1,192 +0,0 @@ -package store - -import ( - "context" - "encoding/json" - "io" - "os" - "path/filepath" - - "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/static" - "github.com/opencontainers/go-digest" - "github.com/opencontainers/image-spec/specs-go" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" - "golang.org/x/sync/errgroup" - "oras.land/oras-go/pkg/oras" - - "github.com/rancherfederal/hauler/pkg/artifact" - "github.com/rancherfederal/hauler/pkg/consts" -) - -type stager interface { - add(ctx context.Context, oci artifact.OCI, ref name.Reference) error - - commit(ctx context.Context, b *Store) (ocispec.Descriptor, error) - - flush(ctx context.Context) error -} - -var _ stager = (*layout)(nil) - -func newLayout() (*layout, error) { - tmpdir, err := os.MkdirTemp("", "hauler") - if err != nil { - return nil, err - } - return &layout{ - root: tmpdir, - blobs: make(map[digest.Digest]v1.Layer), - }, nil -} - -type layout struct { - root string - - descs []ocispec.Descriptor - blobs map[digest.Digest]v1.Layer -} - -func (l *layout) commit(ctx context.Context, b *Store) (ocispec.Descriptor, error) { - defer l.flush(ctx) - - var g errgroup.Group - for d, blob := range l.blobs { - blob := blob - d := d - g.Go(func() error { - rc, err := blob.Compressed() - if err != nil { - return err - } - return l.writeBlob(d, rc) - }) - } - if err := g.Wait(); err != nil { - return ocispec.Descriptor{}, err - } - - idx := ocispec.Index{ - Versioned: specs.Versioned{SchemaVersion: 2}, - Manifests: l.descs, - Annotations: nil, - } - - data, err := json.Marshal(idx) - if err != nil { - return ocispec.Descriptor{}, err - } - if err := os.WriteFile(l.path("index.json"), data, 0666); err != nil { - return ocispec.Descriptor{}, err - } - - src, err := NewOCI(l.path("")) - if err != nil { - return ocispec.Descriptor{}, err - } - - dst, err := NewOCI(b.Root) - if err != nil { - return ocispec.Descriptor{}, err - } - - if err := src.LoadIndex(); err != nil { - return ocispec.Descriptor{}, err - } - - m := src.index.Manifests[0] - ref := m.Annotations[ocispec.AnnotationRefName] - - desc, err := oras.Copy(ctx, src, ref, dst, "", - oras.WithAdditionalCachedMediaTypes(consts.DockerManifestSchema2)) - if err != nil { - return ocispec.Descriptor{}, errors.Wrap(err, "comitting staging layout") - } - - if err := dst.SaveIndex(); err != nil { - return ocispec.Descriptor{}, err - } - - return desc, err -} - -func (l *layout) flush(ctx context.Context) error { - return os.RemoveAll(l.path("")) -} - -func (l *layout) add(ctx context.Context, oci artifact.OCI, ref name.Reference) error { - m, err := oci.Manifest() - if err != nil { - return err - } - mdata, err := json.Marshal(m) - if err != nil { - return err - } - mdigest := digest.FromBytes(mdata) - l.blobs[mdigest] = static.NewLayer(mdata, m.MediaType) - - cdata, err := oci.RawConfig() - if err != nil { - return err - } - - l.blobs[digest.FromBytes(cdata)] = static.NewLayer(cdata, "") - - layers, err := oci.Layers() - if err != nil { - return err - } - - for _, layer := range layers { - h, err := layer.Digest() - if err != nil { - return err - } - - d := digest.NewDigestFromHex(h.Algorithm, h.Hex) - l.blobs[d] = layer - } - - mdesc := ocispec.Descriptor{ - MediaType: oci.MediaType(), - Digest: mdigest, - Size: int64(len(mdata)), - Annotations: map[string]string{ - ocispec.AnnotationRefName: ref.Name(), - }, - - URLs: nil, - Platform: nil, - } - l.descs = append(l.descs, mdesc) - return nil -} - -func (l *layout) writeBlob(d digest.Digest, rc io.ReadCloser) error { - dir := l.path("blobs", d.Algorithm().String()) - if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) { - return err - } - - file := filepath.Join(dir, d.Hex()) - if _, err := os.Stat(file); err == nil { - return err - } - - w, err := os.Create(file) - if err != nil { - return err - } - defer w.Close() - - _, err = io.Copy(w, rc) - return err -} - -func (l *layout) path(elem ...string) string { - complete := []string{string(l.root)} - return filepath.Join(append(complete, elem...)...) -} diff --git a/pkg/store/store.go b/pkg/store/store.go deleted file mode 100644 index ea191612..00000000 --- a/pkg/store/store.go +++ /dev/null @@ -1,184 +0,0 @@ -package store - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - - "github.com/google/go-containerregistry/pkg/name" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "oras.land/oras-go/pkg/oras" - "oras.land/oras-go/pkg/target" - - "github.com/rancherfederal/hauler/internal/cache" - "github.com/rancherfederal/hauler/pkg/artifact" - "github.com/rancherfederal/hauler/pkg/consts" - "github.com/rancherfederal/hauler/pkg/reference" -) - -type Store struct { - Root string - - Content *OCI - cache cache.Cache -} - -func NewStore(rootdir string, opts ...Options) (*Store, error) { - ociStore, err := NewOCI(rootdir) - if err != nil { - return nil, err - } - - b := &Store{ - Root: rootdir, - Content: ociStore, - } - - for _, opt := range opts { - opt(b) - } - return b, nil -} - -// AddArtifact will add an artifact.OCI to the store -// The method to achieve this is to save artifact.OCI to a temporary directory in an OCI layout compatible form. Once -// saved, the entirety of the layout is copied to the store (which is just a registry). This allows us to not only use -// strict types to define generic content, but provides a processing pipeline suitable for extensibility. In the -// future we'll allow users to define their own content that must adhere either by artifact.OCI or simply an OCI layout. -func (s *Store) AddArtifact(ctx context.Context, oci artifact.OCI, r string) (ocispec.Descriptor, error) { - stage, err := newLayout() - if err != nil { - return ocispec.Descriptor{}, err - } - - if s.cache != nil { - cached := cache.Oci(oci, s.cache) - oci = cached - } - - // Validate we have a locatable reference - ref, err := reference.Parse(r) - if err != nil { - return ocispec.Descriptor{}, err - } - - if err := stage.add(ctx, oci, ref); err != nil { - return ocispec.Descriptor{}, fmt.Errorf("adding artifact: %w", err) - } - return stage.commit(ctx, s) -} - -// AddCollection . -func (s *Store) AddCollection(ctx context.Context, coll artifact.Collection) ([]ocispec.Descriptor, error) { - cnts, err := coll.Contents() - if err != nil { - return nil, err - } - - var descs []ocispec.Descriptor - for ref, oci := range cnts { - ds, err := s.AddArtifact(ctx, oci, ref) - if err != nil { - return nil, err - } - descs = append(descs, ds) - } - - return descs, nil -} - -// Flush is a fancy name for delete-all-the-things, in this case it's as trivial as deleting oci-layout content -// This can be a highly destructive operation if the store's directory happens to be inline with other non-store contents -// To reduce the blast radius and likelihood of deleting things we don't own, Flush explicitly deletes oci-layout content only -func (s *Store) Flush(ctx context.Context) error { - blobs := filepath.Join(s.Root, "blobs") - if err := os.RemoveAll(blobs); err != nil { - return err - } - - index := filepath.Join(s.Root, "index.json") - if err := os.RemoveAll(index); err != nil { - return err - } - - layout := filepath.Join(s.Root, "oci-layout") - if err := os.RemoveAll(layout); err != nil { - return err - } - - return nil -} - -// Copy will copy a given reference to a given target.Target -// This is essentially a wrapper around oras.Copy, but locked to this content store -func (s *Store) Copy(ctx context.Context, ref string, to target.Target, toRef string) (ocispec.Descriptor, error) { - return oras.Copy(ctx, s.Content, ref, to, toRef, - oras.WithAdditionalCachedMediaTypes(consts.DockerManifestSchema2)) -} - -// CopyAll performs bulk copy operations on the stores oci layout to a provided target.Target -func (s *Store) CopyAll(ctx context.Context, to target.Target, toMapper func(string) (string, error)) ([]ocispec.Descriptor, error) { - var descs []ocispec.Descriptor - err := s.Content.Walk(func(reference string, desc ocispec.Descriptor) error { - toRef := "" - if toMapper != nil { - tr, err := toMapper(reference) - if err != nil { - return err - } - toRef = tr - } - - desc, err := s.Copy(ctx, reference, to, toRef) - if err != nil { - return err - } - - descs = append(descs, desc) - return nil - }) - if err != nil { - return nil, err - } - return descs, nil -} - -// Identify is a helper function that will identify a human-readable content type given a descriptor -func (s *Store) Identify(ctx context.Context, desc ocispec.Descriptor) string { - rc, err := s.Content.Fetch(ctx, desc) - if err != nil { - return "" - } - defer rc.Close() - - m := struct { - Config struct { - MediaType string `json:"mediaType"` - } `json:"config"` - }{} - if err := json.NewDecoder(rc).Decode(&m); err != nil { - return "" - } - - return m.Config.MediaType -} - -// RelocateReference returns a name.Reference given a reference and registry -func RelocateReference(reference string, registry string) (name.Reference, error) { - ref, err := name.ParseReference(reference) - if err != nil { - return nil, err - } - - relocated, err := name.ParseReference(ref.Context().RepositoryStr(), name.WithDefaultRegistry(registry)) - if err != nil { - return nil, err - } - - if _, err := name.NewDigest(ref.Name()); err == nil { - return relocated.Context().Digest(ref.Identifier()), nil - } - return relocated.Context().Tag(ref.Identifier()), nil -} diff --git a/pkg/store/store_test.go b/pkg/store/store_test.go deleted file mode 100644 index b498ea4d..00000000 --- a/pkg/store/store_test.go +++ /dev/null @@ -1,213 +0,0 @@ -package store_test - -import ( - "bytes" - "context" - "encoding/json" - "os" - "path/filepath" - "reflect" - "testing" - - "github.com/google/go-containerregistry/pkg/name" - gv1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/partial" - "github.com/google/go-containerregistry/pkg/v1/random" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "oras.land/oras-go/pkg/content" - "oras.land/oras-go/pkg/target" - - _ "github.com/distribution/distribution/v3/registry/storage/driver/filesystem" - _ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory" - - "github.com/rancherfederal/hauler/internal/server" - "github.com/rancherfederal/hauler/pkg/artifact" - "github.com/rancherfederal/hauler/pkg/store" -) - -func TestStore_AddArtifact(t *testing.T) { - ctx := context.Background() - - tmpdir, err := os.MkdirTemp("", "hauler") - if err != nil { - t.Fatal(err) - } - - s, err := store.NewStore(tmpdir) - if err != nil { - t.Fatal(err) - } - - type args struct { - ctx context.Context - reference string - } - tests := []struct { - name string - args args - wantErr bool - }{ - { - name: "should add artifact with a valid tagged reference", - args: args{ - ctx: ctx, - reference: "hauler/random:v1", - }, - wantErr: false, - }, - // { - // name: "should fail when an invalid reference is provided", - // args: args{ - // ctx: ctx, - // reference: "n0tV@l!d:v1", - // }, - // wantErr: true, - // }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - oci, want := genArtifact(t, tt.args.reference) - - got, err := s.AddArtifact(tt.args.ctx, oci, tt.args.reference) - if (err != nil) != tt.wantErr { - t.Errorf("AddArtifact() error = %v, wantErr %v", err, tt.wantErr) - return - } - - if !reflect.DeepEqual(got, want) { - t.Errorf("AddArtifact() got = %v, want %v", got, want) - } - }) - } -} - -type testImage struct { - gv1.Image -} - -func (i *testImage) MediaType() string { - mt, err := i.Image.MediaType() - if err != nil { - return "" - } - return string(mt) -} - -func (i *testImage) RawConfig() ([]byte, error) { - return i.RawConfigFile() -} - -func genArtifact(t *testing.T, ref string) (artifact.OCI, ocispec.Descriptor) { - img, err := random.Image(1024, 3) - if err != nil { - t.Fatal(err) - } - - desc, err := partial.Descriptor(img) - if err != nil { - t.Fatal(err) - } - desc.Annotations = make(map[string]string) - desc.Annotations[ocispec.AnnotationRefName] = ref - - data, err := json.Marshal(desc) - if err != nil { - t.Fatal(err) - } - - var m ocispec.Descriptor - if err := json.NewDecoder(bytes.NewBuffer(data)).Decode(&m); err != nil { - t.Fatal(err) - } - return &testImage{Image: img}, m -} - -func TestStore_CopyAll(t *testing.T) { - ctx := context.Background() - - tmpdir, err := os.MkdirTemp("", "hauler") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpdir) - - tmpdirRegistry := filepath.Join(tmpdir, "registry") - if err := os.Mkdir(tmpdirRegistry, os.ModePerm); err != nil { - t.Fatal() - } - - tmpdirStore := filepath.Join(tmpdir, "store") - if err := os.Mkdir(tmpdirStore, os.ModePerm); err != nil { - t.Fatal() - } - - r := server.NewTempRegistry(ctx, tmpdirRegistry) - if err := r.Start(); err != nil { - t.Error(err) - } - defer r.Stop() - - rc, err := content.NewRegistry(content.RegistryOptions{Insecure: true}) - if err != nil { - t.Fatal(err) - } - - type args struct { - ctx context.Context - to target.Target - toMapper func(string) (string, error) - refs []string - } - tests := []struct { - name string - args args - wantErr bool - }{ - { - name: "should copy an artifact to a registry", - args: args{ - ctx: ctx, - to: rc, - toMapper: nil, - refs: []string{"tester:tester"}, - }, - wantErr: false, - }, - { - name: "should copy a lot of artifacts to a registry (test concurrency)", - args: args{ - ctx: ctx, - to: rc, - toMapper: nil, - refs: []string{"a/b:c", "a/c:b", "b/c:d", "z/y:w", "u/q:a", "y/y:y", "z/z:z", "b/b:b", "c/c:c"}, - }, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s, err := store.NewStore(tmpdirStore) - if err != nil { - t.Fatal(err) - } - - for _, ref := range tt.args.refs { - locRef, _ := name.ParseReference(ref, name.WithDefaultRegistry(r.Registry())) - a, _ := genArtifact(t, locRef.Name()) - if _, err := s.AddArtifact(ctx, a, locRef.Name()); err != nil { - t.Errorf("Failed to generate store contents for CopyAll(): %v", err) - } - } - - if descs, err := s.CopyAll(tt.args.ctx, tt.args.to, tt.args.toMapper); (err != nil) != tt.wantErr { - t.Errorf("CopyAll() error = %v, wantErr %v", err, tt.wantErr) - } else if len(descs) != len(tt.args.refs) { - t.Errorf("CopyAll() expected to push %d descriptors, but only pushed %d", len(descs), len(tt.args.refs)) - } - - if err := s.Flush(ctx); err != nil { - t.Fatal(err) - } - }) - } -}