factor out core oci logic into independent library (rancherfederal/ocil)

cmd/hauler/cli/cli.go

@@ -6,11 +6,12 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/rancherfederal/ocil/pkg/layer"
 	"github.com/spf13/cobra"
 
-	cache2 "github.com/rancherfederal/hauler/internal/cache"
+	"github.com/rancherfederal/ocil/pkg/store"
+
 	"github.com/rancherfederal/hauler/pkg/log"
-	"github.com/rancherfederal/hauler/pkg/store"
 )
 
 type rootOpts struct {
@@ -55,7 +56,7 @@ func New() *cobra.Command {
 	return cmd
 }
 
-func (o *rootOpts) getStore(ctx context.Context) (*store.Store, error) {
+func (o *rootOpts) getStore(ctx context.Context) (*store.Layout, error) {
 	l := log.FromContext(ctx)
 	dir := o.storeDir
 
@@ -80,14 +81,14 @@ func (o *rootOpts) getStore(ctx context.Context) (*store.Store, error) {
 		return nil, err
 	}
 
-	s, err := store.NewStore(abs, store.WithCache(c))
+	s, err := store.NewLayout(abs, store.WithCache(c))
 	if err != nil {
 		return nil, err
 	}
 	return s, nil
 }
 
-func (o *rootOpts) getCache(ctx context.Context) (cache2.Cache, error) {
+func (o *rootOpts) getCache(ctx context.Context) (layer.Cache, error) {
 	dir := o.cacheDir
 
 	if dir == "" {
@@ -105,6 +106,6 @@ func (o *rootOpts) getCache(ctx context.Context) (cache2.Cache, error) {
 		dir = abs
 	}
 
-	c := cache2.NewFilesystem(dir)
+	c := layer.NewFilesystemCache(dir)
 	return c, nil
 }

cmd/hauler/cli/download/download.go

@@ -11,8 +11,9 @@ import (
 	"oras.land/oras-go/pkg/content"
 	"oras.land/oras-go/pkg/oras"
 
+	"github.com/rancherfederal/ocil/pkg/consts"
+
 	"github.com/rancherfederal/hauler/internal/mapper"
-	"github.com/rancherfederal/hauler/pkg/consts"
 	"github.com/rancherfederal/hauler/pkg/log"
 	"github.com/rancherfederal/hauler/pkg/reference"
 )

cmd/hauler/cli/store/add.go

@@ -7,13 +7,15 @@ import (
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/spf13/cobra"
 
+	"github.com/rancherfederal/ocil/pkg/artifacts/file"
+	"github.com/rancherfederal/ocil/pkg/artifacts/image"
+
+	"github.com/rancherfederal/ocil/pkg/store"
+
 	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
 	"github.com/rancherfederal/hauler/pkg/content/chart"
-	"github.com/rancherfederal/hauler/pkg/content/file"
-	"github.com/rancherfederal/hauler/pkg/content/image"
 	"github.com/rancherfederal/hauler/pkg/log"
 	"github.com/rancherfederal/hauler/pkg/reference"
-	"github.com/rancherfederal/hauler/pkg/store"
 )
 
 type AddFileOpts struct {
@@ -25,15 +27,15 @@ func (o *AddFileOpts) AddFlags(cmd *cobra.Command) {
 	f.StringVarP(&o.Name, "name", "n", "", "(Optional) Name to assign to file in store")
 }
 
-func AddFileCmd(ctx context.Context, o *AddFileOpts, s *store.Store, reference string) error {
+func AddFileCmd(ctx context.Context, o *AddFileOpts, s *store.Layout, reference string) error {
 	cfg := v1alpha1.File{
 		Path: reference,
 	}
 
 	return storeFile(ctx, s, cfg)
 }
 
-func storeFile(ctx context.Context, s *store.Store, fi v1alpha1.File) error {
+func storeFile(ctx context.Context, s *store.Layout, fi v1alpha1.File) error {
 	l := log.FromContext(ctx)
 
 	f := file.NewFile(fi.Path)
@@ -42,7 +44,7 @@ func storeFile(ctx context.Context, s *store.Store, fi v1alpha1.File) error {
 		return err
 	}
 
-	desc, err := s.AddArtifact(ctx, f, ref.Name())
+	desc, err := s.AddOCI(ctx, f, ref.Name())
 	if err != nil {
 		return err
 	}
@@ -60,15 +62,15 @@ func (o *AddImageOpts) AddFlags(cmd *cobra.Command) {
 	_ = f
 }
 
-func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Store, reference string) error {
+func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Layout, reference string) error {
 	cfg := v1alpha1.Image{
 		Name: reference,
 	}
 
 	return storeImage(ctx, s, cfg)
 }
 
-func storeImage(ctx context.Context, s *store.Store, i v1alpha1.Image) error {
+func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image) error {
 	l := log.FromContext(ctx)
 
 	oci, err := image.NewImage(i.Name)
@@ -81,7 +83,7 @@ func storeImage(ctx context.Context, s *store.Store, i v1alpha1.Image) error {
 		return err
 	}
 
-	desc, err := s.AddArtifact(ctx, oci, r.Name())
+	desc, err := s.AddOCI(ctx, oci, r.Name())
 	if err != nil {
 		return err
 	}
@@ -104,7 +106,7 @@ func (o *AddChartOpts) AddFlags(cmd *cobra.Command) {
 	f.StringVar(&o.Version, "version", "", "(Optional) Version of the chart to download, defaults to latest if not specified")
 }
 
-func AddChartCmd(ctx context.Context, o *AddChartOpts, s *store.Store, chartName string) error {
+func AddChartCmd(ctx context.Context, o *AddChartOpts, s *store.Layout, chartName string) error {
 	path := ""
 	if _, err := os.Stat(chartName); err == nil {
 		path = chartName
@@ -119,7 +121,7 @@ func AddChartCmd(ctx context.Context, o *AddChartOpts, s *store.Store, chartName
 	return storeChart(ctx, s, cfg)
 }
 
-func storeChart(ctx context.Context, s *store.Store, cfg v1alpha1.Chart) error {
+func storeChart(ctx context.Context, s *store.Layout, cfg v1alpha1.Chart) error {
 	l := log.FromContext(ctx)
 
 	oci, err := chart.NewChart(cfg)
@@ -131,7 +133,7 @@ func storeChart(ctx context.Context, s *store.Store, cfg v1alpha1.Chart) error {
 	if err != nil {
 		return err
 	}
-	desc, err := s.AddArtifact(ctx, oci, ref.Name())
+	desc, err := s.AddOCI(ctx, oci, ref.Name())
 	if err != nil {
 		return err
 	}

cmd/hauler/cli/store/copy.go

@@ -9,8 +9,10 @@ import (
 	"github.com/spf13/cobra"
 	"oras.land/oras-go/pkg/content"
 
+	"github.com/rancherfederal/ocil/pkg/store"
+
 	"github.com/rancherfederal/hauler/pkg/log"
-	"github.com/rancherfederal/hauler/pkg/store"
+	"github.com/rancherfederal/hauler/pkg/reference"
 )
 
 type CopyOpts struct {
@@ -31,7 +33,7 @@ func (o *CopyOpts) AddFlags(cmd *cobra.Command) {
 	f.BoolVar(&o.PlainHTTP, "plain-http", false, "Toggle allowing plain http connections when copying to a remote registry")
 }
 
-func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Store, targetRef string) error {
+func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string) error {
 	l := log.FromContext(ctx)
 
 	var descs []ocispec.Descriptor
@@ -60,12 +62,12 @@ func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Store, targetRef string)
 			return err
 		}
 
-		mapperFn := func(reference string) (string, error) {
-			ref, err := store.RelocateReference(reference, components[1])
+		mapperFn := func(ref string) (string, error) {
+			r, err := reference.Relocate(ref, components[1])
 			if err != nil {
 				return "", err
 			}
-			return ref.Name(), nil
+			return r.Name(), nil
 		}
 
 		ds, err := s.CopyAll(ctx, r, mapperFn)

cmd/hauler/cli/store/extract.go

@@ -8,10 +8,11 @@ import (
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/spf13/cobra"
 
+	"github.com/rancherfederal/ocil/pkg/store"
+
 	"github.com/rancherfederal/hauler/internal/mapper"
 	"github.com/rancherfederal/hauler/pkg/log"
 	"github.com/rancherfederal/hauler/pkg/reference"
-	"github.com/rancherfederal/hauler/pkg/store"
 )
 
 type ExtractOpts struct {
@@ -24,7 +25,7 @@ func (o *ExtractOpts) AddArgs(cmd *cobra.Command) {
 	f.StringVarP(&o.DestinationDir, "output", "o", "", "Directory to save contents to (defaults to current directory)")
 }
 
-func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Store, ref string) error {
+func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Layout, ref string) error {
 	l := log.FromContext(ctx)
 
 	r, err := reference.Parse(ref)
@@ -33,13 +34,13 @@ func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Store, ref string)
 	}
 
 	found := false
-	if err := s.Content.Walk(func(reference string, desc ocispec.Descriptor) error {
+	if err := s.Walk(func(reference string, desc ocispec.Descriptor) error {
 		if reference != r.Name() {
 			return nil
 		}
 		found = true
 
-		rc, err := s.Content.Fetch(ctx, desc)
+		rc, err := s.Fetch(ctx, desc)
 		if err != nil {
 			return err
 		}

cmd/hauler/cli/store/info.go

@@ -10,9 +10,11 @@ import (
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/spf13/cobra"
 
-	"github.com/rancherfederal/hauler/pkg/consts"
+	"github.com/rancherfederal/ocil/pkg/consts"
+
+	"github.com/rancherfederal/ocil/pkg/store"
+
 	"github.com/rancherfederal/hauler/pkg/reference"
-	"github.com/rancherfederal/hauler/pkg/store"
 )
 
 type InfoOpts struct {
@@ -28,14 +30,14 @@ func (o *InfoOpts) AddFlags(cmd *cobra.Command) {
 	// TODO: Regex/globbing
 }
 
-func InfoCmd(ctx context.Context, o *InfoOpts, s *store.Store) error {
+func InfoCmd(ctx context.Context, o *InfoOpts, s *store.Layout) error {
 	var items []item
-	if err := s.Content.Walk(func(ref string, desc ocispec.Descriptor) error {
+	if err := s.Walk(func(ref string, desc ocispec.Descriptor) error {
 		if _, ok := desc.Annotations[ocispec.AnnotationRefName]; !ok {
 			return nil
 		}
 
-		rc, err := s.Content.Fetch(ctx, desc)
+		rc, err := s.Fetch(ctx, desc)
 		if err != nil {
 			return err
 		}
@@ -97,7 +99,7 @@ type item struct {
 	Size      string
 }
 
-func newItem(s *store.Store, desc ocispec.Descriptor, m ocispec.Manifest) item {
+func newItem(s *store.Layout, desc ocispec.Descriptor, m ocispec.Manifest) item {
 	var size int64 = 0
 	for _, l := range m.Layers {
 		size = +l.Size

cmd/hauler/cli/store/serve.go

@@ -14,8 +14,9 @@ import (
 	"github.com/distribution/distribution/v3/version"
 	"github.com/spf13/cobra"
 
+	"github.com/rancherfederal/ocil/pkg/store"
+
 	"github.com/rancherfederal/hauler/internal/server"
-	"github.com/rancherfederal/hauler/pkg/store"
 )
 
 type ServeOpts struct {
@@ -37,7 +38,7 @@ func (o *ServeOpts) AddFlags(cmd *cobra.Command) {
 }
 
 // ServeCmd serves the embedded registry almost identically to how distribution/v3 does it
-func ServeCmd(ctx context.Context, o *ServeOpts, s *store.Store) error {
+func ServeCmd(ctx context.Context, o *ServeOpts, s *store.Layout) error {
 	ctx = dcontext.WithVersion(ctx, version.Version)
 
 	tr := server.NewTempRegistry(ctx, o.RootDir)

cmd/hauler/cli/store/sync.go

@@ -10,12 +10,13 @@ import (
 	"github.com/spf13/cobra"
 	"k8s.io/apimachinery/pkg/util/yaml"
 
+	"github.com/rancherfederal/ocil/pkg/store"
+
 	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
 	tchart "github.com/rancherfederal/hauler/pkg/collection/chart"
 	"github.com/rancherfederal/hauler/pkg/collection/k3s"
 	"github.com/rancherfederal/hauler/pkg/content"
 	"github.com/rancherfederal/hauler/pkg/log"
-	"github.com/rancherfederal/hauler/pkg/store"
 )
 
 type SyncOpts struct {
@@ -28,7 +29,7 @@ func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
 	f.StringSliceVarP(&o.ContentFiles, "files", "f", []string{}, "Path to content files")
 }
 
-func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error {
+func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error {
 	l := log.FromContext(ctx)
 
 	// Start from an empty store (contents are cached elsewhere)
@@ -120,7 +121,7 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error {
 					return err
 				}
 
-				if _, err := s.AddCollection(ctx, k); err != nil {
+				if _, err := s.AddOCICollection(ctx, k); err != nil {
 					return err
 				}
 
@@ -136,7 +137,7 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error {
 						return err
 					}
 
-					if _, err := s.AddCollection(ctx, tc); err != nil {
+					if _, err := s.AddOCICollection(ctx, tc); err != nil {
 						return err
 					}
 				}

go.mod

@@ -10,14 +10,12 @@ require (
 	github.com/gorilla/handlers v1.5.1
 	github.com/gorilla/mux v1.8.0
 	github.com/mholt/archiver/v3 v3.5.1
-	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.2
 	github.com/pkg/errors v0.9.1
+	github.com/rancherfederal/ocil v0.1.6
 	github.com/rs/zerolog v1.26.0
 	github.com/sirupsen/logrus v1.8.1
-	github.com/spf13/afero v1.6.0
 	github.com/spf13/cobra v1.2.1
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	helm.sh/helm/v3 v3.6.1-0.20211203140009-0e6e936ab84e
 	k8s.io/apimachinery v0.22.4
 	k8s.io/client-go v0.22.4
@@ -48,7 +46,6 @@ require (
 	github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b // indirect
 	github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
-	github.com/containerd/stargz-snapshotter/estargz v0.10.0 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/cli v20.10.11+incompatible // indirect
@@ -112,21 +109,21 @@ require (
 	github.com/nwaples/rardecode v1.1.0 // indirect
 	github.com/onsi/ginkgo v1.16.4 // indirect
 	github.com/onsi/gomega v1.15.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pierrec/lz4/v4 v4.1.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.11.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.26.0 // indirect
-	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/prometheus/common v0.30.0 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc // indirect
 	github.com/russross/blackfriday v1.5.2 // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/spf13/cast v1.3.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.7.0 // indirect
 	github.com/ulikunitz/xz v0.5.9 // indirect
-	github.com/vbatts/tar-split v0.11.2 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
@@ -139,9 +136,10 @@ require (
 	golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 // indirect
 	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 // indirect
 	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
-	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20211111162719-482062a4217b // indirect