add thick chart builtin collection (chart with required images)

cmd/hauler/cli/store/add.go

@@ -8,6 +8,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
+	"github.com/rancherfederal/hauler/pkg/artifact"
 	"github.com/rancherfederal/hauler/pkg/cache"
 	"github.com/rancherfederal/hauler/pkg/content/chart"
 	"github.com/rancherfederal/hauler/pkg/content/file"
@@ -54,17 +55,18 @@ func storeFile(ctx context.Context, s *store.Store, c cache.Cache, fi v1alpha1.F
 		return err
 	}
 
+	var oci artifact.OCI
 	if c != nil {
-		cf := cache.Oci(f, c)
-		f = cf
+		cached := cache.Oci(f, c)
+		oci = cached
 	}
 
 	ref, err := name.ParseReference(fi.Name, name.WithDefaultRegistry(""))
 	if err != nil {
 		return err
 	}
 
-	desc, err := s.AddArtifact(ctx, f, ref)
+	desc, err := s.AddArtifact(ctx, oci, ref)
 	if err != nil {
 		return err
 	}
@@ -109,12 +111,13 @@ func storeImage(ctx context.Context, s *store.Store, c cache.Cache, i v1alpha1.I
 		return err
 	}
 
+	var oci artifact.OCI
 	if c != nil {
-		ci := cache.Oci(img, c)
-		img = ci
+		cached := cache.Oci(img, c)
+		oci = cached
 	}
 
-	desc, err := s.AddArtifact(ctx, img, ref)
+	desc, err := s.AddArtifact(ctx, oci, ref)
 	if err != nil {
 		return err
 	}
@@ -180,12 +183,13 @@ func storeChart(ctx context.Context, s *store.Store, c cache.Cache, ch v1alpha1.
 		return err
 	}
 
+	var oci artifact.OCI
 	if c != nil {
-		cch := cache.Oci(chrt, c)
-		chrt = cch
+		cached := cache.Oci(chrt, c)
+		oci = cached
 	}
 
-	desc, err := s.AddArtifact(ctx, chrt, ref)
+	desc, err := s.AddArtifact(ctx, oci, ref)
 	if err != nil {
 		return err
 	}

cmd/hauler/cli/store/sync.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
 	"github.com/rancherfederal/hauler/pkg/cache"
+	"github.com/rancherfederal/hauler/pkg/collection/chart"
 	"github.com/rancherfederal/hauler/pkg/collection/k3s"
 	"github.com/rancherfederal/hauler/pkg/content"
 	"github.com/rancherfederal/hauler/pkg/log"
@@ -127,6 +128,23 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store, c cache.Cache) er
 					return err
 				}
 
+			case v1alpha1.ChartsCollectionKind:
+				var cfg v1alpha1.ThickCharts
+				if err := yaml.Unmarshal(doc, &cfg); err != nil {
+					return err
+				}
+
+				for _, cfg := range cfg.Spec.Charts {
+					tc, err := chart.NewChart(cfg.Name, cfg.RepoURL, cfg.Version)
+					if err != nil {
+						return err
+					}
+
+					if _, err := s.AddCollection(ctx, tc); err != nil {
+						return err
+					}
+				}
+
 			default:
 				return fmt.Errorf("unrecognized content/collection type: %s", obj.GroupVersionKind().String())
 			}

pkg/apis/hauler.cattle.io/v1alpha1/chart.go

@@ -4,7 +4,10 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-const ChartsContentKind = "Charts"
+const (
+	ChartsContentKind    = "Charts"
+	ChartsCollectionKind = "ThickCharts"
+)
 
 type Charts struct {
 	*metav1.TypeMeta  `json:",inline"`
@@ -22,3 +25,20 @@ type Chart struct {
 	RepoURL string `json:"repoURL"`
 	Version string `json:"version"`
 }
+
+type ThickCharts struct {
+	*metav1.TypeMeta  `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec ChartSpec `json:"spec,omitempty"`
+}
+
+type ThickChartSpec struct {
+	ThickCharts []ThickChart `json:"charts,omitempty"`
+}
+
+type ThickChart struct {
+	Name    string `json:"name"`
+	RepoURL string `json:"repoURL"`
+	Version string `json:"version"`
+}

pkg/collection/chart/chart.go

@@ -0,0 +1,78 @@
+package chart
+
+import (
+	gname "github.com/google/go-containerregistry/pkg/name"
+
+	"github.com/rancherfederal/hauler/pkg/artifact"
+	"github.com/rancherfederal/hauler/pkg/content/chart"
+	"github.com/rancherfederal/hauler/pkg/content/image"
+)
+
+var _ artifact.Collection = (*tchart)(nil)
+
+// tchart is a thick chart that includes all the dependent images as well as the chart itself
+type tchart struct {
+	chart *chart.Chart
+
+	computed bool
+	contents map[gname.Reference]artifact.OCI
+}
+
+func NewChart(name, repo, version string) (artifact.Collection, error) {
+	o, err := chart.NewChart(name, repo, version)
+	if err != nil {
+		return nil, err
+	}
+
+	return &tchart{
+		chart:    o,
+		contents: make(map[gname.Reference]artifact.OCI),
+	}, nil
+}
+
+func (c *tchart) Contents() (map[gname.Reference]artifact.OCI, error) {
+	if err := c.compute(); err != nil {
+		return nil, err
+	}
+	return c.contents, nil
+}
+
+func (c *tchart) compute() error {
+	if c.computed {
+		return nil
+	}
+
+	if err := c.dependentImages(); err != nil {
+		return err
+	}
+
+	c.computed = true
+	return nil
+}
+
+func (c *tchart) dependentImages() error {
+	ch, err := c.chart.Load()
+	if err != nil {
+		return err
+	}
+
+	imgs, err := ImagesInChart(ch)
+	if err != nil {
+		return err
+	}
+
+	for _, img := range imgs.Spec.Images {
+		ref, err := gname.ParseReference(img.Ref)
+		if err != nil {
+			return err
+		}
+
+		i, err := image.NewImage(img.Ref)
+		if err != nil {
+			return err
+		}
+		c.contents[ref] = i
+	}
+
+	return nil
+}

pkg/collection/k3s/k3s.go

@@ -35,23 +35,23 @@ type k3s struct {
 	version string
 	arch    string
 
-	computed   bool
-	components map[name.Reference]artifact.OCI
-	channels   map[string]string
+	computed bool
+	contents map[name.Reference]artifact.OCI
+	channels map[string]string
 }
 
 func NewK3s(version string) (artifact.Collection, error) {
 	return &k3s{
-		version:    version,
-		components: make(map[name.Reference]artifact.OCI),
+		version:  version,
+		contents: make(map[name.Reference]artifact.OCI),
 	}, nil
 }
 
 func (k *k3s) Contents() (map[name.Reference]artifact.OCI, error) {
 	if err := k.compute(); err != nil {
 		return nil, err
 	}
-	return k.components, nil
+	return k.contents, nil
 }
 
 func (k *k3s) compute() error {
@@ -99,7 +99,7 @@ func (k *k3s) executable() error {
 		return err
 	}
 
-	k.components[ref] = f
+	k.contents[ref] = f
 	return nil
 }
 
@@ -124,7 +124,7 @@ func (k *k3s) images() error {
 			return err
 		}
 
-		k.components[ref] = o
+		k.contents[ref] = o
 	}
 	return nil
 }

pkg/content/chart/chart.go

@@ -12,6 +12,7 @@ import (
 	gtypes "github.com/google/go-containerregistry/pkg/v1/types"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/chart/loader"
 	"helm.sh/helm/v3/pkg/cli"
 
@@ -20,20 +21,15 @@ import (
 	"github.com/rancherfederal/hauler/pkg/artifact/types"
 )
 
-const (
-	// ChartLayerMediaType is the reserved media type for Helm chart package content
-	ChartLayerMediaType = "application/vnd.cncf.helm.chart.content.v1.tar+gzip"
-)
-
-var _ artifact.OCI = (*chrt)(nil)
+var _ artifact.OCI = (*Chart)(nil)
 
-type chrt struct {
+type Chart struct {
 	path string
 
 	annotations map[string]string
 }
 
-func NewChart(name, repo, version string) (artifact.OCI, error) {
+func NewChart(name, repo, version string) (*Chart, error) {
 	cpo := action.ChartPathOptions{
 		RepoURL: repo,
 		Version: version,
@@ -44,16 +40,16 @@ func NewChart(name, repo, version string) (artifact.OCI, error) {
 		return nil, err
 	}
 
-	return &chrt{
+	return &Chart{
 		path: cp,
 	}, nil
 }
 
-func (h *chrt) MediaType() string {
+func (h *Chart) MediaType() string {
 	return types.OCIManifestSchema1
 }
 
-func (h *chrt) Manifest() (*gv1.Manifest, error) {
+func (h *Chart) Manifest() (*gv1.Manifest, error) {
 	cfgDesc, err := h.configDescriptor()
 	if err != nil {
 		return nil, err
@@ -78,15 +74,15 @@ func (h *chrt) Manifest() (*gv1.Manifest, error) {
 	}, nil
 }
 
-func (h *chrt) RawConfig() ([]byte, error) {
+func (h *Chart) RawConfig() ([]byte, error) {
 	ch, err := loader.Load(h.path)
 	if err != nil {
 		return nil, err
 	}
 	return json.Marshal(ch.Metadata)
 }
 
-func (h *chrt) configDescriptor() (gv1.Descriptor, error) {
+func (h *Chart) configDescriptor() (gv1.Descriptor, error) {
 	data, err := h.RawConfig()
 	if err != nil {
 		return gv1.Descriptor{}, err
@@ -104,7 +100,16 @@ func (h *chrt) configDescriptor() (gv1.Descriptor, error) {
 	}, nil
 }
 
-func (h *chrt) Layers() ([]gv1.Layer, error) {
+func (h *Chart) Load() (*chart.Chart, error) {
+	rc, err := chartOpener(h.path)()
+	if err != nil {
+		return nil, err
+	}
+	defer rc.Close()
+	return loader.LoadArchive(rc)
+}
+
+func (h *Chart) Layers() ([]gv1.Layer, error) {
 	chartDataLayer, err := h.chartDataLayer()
 	if err != nil {
 		return nil, err
@@ -116,11 +121,11 @@ func (h *chrt) Layers() ([]gv1.Layer, error) {
 	}, nil
 }
 
-func (h *chrt) RawChartData() ([]byte, error) {
+func (h *Chart) RawChartData() ([]byte, error) {
 	return os.ReadFile(h.path)
 }
 
-func (h *chrt) chartDataLayer() (gv1.Layer, error) {
+func (h *Chart) chartDataLayer() (gv1.Layer, error) {
 	annotations := make(map[string]string)
 	annotations[ocispec.AnnotationTitle] = filepath.Base(h.path)
 

pkg/content/file/file.go

@@ -26,7 +26,7 @@ type file struct {
 	annotations map[string]string
 }
 
-func NewFile(ref string, filename string) (artifact.OCI, error) {
+func NewFile(ref string, filename string) (*file, error) {
 	var getter local.Opener
 	if strings.HasPrefix(ref, "http") || strings.HasPrefix(ref, "https") {
 		getter = remoteOpener(ref)

pkg/content/image/image.go

@@ -26,7 +26,7 @@ type image struct {
 	gv1.Image
 }
 
-func NewImage(ref string) (artifact.OCI, error) {
+func NewImage(ref string) (*image, error) {
 	r, err := name.ParseReference(ref)
 	if err != nil {
 		return nil, err

testdata/chart-collection.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: collection.hauler.cattle.io/v1alpha1
+kind: ThickCharts
+metadata:
+  name: mythickchart
+spec:
+  charts:
+    # charts are also fetched and served as OCI content (currently experimental in helm)
+    #   HELM_EXPERIMENTAL_OCI=1 helm chart pull <hauler-registry>/loki:2.6.2
+    - name: loki
+      repoURL: https://grafana.github.io/helm-charts