Some OCI Optimizations

src/config/config.go

@@ -57,6 +57,8 @@ const (
 	ZarfSBOMTar       = "sboms.tar"
 	ZarfPackagePrefix = "zarf-package-"
 
+	ZarfComponentsDir = "components"
+
 	ZarfInClusterContainerRegistryNodePort = 31999
 
 	ZarfInClusterGitServiceURL      = "http://zarf-gitea-http.zarf.svc.cluster.local:3000"

src/pkg/packager/common.go

@@ -224,7 +224,7 @@ func createPaths() (paths types.TempPaths, err error) {
 		InjectBinary: filepath.Join(basePath, "zarf-injector"),
 		SeedImages:   filepath.Join(basePath, "seed-images"),
 		Images:       filepath.Join(basePath, "images"),
-		Components:   filepath.Join(basePath, "components"),
+		Components:   filepath.Join(basePath, config.ZarfComponentsDir),
 		SbomTar:      filepath.Join(basePath, config.ZarfSBOMTar),
 		Sboms:        filepath.Join(basePath, "sboms"),
 		Checksums:    filepath.Join(basePath, config.ZarfChecksumsTxt),
@@ -272,7 +272,7 @@ func (p *Packager) loadZarfPkg() error {
 	}
 
 	// Load the config from the extracted archive zarf.yaml
-	spinner.Updatef("Loading the zarf package config")
+	spinner.Updatef("Loading the Zarf package config")
 	configPath := p.tmp.ZarfYaml
 	if err := p.readYaml(configPath); err != nil {
 		return fmt.Errorf("unable to read the zarf.yaml in %s: %w", p.tmp.Base, err)
@@ -474,7 +474,13 @@ func (p *Packager) validatePackageChecksums() error {
 		itemPath := strs[1]
 		expectedShasum := strs[0]
 
-		actualShasum, err := utils.GetSHA256OfFile(filepath.Join(p.tmp.Base, itemPath))
+		path := filepath.Join(p.tmp.Base, itemPath)
+		if utils.InvalidPath(path) {
+			message.Debugf("skipping checksum of %s as it does not exist", path)
+			continue
+		}
+
+		actualShasum, err := utils.GetSHA256OfFile(path)
 		if err != nil {
 			return err
 		}
@@ -483,7 +489,7 @@ func (p *Packager) validatePackageChecksums() error {
 			return fmt.Errorf("mismatch on the checksum of the %s file (expected: %s, actual: %s)", itemPath, expectedShasum, actualShasum)
 		}
 
-		filepathMap[filepath.Join(p.tmp.Base, itemPath)] = true
+		filepathMap[path] = true
 	}
 
 	for path, processed := range filepathMap {

src/pkg/packager/compose.go

@@ -102,8 +102,16 @@ func (p *Packager) getChildComponent(parent types.ZarfComponent, pathAncestry st
 			return child, fmt.Errorf("unable to create cache path %s: %w", cachePath, err)
 		}
 
-		componentLayer := filepath.Join("components", fmt.Sprintf("%s.tar", childComponentName))
-		err = p.handleOciPackage(skelURL, cachePath, 3, componentLayer)
+		componentLayer := filepath.Join(config.ZarfComponentsDir, fmt.Sprintf("%s.tar", childComponentName))
+		client, err := utils.NewOrasRemote(parent.Import.URL)
+		if err != nil {
+			return child, err
+		}
+		manifest, err := client.FetchRoot()
+		if err != nil {
+			return child, err
+		}
+		err = client.PullPackage(cachePath, 3, manifest.Locate(componentLayer))
 		if err != nil {
 			return child, fmt.Errorf("unable to pull skeleton from %s: %w", skelURL, err)
 		}
@@ -149,17 +157,17 @@ func (p *Packager) getChildComponent(parent types.ZarfComponent, pathAncestry st
 
 	// If it's OCI, we need to unpack the component tarball
 	if parent.Import.URL != "" {
-		dir := filepath.Join(cachePath, "components", child.Name)
+		dir := filepath.Join(cachePath, config.ZarfComponentsDir, child.Name)
 		componentTarball := fmt.Sprintf("%s.tar", dir)
-		parent.Import.Path = filepath.Join(parent.Import.Path, "components", child.Name)
+		parent.Import.Path = filepath.Join(parent.Import.Path, config.ZarfComponentsDir, child.Name)
 		if !utils.InvalidPath(componentTarball) {
 			if !utils.InvalidPath(dir) {
 				err = os.RemoveAll(dir)
 				if err != nil {
 					return child, fmt.Errorf("unable to remove composed component cache path %s: %w", cachePath, err)
 				}
 			}
-			err = archiver.Unarchive(componentTarball, filepath.Join(cachePath, "components"))
+			err = archiver.Unarchive(componentTarball, filepath.Join(cachePath, config.ZarfComponentsDir))
 			if err != nil {
 				return child, fmt.Errorf("unable to unpack composed component tarball: %w", err)
 			}

src/pkg/packager/create.go

@@ -601,8 +601,21 @@ func (p *Packager) loadDifferentialData() error {
 
 	// Load the package spec of the package we're using as a 'reference' for the differential build
 	if utils.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) {
-		if err := p.pullPackageLayers(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, tmpDir, []string{config.ZarfYAML}); err != nil {
-			return fmt.Errorf("unable to pull the differential zarf package spec: %s", err.Error())
+		client, err := utils.NewOrasRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath)
+		if err != nil {
+			return err
+		}
+		manifest, err := client.FetchRoot()
+		if err != nil {
+			return err
+		}
+		pkg, err := client.FetchZarfYAML(manifest)
+		if err != nil {
+			return err
+		}
+		err = utils.WriteYaml(filepath.Join(tmpDir, config.ZarfYAML), pkg, 0600)
+		if err != nil {
+			return err
 		}
 	} else {
 		if err := archiver.Extract(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, config.ZarfYAML, tmpDir); err != nil {

src/pkg/packager/inspect.go

@@ -25,17 +25,26 @@ func (p *Packager) Inspect(includeSBOM bool, outputSBOM string, inspectPublicKey
 		pullSBOM := includeSBOM || outputSBOM != ""
 		pullZarfSig := inspectPublicKey != ""
 
-		layersToPull := []string{config.ZarfYAML}
+		requestedFiles := []string{config.ZarfYAML}
 		if pullSBOM {
-			layersToPull = append(layersToPull, config.ZarfSBOMTar)
+			requestedFiles = append(requestedFiles, config.ZarfSBOMTar)
 		}
 		if pullZarfSig {
-			layersToPull = append(layersToPull, config.ZarfYAMLSignature)
+			requestedFiles = append(requestedFiles, config.ZarfYAMLSignature)
 		}
 
-		message.Debugf("Pulling layers %v from %s", layersToPull, p.cfg.DeployOpts.PackagePath)
-		if err := p.pullPackageLayers(p.cfg.DeployOpts.PackagePath, p.tmp.Base, layersToPull); err != nil {
-			return fmt.Errorf("unable to pull layers for inspect: %w", err)
+		message.Debugf("Pulling layers %v from %s", requestedFiles, p.cfg.DeployOpts.PackagePath)
+
+		client, err := utils.NewOrasRemote(p.cfg.DeployOpts.PackagePath)
+		if err != nil {
+			return err
+		}
+		layersToPull, err := client.LayersFromPaths(requestedFiles)
+		if err != nil {
+			return err
+		}
+		if err := client.PullPackage(p.tmp.Base, p.cfg.PullOpts.CopyOptions.Concurrency, layersToPull...); err != nil {
+			return fmt.Errorf("unable to pull the package: %w", err)
 		}
 		if err := p.readYaml(p.tmp.ZarfYaml); err != nil {
 			return fmt.Errorf("unable to read the zarf.yaml in %s: %w", p.tmp.Base, err)

src/pkg/packager/network.go

@@ -4,24 +4,18 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
-	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
-	"sync"
 
 	"github.com/defenseunicorns/zarf/src/config"
 	"github.com/defenseunicorns/zarf/src/pkg/message"
 	"github.com/defenseunicorns/zarf/src/pkg/utils"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"oras.land/oras-go/v2"
-	"oras.land/oras-go/v2/content"
-	"oras.land/oras-go/v2/content/file"
-	"oras.land/oras-go/v2/registry"
 )
 
 // handlePackagePath If provided package is a URL download it to a temp directory.
@@ -41,7 +35,20 @@ func (p *Packager) handlePackagePath() error {
 	if utils.IsOCIURL(opts.PackagePath) {
 		ociURL := opts.PackagePath
 		p.cfg.DeployOpts.PackagePath = p.tmp.Base
-		return p.handleOciPackage(ociURL, p.tmp.Base, p.cfg.PublishOpts.CopyOptions.Concurrency)
+		client, err := utils.NewOrasRemote(ociURL)
+		if err != nil {
+			return err
+		}
+		requestedComponents := getRequestedComponentList(p.cfg.DeployOpts.Components)
+		layersToPull := []ocispec.Descriptor{}
+		if len(requestedComponents) > 0 {
+			layers, err := client.LayersFromRequestedComponents(requestedComponents)
+			if err != nil {
+				return fmt.Errorf("unable to get published component image layers: %s", err.Error())
+			}
+			layersToPull = append(layersToPull, layers...)
+		}
+		return client.PullPackage(p.tmp.Base, p.cfg.PublishOpts.CopyOptions.Concurrency, layersToPull...)
 	}
 
 	// Handle case where deploying remote package validated via sget
@@ -130,145 +137,3 @@ func (p *Packager) handleSgetPackage() error {
 	spinner.Success()
 	return nil
 }
-
-func (p *Packager) handleOciPackage(url string, out string, concurrency int, layers ...string) error {
-	message.Debugf("packager.handleOciPackage(%s, %s, %d, %s)", url, out, concurrency, layers)
-	ref, err := registry.ParseReference(strings.TrimPrefix(url, utils.OCIURLPrefix))
-	if err != nil {
-		return fmt.Errorf("failed to parse OCI reference: %w", err)
-	}
-
-	message.Debugf("Pulling %s", ref.String())
-	message.Infof("Pulling Zarf package from %s", ref)
-
-	src, err := utils.NewOrasRemote(ref)
-	if err != nil {
-		return err
-	}
-
-	estimatedBytes, err := getOCIPackageSize(src, layers...)
-	if err != nil {
-		return err
-	}
-
-	dst, err := file.New(out)
-	if err != nil {
-		return err
-	}
-	defer dst.Close()
-
-	copyOpts := oras.DefaultCopyOptions
-	copyOpts.Concurrency = concurrency
-	copyOpts.OnCopySkipped = func(ctx context.Context, desc ocispec.Descriptor) error {
-		title := desc.Annotations[ocispec.AnnotationTitle]
-		var format string
-		if title != "" {
-			format = fmt.Sprintf("%s %s", desc.Digest.Encoded()[:12], utils.First30last30(title))
-		} else {
-			format = fmt.Sprintf("%s [%s]", desc.Digest.Encoded()[:12], desc.MediaType)
-		}
-		message.Successf(format)
-		return nil
-	}
-	copyOpts.PostCopy = copyOpts.OnCopySkipped
-	isPartialPull := len(layers) > 0
-	if isPartialPull {
-		alwaysPull := []string{config.ZarfYAML, config.ZarfChecksumsTxt, config.ZarfYAMLSignature}
-		layers = append(layers, alwaysPull...)
-		copyOpts.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
-			nodes, err := content.Successors(ctx, fetcher, desc)
-			if err != nil {
-				return nil, err
-			}
-			var ret []ocispec.Descriptor
-			for _, node := range nodes {
-				if utils.SliceContains(layers, node.Annotations[ocispec.AnnotationTitle]) {
-					ret = append(ret, node)
-				}
-			}
-			return ret, nil
-		}
-	}
-
-	// Create a thread to update a progress bar as we save the package to disk
-	doneSaving := make(chan int)
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go utils.RenderProgressBarForLocalDirWrite(out, estimatedBytes, &wg, doneSaving, "Pulling Zarf package data")
-	_, err = oras.Copy(src.Context, src.Repository, ref.Reference, dst, ref.Reference, copyOpts)
-	if err != nil {
-		return err
-	}
-
-	// Send a signal to the progress bar that we're done and wait for it to finish
-	doneSaving <- 1
-	wg.Wait()
-
-	message.Debugf("Pulled %s", ref.String())
-	message.Successf("Pulled %s", ref.String())
-
-	return nil
-}
-
-func getOCIPackageSize(src *utils.OrasRemote, layers ...string) (int64, error) {
-	var total int64
-
-	manifest, err := getManifest(src)
-	if err != nil {
-		return 0, err
-	}
-
-	manifestLayers := manifest.Layers
-
-	processedLayers := make(map[string]bool)
-	for _, layer := range manifestLayers {
-		// Only include this layer's size if we haven't already processed it
-		hasBeenProcessed := processedLayers[layer.Digest.String()]
-		if !hasBeenProcessed {
-			if len(layers) > 0 {
-				// If we're only pulling a subset of layers, only include the size of the layers we're pulling
-				if utils.SliceContains(layers, layer.Annotations[ocispec.AnnotationTitle]) {
-					total += layer.Size
-					processedLayers[layer.Digest.String()] = true
-					continue
-				}
-			}
-			total += layer.Size
-			processedLayers[layer.Digest.String()] = true
-		}
-	}
-
-	return total, nil
-}
-
-// getManifest fetches the manifest from a Zarf OCI package
-func getManifest(dst *utils.OrasRemote) (*ocispec.Manifest, error) {
-	// get the manifest descriptor
-	// ref.Reference can be a tag or a digest
-	descriptor, err := dst.Resolve(dst.Context, dst.Reference.Reference)
-	if err != nil {
-		return nil, err
-	}
-
-	// get the manifest itself
-	pulled, err := content.FetchAll(dst.Context, dst, descriptor)
-	if err != nil {
-		return nil, err
-	}
-	manifest := ocispec.Manifest{}
-
-	if err = json.Unmarshal(pulled, &manifest); err != nil {
-		return nil, err
-	}
-	return &manifest, nil
-}
-
-// pullLayer fetches a single layer from a Zarf OCI package
-func pullLayer(dst *utils.OrasRemote, desc ocispec.Descriptor, out string) error {
-	bytes, err := content.FetchAll(dst.Context, dst, desc)
-	if err != nil {
-		return err
-	}
-	err = utils.WriteFile(out, bytes)
-	return err
-}

src/pkg/packager/publish.go

@@ -81,11 +81,11 @@ func (p *Packager) Publish() error {
 	}
 
 	message.HeaderInfof("📦 PACKAGE PUBLISH %s:%s", p.cfg.Pkg.Metadata.Name, ref.Reference)
-	return p.publish(ref)
+	return p.publish(ref.String())
 }
 
-func (p *Packager) publish(ref registry.Reference) error {
-	message.Infof("Publishing package to %s", ref)
+func (p *Packager) publish(url string) error {
+	message.Infof("Publishing package to %s", url)
 	spinner := message.NewProgressSpinner("")
 	defer spinner.Stop()
 
@@ -108,7 +108,7 @@ func (p *Packager) publish(ref registry.Reference) error {
 	}
 
 	// destination remote
-	dst, err := utils.NewOrasRemote(ref)
+	dst, err := utils.NewOrasRemote(url)
 	if err != nil {
 		return err
 	}
@@ -150,9 +150,9 @@ func (p *Packager) publish(ref registry.Reference) error {
 		return err
 	}
 
-	dst.Transport.ProgressBar.Successf("Published %s [%s]", ref, root.MediaType)
+	dst.Transport.ProgressBar.Successf("Published %s [%s]", url, root.MediaType)
 	fmt.Println()
-	if strings.HasSuffix(ref.Reference, skeletonSuffix) {
+	if strings.HasSuffix(url, skeletonSuffix) {
 		message.Info("Example of importing components from this package:")
 		fmt.Println()
 		ex := []types.ZarfComponent{}
@@ -161,7 +161,7 @@ func (p *Packager) publish(ref registry.Reference) error {
 				Name: fmt.Sprintf("import-%s", c.Name),
 				Import: types.ZarfComponentImport{
 					ComponentName: c.Name,
-					URL:           fmt.Sprintf("oci://%s", ref.String()),
+					URL:           fmt.Sprintf("oci://%s", url),
 				},
 			})
 		}
@@ -173,9 +173,9 @@ func (p *Packager) publish(ref registry.Reference) error {
 			flags = "--insecure"
 		}
 		message.Info("To inspect/deploy/pull:")
-		message.Infof("zarf package inspect oci://%s %s", ref, flags)
-		message.Infof("zarf package deploy oci://%s %s", ref, flags)
-		message.Infof("zarf package pull oci://%s %s", ref, flags)
+		message.Infof("zarf package inspect oci://%s %s", url, flags)
+		message.Infof("zarf package deploy oci://%s %s", url, flags)
+		message.Infof("zarf package pull oci://%s %s", url, flags)
 	}
 
 	return nil