From 0f2f9f9711312d52d73c5ae2b56cac353f7f0e97 Mon Sep 17 00:00:00 2001 From: Philip Laine Date: Mon, 4 Nov 2024 20:47:46 +0100 Subject: [PATCH] Refactor zoci push to support new package layout Signed-off-by: Philip Laine --- src/internal/packager2/layout/oci.go | 148 ++++++++++++++++++ src/internal/packager2/layout/oci_test.go | 36 +++++ src/internal/packager2/layout/package.go | 30 +++- src/internal/packager2/layout/package_test.go | 20 +++ 4 files changed, 226 insertions(+), 8 deletions(-) create mode 100644 src/internal/packager2/layout/oci.go create mode 100644 src/internal/packager2/layout/oci_test.go diff --git a/src/internal/packager2/layout/oci.go b/src/internal/packager2/layout/oci.go new file mode 100644 index 0000000000..6ee23f43b7 --- /dev/null +++ b/src/internal/packager2/layout/oci.go @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package layout + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + + "github.com/defenseunicorns/pkg/helpers/v2" + "github.com/defenseunicorns/pkg/oci" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content/file" + "oras.land/oras-go/v2/registry" + + "github.com/zarf-dev/zarf/src/api/v1alpha1" + "github.com/zarf-dev/zarf/src/config" + "github.com/zarf-dev/zarf/src/pkg/logger" + "github.com/zarf-dev/zarf/src/pkg/message" +) + +const ( + // ZarfConfigMediaType is the media type for the manifest config + ZarfConfigMediaType = "application/vnd.zarf.config.v1+json" + // ZarfLayerMediaTypeBlob is the media type for all Zarf layers due to the range of possible content + ZarfLayerMediaTypeBlob = "application/vnd.zarf.layer.v1.blob" +) + +// Remote is a wrapper around the Oras remote repository with zarf specific functions +type Remote struct { + orasRemote *oci.OrasRemote +} + +// NewRemote returns an oras remote repository client and context for the given url with zarf opination embedded. +func NewRemote(ctx context.Context, url string, platform ocispec.Platform, mods ...oci.Modifier) (*Remote, error) { + l := slog.New(message.ZarfHandler{}) + if logger.Enabled(ctx) { + l = logger.From(ctx) + } + modifiers := append([]oci.Modifier{ + oci.WithPlainHTTP(config.CommonOptions.PlainHTTP), + oci.WithInsecureSkipVerify(config.CommonOptions.InsecureSkipTLSVerify), + oci.WithLogger(l), + oci.WithUserAgent("zarf/" + config.CLIVersion), + }, mods...) + remote, err := oci.NewOrasRemote(url, platform, modifiers...) + if err != nil { + return nil, err + } + return &Remote{orasRemote: remote}, nil +} + +// Push pushes the given package layout to the remote registry. +func (r *Remote) Push(ctx context.Context, pkgLayout PackageLayout, concurrency int) (err error) { + src, err := file.New("") + if err != nil { + return err + } + defer func(src *file.Store) { + err2 := src.Close() + err = errors.Join(err, err2) + }(src) + + descs := []ocispec.Descriptor{} + files, err := pkgLayout.Files() + if err != nil { + return err + } + for path, name := range files { + desc, err := src.Add(ctx, name, ZarfLayerMediaTypeBlob, path) + if err != nil { + return err + } + descs = append(descs, desc) + } + + annotations := annotationsFromMetadata(pkgLayout.Pkg.Metadata) + manifestConfigDesc, err := r.orasRemote.CreateAndPushManifestConfig(ctx, annotations, ZarfConfigMediaType) + if err != nil { + return err + } + root, err := r.orasRemote.PackAndTagManifest(ctx, src, descs, manifestConfigDesc, annotations) + if err != nil { + return err + } + + copyOpts := r.orasRemote.GetDefaultCopyOpts() + copyOpts.Concurrency = concurrency + publishedDesc, err := oras.Copy(ctx, src, root.Digest.String(), r.orasRemote.Repo(), "", copyOpts) + if err != nil { + return err + } + + err = r.orasRemote.UpdateIndex(ctx, r.orasRemote.Repo().Reference.Reference, publishedDesc) + if err != nil { + return err + } + + return nil +} + +func ReferenceFromMetadata(registryLocation string, metadata *v1alpha1.ZarfMetadata, build *v1alpha1.ZarfBuildData) (string, error) { + if len(metadata.Version) == 0 { + return "", errors.New("version is required for publishing") + } + if !strings.HasSuffix(registryLocation, "/") { + registryLocation = registryLocation + "/" + } + registryLocation = strings.TrimPrefix(registryLocation, helpers.OCIURLPrefix) + + raw := fmt.Sprintf("%s%s:%s", registryLocation, metadata.Name, metadata.Version) + if build != nil && build.Flavor != "" { + raw = fmt.Sprintf("%s-%s", raw, build.Flavor) + } + + ref, err := registry.ParseReference(raw) + if err != nil { + return "", fmt.Errorf("failed to parse %s: %w", raw, err) + } + return ref.String(), nil +} + +func annotationsFromMetadata(metadata v1alpha1.ZarfMetadata) map[string]string { + annotations := map[string]string{ + ocispec.AnnotationTitle: metadata.Name, + ocispec.AnnotationDescription: metadata.Description, + } + if url := metadata.URL; url != "" { + annotations[ocispec.AnnotationURL] = url + } + if authors := metadata.Authors; authors != "" { + annotations[ocispec.AnnotationAuthors] = authors + } + if documentation := metadata.Documentation; documentation != "" { + annotations[ocispec.AnnotationDocumentation] = documentation + } + if source := metadata.Source; source != "" { + annotations[ocispec.AnnotationSource] = source + } + if vendor := metadata.Vendor; vendor != "" { + annotations[ocispec.AnnotationVendor] = vendor + } + return annotations +} diff --git a/src/internal/packager2/layout/oci_test.go b/src/internal/packager2/layout/oci_test.go new file mode 100644 index 0000000000..53341047f1 --- /dev/null +++ b/src/internal/packager2/layout/oci_test.go @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package layout + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/zarf-dev/zarf/src/api/v1alpha1" +) + +func TestAnnotationsFromMetadata(t *testing.T) { + t.Parallel() + + metadata := v1alpha1.ZarfMetadata{ + Name: "foo", + Description: "bar", + URL: "https://example.com", + Authors: "Zarf", + Documentation: "documentation", + Source: "source", + Vendor: "vendor", + } + annotations := annotationsFromMetadata(metadata) + expectedAnnotations := map[string]string{ + "org.opencontainers.image.title": "foo", + "org.opencontainers.image.description": "bar", + "org.opencontainers.image.url": "https://example.com", + "org.opencontainers.image.authors": "Zarf", + "org.opencontainers.image.documentation": "documentation", + "org.opencontainers.image.source": "source", + "org.opencontainers.image.vendor": "vendor", + } + require.Equal(t, expectedAnnotations, annotations) +} diff --git a/src/internal/packager2/layout/package.go b/src/internal/packager2/layout/package.go index 966bcc07dd..708fd19b41 100644 --- a/src/internal/packager2/layout/package.go +++ b/src/internal/packager2/layout/package.go @@ -189,6 +189,27 @@ func (p *PackageLayout) GetImage(ref transform.Image) (registryv1.Image, error) return nil, fmt.Errorf("unable to find the image %s", ref.Reference) } +// Files returns a map off all the files in the package. +func (p *PackageLayout) Files() (map[string]string, error) { + files := map[string]string{} + err := filepath.Walk(p.dirPath, func(path string, info fs.FileInfo, err error) error { + if info.IsDir() { + return nil + } + rel, err := filepath.Rel(p.dirPath, path) + if err != nil { + return err + } + name := filepath.ToSlash(rel) + files[path] = name + return err + }) + if err != nil { + return nil, err + } + return files, nil +} + func validatePackageIntegrity(pkgLayout *PackageLayout, isPartial bool) error { _, err := os.Stat(filepath.Join(pkgLayout.dirPath, ZarfYAML)) if err != nil { @@ -203,14 +224,7 @@ func validatePackageIntegrity(pkgLayout *PackageLayout, isPartial bool) error { return err } - packageFiles := map[string]interface{}{} - err = filepath.Walk(pkgLayout.dirPath, func(path string, info fs.FileInfo, err error) error { - if info.IsDir() { - return nil - } - packageFiles[path] = nil - return err - }) + packageFiles, err := pkgLayout.Files() if err != nil { return err } diff --git a/src/internal/packager2/layout/package_test.go b/src/internal/packager2/layout/package_test.go index 79fd057429..ff75802186 100644 --- a/src/internal/packager2/layout/package_test.go +++ b/src/internal/packager2/layout/package_test.go @@ -52,4 +52,24 @@ func TestPackageLayout(t *testing.T) { dgst, err := img.Digest() require.NoError(t, err) require.Equal(t, "sha256:33735bd63cf84d7e388d9f6d297d348c523c044410f553bd878c6d7829612735", dgst.String()) + + files, err := pkgLayout.Files() + require.NoError(t, err) + expectedNames := []string{ + "checksums.txt", + "components/test.tar", + "images/blobs/sha256/33735bd63cf84d7e388d9f6d297d348c523c044410f553bd878c6d7829612735", + "images/blobs/sha256/43c4264eed91be63b206e17d93e75256a6097070ce643c5e8f0379998b44f170", + "images/blobs/sha256/91ef0af61f39ece4d6710e465df5ed6ca12112358344fd51ae6a3b886634148b", + "images/index.json", + "images/oci-layout", + "sboms.tar", + "zarf.yaml", + } + require.Equal(t, len(expectedNames), len(files)) + for _, expectedName := range expectedNames { + path := filepath.Join(pkgLayout.dirPath, filepath.FromSlash(expectedName)) + name := files[path] + require.Equal(t, expectedName, name) + } }