From b7fc1744f73bc309f0cfdec7cad569d63ef9cde5 Mon Sep 17 00:00:00 2001 From: Philip Laine Date: Wed, 11 Sep 2024 19:34:39 +0200 Subject: [PATCH] refactor: pull Signed-off-by: Philip Laine --- src/cmd/package.go | 18 +- src/internal/packager2/packager2.go | 225 +++++++++++++++++++++++ src/test/e2e/11_oci_pull_inspect_test.go | 5 - 3 files changed, 237 insertions(+), 11 deletions(-) create mode 100644 src/internal/packager2/packager2.go diff --git a/src/cmd/package.go b/src/cmd/package.go index a40439d53f..6f073a373f 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -8,14 +8,17 @@ import ( "context" "errors" "fmt" + "os" "path/filepath" "regexp" "strings" "github.com/zarf-dev/zarf/src/cmd/common" "github.com/zarf-dev/zarf/src/config/lang" + "github.com/zarf-dev/zarf/src/internal/packager2" "github.com/zarf-dev/zarf/src/pkg/lint" "github.com/zarf-dev/zarf/src/pkg/message" + "github.com/zarf-dev/zarf/src/pkg/packager/filters" "github.com/zarf-dev/zarf/src/pkg/packager/sources" "github.com/zarf-dev/zarf/src/types" @@ -278,15 +281,18 @@ var packagePullCmd = &cobra.Command{ Example: lang.CmdPackagePullExample, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - pkgConfig.PkgOpts.PackageSource = args[0] - pkgClient, err := packager.New(&pkgConfig) + outputDir := pkgConfig.PullOpts.OutputDirectory + if outputDir == "" { + wd, err := os.Getwd() + if err != nil { + return err + } + outputDir = wd + } + err := packager2.Fetch(cmd.Context(), args[0], outputDir, pkgConfig.PkgOpts.Shasum, filters.Empty()) if err != nil { return err } - defer pkgClient.ClearTempPaths() - if err := pkgClient.Pull(cmd.Context()); err != nil { - return fmt.Errorf("failed to pull package: %w", err) - } return nil }, } diff --git a/src/internal/packager2/packager2.go b/src/internal/packager2/packager2.go new file mode 100644 index 0000000000..38e68d505c --- /dev/null +++ b/src/internal/packager2/packager2.go @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager2 is the new implementation for packager. +package packager2 + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/defenseunicorns/pkg/helpers/v2" + "github.com/defenseunicorns/pkg/oci" + goyaml "github.com/goccy/go-yaml" + "github.com/mholt/archiver/v3" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/zarf-dev/zarf/src/api/v1alpha1" + "github.com/zarf-dev/zarf/src/config" + "github.com/zarf-dev/zarf/src/pkg/layout" + "github.com/zarf-dev/zarf/src/pkg/packager/filters" + "github.com/zarf-dev/zarf/src/pkg/utils" + "github.com/zarf-dev/zarf/src/pkg/zoci" +) + +// Fetch fetches the Zarf package from the given sources. +func Fetch(ctx context.Context, src, dir, shasum string, filter filters.ComponentFilterStrategy) error { + u, err := url.Parse(src) + if err != nil { + return err + } + if u.Scheme == "" { + return errors.New("scheme cannot be empty") + } + if u.Host == "" { + return errors.New("host cannot be empty") + } + + tmpDir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) + if err != nil { + return err + } + defer os.Remove(tmpDir) + tmpPath := filepath.Join(tmpDir, "data.tar.zst") + + switch u.Scheme { + case "oci": + err := fetchOCI(ctx, src, tmpPath, shasum, filter) + if err != nil { + return err + } + case "http", "https": + err := fetchHTTP(ctx, src, tmpPath, shasum) + if err != nil { + return err + } + default: + return fmt.Errorf("unknown scheme %s", u.Scheme) + } + + name, err := nameFromMetadata(tmpPath) + if err != nil { + return err + } + tarPath := filepath.Join(dir, name) + err = os.Remove(tarPath) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + dstFile, err := os.Create(tarPath) + if err != nil { + return err + } + defer dstFile.Close() + srcFile, err := os.Open(tmpPath) + if err != nil { + return err + } + defer srcFile.Close() + _, err = io.Copy(dstFile, srcFile) + if err != nil { + return err + } + return nil +} + +func fetchOCI(ctx context.Context, src, tarPath, shasum string, filter filters.ComponentFilterStrategy) error { + tmpDir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) + if err != nil { + return err + } + defer os.Remove(tmpDir) + if shasum != "" { + src = fmt.Sprintf("%s@sha256:%s", src, shasum) + } + arch := config.GetArch() + remote, err := zoci.NewRemote(src, oci.PlatformForArch(arch)) + if err != nil { + return err + } + desc, err := remote.ResolveRoot(ctx) + if err != nil { + return fmt.Errorf("could not fetch images index: %w", err) + } + layersToPull := []ocispec.Descriptor{} + skeletonPlatform := zoci.PlatformForSkeleton() + if desc.Platform == nil || !(desc.Platform.Architecture == skeletonPlatform.Architecture && desc.Platform.OS == skeletonPlatform.OS) { + pkg, err := remote.FetchZarfYAML(ctx) + if err != nil { + return err + } + pkg.Components, err = filter.Apply(pkg) + if err != nil { + return err + } + layersToPull, err = remote.LayersFromRequestedComponents(ctx, pkg.Components) + if err != nil { + return err + } + } + _, err = remote.PullPackage(ctx, tmpDir, config.CommonOptions.OCIConcurrency, layersToPull...) + if err != nil { + return err + } + allTheLayers, err := filepath.Glob(filepath.Join(tmpDir, "*")) + if err != nil { + return err + } + err = archiver.Archive(allTheLayers, tarPath) + if err != nil { + return err + } + return nil +} + +func fetchHTTP(ctx context.Context, src, tarPath, shasum string) error { + if !config.CommonOptions.Insecure && shasum == "" { + return errors.New("remote package provided without shasum while insecure is not enabled") + } + f, err := os.Create(tarPath) + if err != nil { + return err + } + defer f.Close() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, src, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + _, err := io.Copy(io.Discard, resp.Body) + if err != nil { + return err + } + return fmt.Errorf("unexpected http response status code %s for source %s", resp.Status, src) + } + _, err = io.Copy(f, resp.Body) + if err != nil { + return err + } + // Check checksum if src included one. + if shasum != "" { + received, err := helpers.GetSHA256OfFile(tarPath) + if err != nil { + return err + } + if received != shasum { + return fmt.Errorf("shasum mismatch for file %s, expected %s bu got %s ", tarPath, shasum, received) + } + } + return nil +} + +func nameFromMetadata(path string) (string, error) { + var pkg v1alpha1.ZarfPackage + err := archiver.Walk(path, func(f archiver.File) error { + if f.Name() == layout.ZarfYAML { + b, err := io.ReadAll(f) + if err != nil { + return err + } + if err := goyaml.Unmarshal(b, &pkg); err != nil { + return err + } + } + return nil + }) + if err != nil { + return "", err + } + if pkg.Metadata.Name == "" { + return "", fmt.Errorf("%s does not contain a zarf.yaml", path) + } + + arch := config.GetArch(pkg.Metadata.Architecture, pkg.Build.Architecture) + if pkg.Build.Architecture == zoci.SkeletonArch { + arch = zoci.SkeletonArch + } + + var name string + switch pkg.Kind { + case v1alpha1.ZarfInitConfig: + name = fmt.Sprintf("zarf-init-%s", arch) + case v1alpha1.ZarfPackageConfig: + name = fmt.Sprintf("zarf-package-%s-%s", pkg.Metadata.Name, arch) + default: + name = fmt.Sprintf("zarf-%s-%s", strings.ToLower(string(pkg.Kind)), arch) + } + if pkg.Build.Differential { + name = fmt.Sprintf("%s-%s-differential-%s", name, pkg.Build.DifferentialPackageVersion, pkg.Metadata.Version) + } else if pkg.Metadata.Version != "" { + name = fmt.Sprintf("%s-%s", name, pkg.Metadata.Version) + } + return fmt.Sprintf("%s.tar.zst", name), nil +} diff --git a/src/test/e2e/11_oci_pull_inspect_test.go b/src/test/e2e/11_oci_pull_inspect_test.go index cd045ae0a6..6384828b61 100644 --- a/src/test/e2e/11_oci_pull_inspect_test.go +++ b/src/test/e2e/11_oci_pull_inspect_test.go @@ -47,9 +47,6 @@ func (suite *PullInspectTestSuite) Test_0_Pull() { // Pull the package via OCI. stdOut, stdErr, err := e2e.Zarf(suite.T(), "package", "pull", ref) suite.NoError(err, stdOut, stdErr) - suite.Contains(stdErr, fmt.Sprintf("Pulling %q", ref)) - suite.Contains(stdErr, "Validating full package checksums") - suite.NotContains(stdErr, "Package signature validated!") sbomTmp := suite.T().TempDir() @@ -57,8 +54,6 @@ func (suite *PullInspectTestSuite) Test_0_Pull() { suite.FileExists(out) stdOut, stdErr, err = e2e.Zarf(suite.T(), "package", "inspect", out, "--key", "https://raw.githubusercontent.com/zarf-dev/zarf/v0.38.2/cosign.pub", "--sbom-out", sbomTmp) suite.NoError(err, stdOut, stdErr) - suite.Contains(stdErr, "Validating SBOM checksums") - suite.Contains(stdErr, "Package signature validated!") // Test pull w/ bad ref. stdOut, stdErr, err = e2e.Zarf(suite.T(), "package", "pull", "oci://"+badPullInspectRef.String(), "--insecure")