From 94e3aeae25b1f1d256fea8b14b8b89074ce840bd Mon Sep 17 00:00:00 2001 From: Philip Laine Date: Fri, 6 Sep 2024 19:48:47 +0200 Subject: [PATCH] refactor: mirror-resources Signed-off-by: Philip Laine --- src/cmd/package.go | 46 ++- src/internal/dns/dns.go | 45 +++ src/internal/dns/dns_test.go | 84 ++++++ src/internal/packager2/load.go | 346 +++++++++++++++++++++++ src/internal/packager2/load_test.go | 76 +++++ src/internal/packager2/mirror.go | 213 ++++++++++++++ src/internal/packager2/packager.go | 5 + src/test/external/ext_in_cluster_test.go | 3 +- 8 files changed, 802 insertions(+), 16 deletions(-) create mode 100644 src/internal/dns/dns.go create mode 100644 src/internal/dns/dns_test.go create mode 100644 src/internal/packager2/load.go create mode 100644 src/internal/packager2/load_test.go create mode 100644 src/internal/packager2/mirror.go create mode 100644 src/internal/packager2/packager.go diff --git a/src/cmd/package.go b/src/cmd/package.go index a40439d53f..a42f126736 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -8,19 +8,12 @@ import ( "context" "errors" "fmt" + "os" "path/filepath" "regexp" + "runtime" "strings" - "github.com/zarf-dev/zarf/src/cmd/common" - "github.com/zarf-dev/zarf/src/config/lang" - "github.com/zarf-dev/zarf/src/pkg/lint" - "github.com/zarf-dev/zarf/src/pkg/message" - "github.com/zarf-dev/zarf/src/pkg/packager/sources" - "github.com/zarf-dev/zarf/src/types" - - "oras.land/oras-go/v2/registry" - "github.com/AlecAivazis/survey/v2" "github.com/defenseunicorns/pkg/helpers/v2" "github.com/spf13/cobra" @@ -28,6 +21,17 @@ import ( "github.com/zarf-dev/zarf/src/config" "github.com/zarf-dev/zarf/src/pkg/cluster" "github.com/zarf-dev/zarf/src/pkg/packager" + "oras.land/oras-go/v2/registry" + + "github.com/zarf-dev/zarf/src/cmd/common" + "github.com/zarf-dev/zarf/src/config/lang" + "github.com/zarf-dev/zarf/src/internal/dns" + "github.com/zarf-dev/zarf/src/internal/packager2" + "github.com/zarf-dev/zarf/src/pkg/lint" + "github.com/zarf-dev/zarf/src/pkg/message" + "github.com/zarf-dev/zarf/src/pkg/packager/filters" + "github.com/zarf-dev/zarf/src/pkg/packager/sources" + "github.com/zarf-dev/zarf/src/types" ) var packageCmd = &cobra.Command{ @@ -113,18 +117,30 @@ var packageMirrorCmd = &cobra.Command{ Example: lang.CmdPackageMirrorExample, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - packageSource, err := choosePackage(args) + var c *cluster.Cluster + if dns.IsServiceURL(pkgConfig.InitOpts.RegistryInfo.Address) || dns.IsServiceURL(pkgConfig.InitOpts.GitServer.Address) { + var err error + c, err = cluster.NewCluster() + if err != nil { + return err + } + } + src, err := choosePackage(args) if err != nil { return err } - pkgConfig.PkgOpts.PackageSource = packageSource - pkgClient, err := packager.New(&pkgConfig) + filter := filters.Combine( + filters.ByLocalOS(runtime.GOOS), + filters.BySelectState(pkgConfig.PkgOpts.OptionalComponents), + ) + pkgPaths, err := packager2.LoadPackageFromSource(cmd.Context(), src, pkgConfig.PkgOpts.Shasum, pkgConfig.PkgOpts.PublicKeyPath, filter) if err != nil { return err } - defer pkgClient.ClearTempPaths() - if err := pkgClient.Mirror(cmd.Context()); err != nil { - return fmt.Errorf("failed to mirror package: %w", err) + defer os.Remove(pkgPaths.Base) + err = packager2.Mirror(cmd.Context(), c, *pkgPaths, pkgConfig.InitOpts.RegistryInfo, pkgConfig.InitOpts.GitServer, pkgConfig.MirrorOpts.NoImgChecksum, pkgConfig.PkgOpts.Retries) + if err != nil { + return err } return nil }, diff --git a/src/internal/dns/dns.go b/src/internal/dns/dns.go new file mode 100644 index 0000000000..fcc627a6f5 --- /dev/null +++ b/src/internal/dns/dns.go @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package dns contains DNS related functionality. +package dns + +import ( + "errors" + "fmt" + "net/url" + "regexp" + "strconv" +) + +var ( + // localClusterServiceRegex is used to match the local cluster service format: + localClusterServiceRegex = regexp.MustCompile(`^(?P[^\.]+)\.(?P[^\.]+)\.svc\.cluster\.local$`) +) + +// IsServiceURL returns true if the give url complies with the service url format. +func IsServiceURL(serviceURL string) bool { + _, _, _, err := ParseServiceURL(serviceURL) + return err == nil +} + +// ParseServiceURL takes a serviceURL and parses it to find the service info for connecting to the cluster. The string is expected to follow the following format: +// Example serviceURL: http://{SERVICE_NAME}.{NAMESPACE}.svc.cluster.local:{PORT}. +func ParseServiceURL(serviceURL string) (string, string, int, error) { + parsedURL, err := url.Parse(serviceURL) + if err != nil { + return "", "", 0, err + } + if parsedURL.Port() == "" { + return "", "", 0, errors.New("service url does not have a port") + } + remotePort, err := strconv.Atoi(parsedURL.Port()) + if err != nil { + return "", "", 0, err + } + matches := localClusterServiceRegex.FindStringSubmatch(parsedURL.Hostname()) + if len(matches) != 3 { + return "", "", 0, fmt.Errorf("invalid service url %s", serviceURL) + } + return matches[2], matches[1], remotePort, nil +} diff --git a/src/internal/dns/dns_test.go b/src/internal/dns/dns_test.go new file mode 100644 index 0000000000..e9b62f6777 --- /dev/null +++ b/src/internal/dns/dns_test.go @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package dns + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIsServiceURL(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + serviceURL string + expected bool + }{ + { + name: "is service url", + serviceURL: "http://registry.zarf.svc.cluster.local:1", + expected: true, + }, + { + name: "is not service url", + serviceURL: "https://zarf.dev", + expected: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + result := IsServiceURL(tt.serviceURL) + require.Equal(t, tt.expected, result) + }) + } +} + +func TestParseServiceURL(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + serviceURL string + expectedErr string + expectedNamespace string + expectedName string + expectedPort int + }{ + { + name: "correct service url", + serviceURL: "http://foo.bar.svc.cluster.local:5000", + expectedNamespace: "bar", + expectedName: "foo", + expectedPort: 5000, + }, + { + name: "invalid service url without port", + serviceURL: "http://google.com", + expectedErr: "service url does not have a port", + }, + { + name: "invalid service url with port", + serviceURL: "http://google.com:3000", + expectedErr: "invalid service url http://google.com:3000", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + namespace, name, port, err := ParseServiceURL(tt.serviceURL) + if tt.expectedErr != "" { + require.EqualError(t, err, tt.expectedErr) + return + } + require.Equal(t, tt.expectedNamespace, namespace) + require.Equal(t, tt.expectedName, name) + require.Equal(t, tt.expectedPort, port) + }) + } +} diff --git a/src/internal/packager2/load.go b/src/internal/packager2/load.go new file mode 100644 index 0000000000..0fcea0780e --- /dev/null +++ b/src/internal/packager2/load.go @@ -0,0 +1,346 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package packager2 + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/defenseunicorns/pkg/helpers/v2" + "github.com/defenseunicorns/pkg/oci" + "github.com/mholt/archiver/v3" + + "github.com/zarf-dev/zarf/src/config" + "github.com/zarf-dev/zarf/src/pkg/layout" + "github.com/zarf-dev/zarf/src/pkg/packager/filters" + "github.com/zarf-dev/zarf/src/pkg/packager/sources" + "github.com/zarf-dev/zarf/src/pkg/utils" + "github.com/zarf-dev/zarf/src/pkg/zoci" + "github.com/zarf-dev/zarf/src/types" +) + +// LoadPackageFromSource optionally fetches and loads the package from the given source. +func LoadPackageFromSource(ctx context.Context, src, shasum, publicKeyPath string, filter filters.ComponentFilterStrategy) (*layout.PackagePaths, error) { + srcType, err := identifySource(src) + if err != nil { + return nil, err + } + + packageDir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) + if err != nil { + return nil, err + } + + // OCI loads differently as it can fetch partial packages. + if srcType == "oci" { + pkgPaths, err := fetchOCI(ctx, src, shasum, publicKeyPath, packageDir, filter) + if err != nil { + return nil, err + } + return pkgPaths, nil + } + + tarDir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) + if err != nil { + return nil, err + } + defer os.RemoveAll(tarDir) + tarPath := filepath.Join(tarDir, "packager.tar") + + switch srcType { + case "tarball": + tarPath = src + case "sget": + return nil, fmt.Errorf("deprecated") + case "http", "https": + err := fetchHTTP(ctx, src, tarPath) + if err != nil { + return nil, err + } + case "split": + err := assembleSplitTar(src, tarPath) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unknown source type: %s", src) + } + + if shasum != "" { + err := helpers.SHAsMatch(tarPath, shasum) + if err != nil { + return nil, err + } + } + pathsExtracted := []string{} + err = archiver.Walk(src, func(f archiver.File) error { + if f.IsDir() { + return nil + } + header, ok := f.Header.(*tar.Header) + if !ok { + return fmt.Errorf("expected header to be *tar.Header but was %T", f.Header) + } + // If path has nested directories we want to create them. + dir := filepath.Dir(header.Name) + if dir != "." { + err := os.MkdirAll(filepath.Join(packageDir, dir), helpers.ReadExecuteAllWriteUser) + if err != nil { + return err + } + } + dst, err := os.Create(filepath.Join(packageDir, header.Name)) + if err != nil { + return err + } + defer dst.Close() + _, err = io.Copy(dst, f) + if err != nil { + return err + } + pathsExtracted = append(pathsExtracted, header.Name) + return nil + }) + if err != nil { + return nil, err + } + + // Load the package paths + pkgPaths := layout.New(packageDir) + pkgPaths.SetFromPaths(pathsExtracted) + pkg, _, err := pkgPaths.ReadZarfYAML() + if err != nil { + return nil, err + } + pkg.Components, err = filter.Apply(pkg) + if err != nil { + return nil, err + } + if err := pkgPaths.MigrateLegacy(); err != nil { + return nil, err + } + if !pkgPaths.IsLegacyLayout() { + if err := sources.ValidatePackageIntegrity(pkgPaths, pkg.Metadata.AggregateChecksum, false); err != nil { + return nil, err + } + if err := sources.ValidatePackageSignature(ctx, pkgPaths, publicKeyPath); err != nil { + return nil, err + } + } + for _, component := range pkg.Components { + if err := pkgPaths.Components.Unarchive(component); err != nil { + if errors.Is(err, layout.ErrNotLoaded) { + _, err := pkgPaths.Components.Create(component) + if err != nil { + return nil, err + } + } else { + return nil, err + } + } + } + if pkgPaths.SBOMs.Path != "" { + if err := pkgPaths.SBOMs.Unarchive(); err != nil { + return nil, err + } + } + return pkgPaths, nil +} + +func identifySource(src string) (string, error) { + parsed, err := url.Parse(src) + if err == nil && parsed.Scheme != "" && parsed.Host != "" { + return parsed.Scheme, nil + } + if strings.HasSuffix(src, ".tar.zst") || strings.HasSuffix(src, ".tar") { + return "tarball", nil + } + if strings.Contains(src, ".part000") { + return "split", nil + } + return "", fmt.Errorf("unknown source %s", src) +} + +func assembleSplitTar(src, tarPath string) error { + pattern := strings.Replace(src, ".part000", ".part*", 1) + splitFiles, err := filepath.Glob(pattern) + if err != nil { + return fmt.Errorf("unable to find split tarball files: %w", err) + } + // Ensure the files are in order so they are appended in the correct order + sort.Strings(splitFiles) + + tarFile, err := os.Create(tarPath) + if err != nil { + return err + } + defer tarFile.Close() + + var pkgData types.ZarfSplitPackageData + for _, splitFile := range splitFiles { + f, err := os.Open(splitFile) + if err != nil { + return fmt.Errorf("unable to open file %s: %w", splitFile, err) + } + defer f.Close() + _, err = io.Copy(tarFile, f) + if err != nil { + return fmt.Errorf("unable to copy file %s: %w", splitFile, err) + } + err = f.Close() + if err != nil { + return fmt.Errorf("unable to close file %s: %w", splitFile, err) + } + } + if err := helpers.SHAsMatch(tarPath, pkgData.Sha256Sum); err != nil { + return fmt.Errorf("package integrity check failed: %w", err) + } + return nil +} + +func fetchHTTP(ctx context.Context, src, tarPath string) error { + src, checksum, err := parseChecksum(src) + if err != nil { + return err + } + f, err := os.Create(tarPath) + if err != nil { + return err + } + defer f.Close() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, src, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + _, err := io.Copy(io.Discard, resp.Body) + if err != nil { + return err + } + return fmt.Errorf("unexpected http response status code %s for source %s", resp.Status, src) + } + _, err = io.Copy(f, resp.Body) + if err != nil { + return err + } + // Check checksum if src inccluded one. + if checksum != "" { + received, err := helpers.GetSHA256OfFile(tarPath) + if err != nil { + return err + } + if received != checksum { + return fmt.Errorf("shasum mismatch for file %s, expected %s bu got %s ", tarPath, checksum, received) + } + } + return nil +} + +func parseChecksum(src string) (string, string, error) { + atSymbolCount := strings.Count(src, "@") + var checksum string + if atSymbolCount > 0 { + parsed, err := url.Parse(src) + if err != nil { + return src, checksum, fmt.Errorf("unable to parse the URL: %s", src) + } + if atSymbolCount == 1 && parsed.User != nil { + return src, checksum, nil + } + + index := strings.LastIndex(src, "@") + checksum = src[index+1:] + src = src[:index] + } + return src, checksum, nil +} + +func fetchOCI(ctx context.Context, src, shasum, publicKeyPath, packageDir string, filter filters.ComponentFilterStrategy) (*layout.PackagePaths, error) { + if shasum != "" { + src = fmt.Sprintf("%s@sha256:%s", src, shasum) + } + arch := config.GetArch() + remote, err := zoci.NewRemote(src, oci.PlatformForArch(arch)) + if err != nil { + return nil, err + } + + pkg, err := remote.FetchZarfYAML(ctx) + if err != nil { + return nil, err + } + pkg.Components, err = filter.Apply(pkg) + if err != nil { + return nil, err + } + + layersToPull, err := remote.LayersFromRequestedComponents(ctx, pkg.Components) + if err != nil { + return nil, fmt.Errorf("unable to get published component image layers: %s", err.Error()) + } + + isPartial := true + root, err := remote.FetchRoot(ctx) + if err != nil { + return nil, err + } + if len(root.Layers) == len(layersToPull) { + isPartial = false + } + + layersFetched, err := remote.PullPackage(ctx, packageDir, config.CommonOptions.OCIConcurrency, layersToPull...) + if err != nil { + return nil, fmt.Errorf("unable to pull the package: %w", err) + } + + pkgPaths := layout.New(packageDir) + pkgPaths.SetFromLayers(layersFetched) + + if err := pkgPaths.MigrateLegacy(); err != nil { + return nil, err + } + + if !pkgPaths.IsLegacyLayout() { + if err := sources.ValidatePackageIntegrity(pkgPaths, pkg.Metadata.AggregateChecksum, isPartial); err != nil { + return nil, err + } + if err := sources.ValidatePackageSignature(ctx, pkgPaths, publicKeyPath); err != nil { + return nil, err + } + } + + for _, component := range pkg.Components { + if err := pkgPaths.Components.Unarchive(component); err != nil { + if errors.Is(err, layout.ErrNotLoaded) { + _, err := pkgPaths.Components.Create(component) + if err != nil { + return nil, err + } + } else { + return nil, err + } + } + } + + if pkgPaths.SBOMs.Path != "" { + if err := pkgPaths.SBOMs.Unarchive(); err != nil { + return nil, err + } + } + return pkgPaths, nil +} diff --git a/src/internal/packager2/load_test.go b/src/internal/packager2/load_test.go new file mode 100644 index 0000000000..93ad6b23cd --- /dev/null +++ b/src/internal/packager2/load_test.go @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package packager2 + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIdentifySource(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + src string + expectedSrcType string + }{ + { + name: "oci", + src: "oci://ghcr.io/defenseunicorns/packages/init:1.0.0", + expectedSrcType: "oci", + }, + { + name: "sget with sub path", + src: "sget://github.com/defenseunicorns/zarf-hello-world:x86", + expectedSrcType: "sget", + }, + { + name: "sget without host", + src: "sget://defenseunicorns/zarf-hello-world:x86_64", + expectedSrcType: "sget", + }, + { + name: "https", + src: "https://github.com/zarf-dev/zarf/releases/download/v1.0.0/zarf-init-amd64-v1.0.0.tar.zst", + expectedSrcType: "https", + }, + { + name: "http", + src: "http://github.com/zarf-dev/zarf/releases/download/v1.0.0/zarf-init-amd64-v1.0.0.tar.zst", + expectedSrcType: "http", + }, + { + name: "local tar init zst", + src: "zarf-init-amd64-v1.0.0.tar.zst", + expectedSrcType: "tarball", + }, + { + name: "local tar", + src: "zarf-package-manifests-amd64-v1.0.0.tar", + expectedSrcType: "tarball", + }, + { + name: "local tar manifest zst", + src: "zarf-package-manifests-amd64-v1.0.0.tar.zst", + expectedSrcType: "tarball", + }, + { + name: "local tar split", + src: "testdata/.part000", + expectedSrcType: "split", + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + srcType, err := identifySource(tt.src) + require.NoError(t, err) + require.Equal(t, tt.expectedSrcType, srcType) + }) + } +} diff --git a/src/internal/packager2/mirror.go b/src/internal/packager2/mirror.go new file mode 100644 index 0000000000..e24123a187 --- /dev/null +++ b/src/internal/packager2/mirror.go @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package packager2 + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/avast/retry-go/v4" + "github.com/defenseunicorns/pkg/helpers/v2" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/logs" + v1 "github.com/google/go-containerregistry/pkg/v1" + + "github.com/zarf-dev/zarf/src/config" + "github.com/zarf-dev/zarf/src/internal/dns" + "github.com/zarf-dev/zarf/src/internal/git" + "github.com/zarf-dev/zarf/src/internal/gitea" + "github.com/zarf-dev/zarf/src/pkg/cluster" + "github.com/zarf-dev/zarf/src/pkg/layout" + "github.com/zarf-dev/zarf/src/pkg/message" + "github.com/zarf-dev/zarf/src/pkg/transform" + "github.com/zarf-dev/zarf/src/pkg/utils" + "github.com/zarf-dev/zarf/src/types" +) + +// Mirror mirrors the package contents to the give registry and git server. +func Mirror(ctx context.Context, c *cluster.Cluster, pkgPaths layout.PackagePaths, regInfo types.RegistryInfo, gitInfo types.GitServerInfo, noImgChecksum bool, retries int) error { + err := pushImagesToRegistry(ctx, c, pkgPaths, regInfo, noImgChecksum, retries) + if err != nil { + return err + } + err = pushReposToRepository(ctx, c, pkgPaths, gitInfo, retries) + if err != nil { + return err + } + return nil +} + +func pushImagesToRegistry(ctx context.Context, c *cluster.Cluster, pkgPaths layout.PackagePaths, regInfo types.RegistryInfo, noImgChecksum bool, retries int) error { + logs.Warn.SetOutput(&message.DebugWriter{}) + logs.Progress.SetOutput(&message.DebugWriter{}) + + pkg, _, err := pkgPaths.ReadZarfYAML() + if err != nil { + return err + } + + images := map[transform.Image]v1.Image{} + for _, component := range pkg.Components { + for _, img := range component.Images { + ref, err := transform.ParseImageRef(img) + if err != nil { + return fmt.Errorf("failed to create ref for image %s: %w", img, err) + } + if _, ok := images[ref]; ok { + continue + } + ociImage, err := utils.LoadOCIImage(pkgPaths.Images.Base, ref) + if err != nil { + return err + } + images[ref] = ociImage + } + } + if len(images) == 0 { + return nil + } + + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig.InsecureSkipVerify = config.CommonOptions.Insecure + // TODO (@WSTARR) This is set to match the TLSHandshakeTimeout to potentially mitigate effects of https://github.com/zarf-dev/zarf/issues/1444 + transport.ResponseHeaderTimeout = 10 * time.Second + transportWithProgressBar := helpers.NewTransport(transport, nil) + + pushOptions := []crane.Option{ + crane.WithPlatform(&v1.Platform{OS: "linux", Architecture: pkg.Build.Architecture}), + crane.WithTransport(transportWithProgressBar), + crane.WithAuth(authn.FromConfig(authn.AuthConfig{ + Username: regInfo.PushUsername, + Password: regInfo.PushPassword, + })), + crane.WithUserAgent("zarf"), + crane.WithNoClobber(true), + crane.WithJobs(1), + } + if config.CommonOptions.Insecure { + pushOptions = append(pushOptions, crane.Insecure) + } + + for refInfo, img := range images { + err = retry.Do(func() error { + pushImage := func(registryUrl string) error { + names := []string{} + if !noImgChecksum { + offlineNameCRC, err := transform.ImageTransformHost(registryUrl, refInfo.Reference) + if err != nil { + return retry.Unrecoverable(err) + } + names = append(names, offlineNameCRC) + } + offlineName, err := transform.ImageTransformHostWithoutChecksum(registryUrl, refInfo.Reference) + if err != nil { + return retry.Unrecoverable(err) + } + names = append(names, offlineName) + for _, name := range names { + err = crane.Push(img, name, pushOptions...) + if err != nil { + return err + } + } + return nil + } + + if c == nil { + return pushImage(regInfo.Address) + } + + namespace, name, port, err := dns.ParseServiceURL(regInfo.Address) + if err != nil { + return err + } + tunnel, err := c.NewTunnel(namespace, cluster.SvcResource, name, "", 0, port) + if err != nil { + return err + } + _, err = tunnel.Connect(ctx) + if err != nil { + return err + } + defer tunnel.Close() + err = tunnel.Wrap(func() error { + return pushImage(tunnel.Endpoint()) + }) + if err != nil { + return err + } + return nil + }, retry.Context(ctx), retry.Attempts(uint(retries)), retry.Delay(500*time.Millisecond)) + if err != nil { + return err + } + } + return nil +} + +func pushReposToRepository(ctx context.Context, c *cluster.Cluster, pkgPaths layout.PackagePaths, gitInfo types.GitServerInfo, retries int) error { + pkg, _, err := pkgPaths.ReadZarfYAML() + if err != nil { + return err + } + for _, component := range pkg.Components { + for _, repoURL := range component.Repos { + repository, err := git.Open(pkgPaths.Components.Dirs[component.Name].Repos, repoURL) + if err != nil { + return err + } + err = retry.Do(func() error { + if c == nil { + err = repository.Push(ctx, gitInfo.Address, gitInfo.PushUsername, gitInfo.PushPassword) + if err != nil { + return err + } + return nil + } + + namespace, name, port, err := dns.ParseServiceURL(gitInfo.Address) + if err != nil { + return retry.Unrecoverable(err) + } + tunnel, err := c.NewTunnel(namespace, cluster.SvcResource, name, "", 0, port) + if err != nil { + return err + } + _, err = tunnel.Connect(ctx) + if err != nil { + return err + } + defer tunnel.Close() + giteaClient, err := gitea.NewClient(tunnel.HTTPEndpoint(), gitInfo.PushUsername, gitInfo.PushPassword) + if err != nil { + return err + } + return tunnel.Wrap(func() error { + err = repository.Push(ctx, tunnel.HTTPEndpoint(), gitInfo.PushUsername, gitInfo.PushPassword) + if err != nil { + return err + } + // Add the read-only user to this repo + // TODO: This should not be done here. Or the function name should be changed. + repoName, err := transform.GitURLtoRepoName(repoURL) + if err != nil { + return retry.Unrecoverable(err) + } + err = giteaClient.AddReadOnlyUserToRepository(ctx, repoName, gitInfo.PullUsername) + if err != nil { + return fmt.Errorf("unable to add the read only user to the repo %s: %w", repoName, err) + } + return nil + }) + }, retry.Context(ctx), retry.Attempts(uint(retries)), retry.Delay(500*time.Millisecond)) + if err != nil { + return fmt.Errorf("unable to push repo %s to the Git Server: %w", repoURL, err) + } + } + } + return nil +} diff --git a/src/internal/packager2/packager.go b/src/internal/packager2/packager.go new file mode 100644 index 0000000000..b0e8dc79a0 --- /dev/null +++ b/src/internal/packager2/packager.go @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager2 is the new implementation for packager. +package packager2 diff --git a/src/test/external/ext_in_cluster_test.go b/src/test/external/ext_in_cluster_test.go index ffe5a08b73..deb79a33b4 100644 --- a/src/test/external/ext_in_cluster_test.go +++ b/src/test/external/ext_in_cluster_test.go @@ -30,7 +30,8 @@ var inClusterCredentialArgs = []string{ "--git-url=http://gitea-http.git-server.svc.cluster.local:3000", "--registry-push-username=push-user", "--registry-push-password=superSecurePassword", - "--registry-url=127.0.0.1:31999"} + "--registry-url=http://external-registry-docker-registry.external-registry.svc.cluster.local:5000", +} type ExtInClusterTestSuite struct { suite.Suite