zarf package mirror command introduction

Makefile

@@ -175,6 +175,7 @@ test-external: ## Run the Zarf CLI E2E tests for an external registry and cluste
 	@test -s $(ZARF_BIN) || $(MAKE) build-cli
 	@test -s ./build/zarf-init-$(ARCH)-$(CLI_VERSION).tar.zst || $(MAKE) init-package
 	@test -s ./build/zarf-package-podinfo-flux-$(ARCH).tar.zst || $(ZARF_BIN) package create examples/podinfo-flux -o build -a $(ARCH) --confirm
+	@test -s ./build/zarf-package-argocd-$(ARCH).tar.zst || $(ZARF_BIN) package create examples/argocd -o build -a $(ARCH) --confirm
 	cd src/test/external && go test -failfast -v -timeout 30m
 
 ## NOTE: Requires an existing cluster and

docs/2-the-zarf-cli/100-cli-commands/zarf_package.md

@@ -30,6 +30,7 @@ Zarf package commands for creating, deploying, and inspecting packages
 * [zarf package deploy](zarf_package_deploy.md)	 - Deploys a Zarf package from a local file or URL (runs offline)
 * [zarf package inspect](zarf_package_inspect.md)	 - Displays the definition of a Zarf package (runs offline)
 * [zarf package list](zarf_package_list.md)	 - Lists out all of the packages that have been deployed to the cluster (runs offline)
+* [zarf package mirror-resources](zarf_package_mirror-resources.md)	 - Mirrors a Zarf package's internal resources to specified image registries and git repositories
 * [zarf package publish](zarf_package_publish.md)	 - Publishes a Zarf package to a remote registry
 * [zarf package pull](zarf_package_pull.md)	 - Pulls a Zarf package from a remote registry and save to the local file system
 * [zarf package remove](zarf_package_remove.md)	 - Removes a Zarf package that has been deployed already (runs offline)

docs/2-the-zarf-cli/100-cli-commands/zarf_package_mirror-resources.md

@@ -0,0 +1,46 @@
+# zarf package mirror-resources
+<!-- Auto-generated by hack/gen-cli-docs.sh -->
+
+Mirrors a Zarf package's internal resources to specified image registries and git repositories
+
+## Synopsis
+
+Unpacks resources and dependencies from a Zarf package archive and mirrors them into the specified 
+image registries and git repositories within the target environment
+
+```
+zarf package mirror-resources [ PACKAGE ] [flags]
+```
+
+## Options
+
+```
+      --components string               Comma-separated list of components to mirror.  This list will be respected regardless of a component's 'required' status.
+      --confirm                         Confirms package deployment without prompting. ONLY use with packages you trust. Skips prompts to review SBOM, configure variables, select optional components and review potential breaking changes.
+      --git-push-password string        Password for the push-user to access the git server
+      --git-push-username string        Username to access to the git server Zarf is configured to use. User must be able to create repositories via 'git push' (default "zarf-git-user")
+      --git-url string                  External git server url to use for this Zarf cluster
+  -h, --help                            help for mirror-resources
+      --no-img-checksum                 Turns off the addition of a checksum to image tags (as would be used by the Zarf Agent) while mirroring images.
+      --registry-push-password string   Password for the push-user to connect to the registry
+      --registry-push-username string   Username to access to the registry Zarf is configured to use (default "zarf-push")
+      --registry-url string             External registry url address to use for this Zarf cluster
+```
+
+## Options inherited from parent commands
+
+```
+  -a, --architecture string   Architecture for OCI images and Zarf packages
+      --insecure              Allow access to insecure registries and disable other recommended security enforcements such as package checksum and signature validation. This flag should only be used if you have a specific reason and accept the reduced security posture.
+  -l, --log-level string      Log level when running Zarf. Valid options are: warn, info, debug, trace (default "info")
+      --no-color              Disable colors in output
+      --no-log-file           Disable log file creation
+      --no-progress           Disable fancy UI progress bars, spinners, logos, etc
+      --oci-concurrency int   Number of concurrent layer operations to perform when interacting with a remote package. (default 3)
+      --tmpdir string         Specify the temporary directory to use for intermediate files
+      --zarf-cache string     Specify the location of the Zarf cache directory (default "~/.zarf-cache")
+```
+
+## SEE ALSO
+
+* [zarf package](zarf_package.md)	 - Zarf package commands for creating, deploying, and inspecting packages

docs/5-zarf-tutorials/2-deploying-zarf-packages.md

@@ -52,6 +52,12 @@ To accept a default value for a given variable, simply press the `enter` key.  Y
 
 ![Zarf Tools Monitor](../.images/tutorials/zarf_tools_monitor.png)
 
+:::tip
+
+Deploying packages isn't the only way to interact with them in the air gap.  If you would like to quickly inspect a package and it's SBOMs you can use [`zarf package inspect`](../4-deploy-a-zarf-package/4-view-sboms.md) to view them, and if you would like to push resources inside of a Zarf package (i.e. the images in this Wordpress package) to services in the air gap without running a deployment, you can do so with [`zarf package mirror-resources`](../2-the-zarf-cli/100-cli-commands/zarf_package_mirror-resources.md).
+
+:::
+
 ## Removal
 
 1. Use the `zarf package list` command to get a list of the installed packages.  This will give you the name of the WordPress package to remove it.

examples/composable-packages/README.md

@@ -18,6 +18,8 @@ You can create a skeleton package from a `zarf.yaml` by pointing `zarf package p
 zarf package publish path/containing/package/definition oci://your-registry.com
 ```
 
+:::
+
 ## Merge Strategies
 
 When merging components together Zarf will adopt the following strategies depending on the kind of primitive (`files`, `required`, `manifests`) that it is merging:
@@ -30,8 +32,6 @@ When merging components together Zarf will adopt the following strategies depend
 | Un'name'd Primitive Arrays | `actions`, `dataInjections`, `files`, `images`, `repos` | These keys will append the overriding component's version of the array to the end of the base component's array |
 | 'name'd Primitive Arrays   | `charts`, `manifests` | For any given element in the overriding component, if the element matches based on `name` then its values will be merged with the base element of the same `name`. If not then the element will be appended to the end of the array |
 
-:::
-
 ## `zarf.yaml` {#zarf.yaml}
 
 :::info

examples/dos-games/README.md

@@ -6,7 +6,7 @@ This example provides the basis for Zarf's [Retro Arcade](../../docs/5-zarf-tuto
 
 :::note
 
-In this example, there is no "GitOps" service. Zarf is only showing off its ability to act as a standard means of packaging, distribution, and runtime.
+In this example, there is no requirement for a "GitOps" service; Zarf is only showing off its ability to act as a standard means of packaging, distribution, and deployment runtime.
 
 :::
 

examples/git-data/README.md

@@ -4,6 +4,12 @@ import ExampleYAML from "@site/src/components/ExampleYAML";
 
 This example shows how to package `git` repositories within a Zarf package.  This package does not deploy anything itself but pushes assets to the specified `git` service to be consumed as desired.  Within Zarf, there are a few ways to include `git` repositories (as described below).
 
+:::tip
+
+Git repositories included in a package can be deployed with `zarf package deploy` if an existing Kubernetes cluster has been initialized with `zarf init`.  If you do not have an initialized cluster but want to push resources to a remote registry anyway, you can use [`zarf package mirror-resources`](./../../docs/2-the-zarf-cli/100-cli-commands/zarf_package_mirror-resources.md).
+
+:::
+
 ## Tag-Based Git Repository Clone
 
 Tag-based `git` repository cloning is the **recommended** way of cloning a `git` repository for air-gapped deployments because it wraps meaning around a specific point in git history that can easily be traced back to the online world. Tag-based clones are defined using the `scheme://host/repo@tag` format as seen in the example of the `defenseunicorns/zarf` repository (`https://github.com/defenseunicorns/zarf.git@v0.15.0`).

src/cmd/package.go

@@ -102,6 +102,28 @@ var packageDeployCmd = &cobra.Command{
 	},
 }
 
+var packageMirrorCmd = &cobra.Command{
+	Use:     "mirror-resources [ PACKAGE ]",
+	Aliases: []string{"mr"},
+	Short:   lang.CmdPackageMirrorShort,
+	Long:    lang.CmdPackageMirrorLong,
+	Args:    cobra.MaximumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		pkgConfig.PkgOpts.PackagePath = choosePackage(args)
+
+		pkgConfig.PkgSource = pkgConfig.PkgOpts.PackagePath
+
+		// Configure the packager
+		pkgClient := packager.NewOrDie(&pkgConfig)
+		defer pkgClient.ClearTempPaths()
+
+		// Deploy the package
+		if err := pkgClient.Mirror(); err != nil {
+			message.Fatalf(err, lang.CmdPackageDeployErr, err.Error())
+		}
+	},
+}
+
 var packageInspectCmd = &cobra.Command{
 	Use:     "inspect [ PACKAGE ]",
 	Aliases: []string{"i"},
@@ -268,6 +290,7 @@ func init() {
 	rootCmd.AddCommand(packageCmd)
 	packageCmd.AddCommand(packageCreateCmd)
 	packageCmd.AddCommand(packageDeployCmd)
+	packageCmd.AddCommand(packageMirrorCmd)
 	packageCmd.AddCommand(packageInspectCmd)
 	packageCmd.AddCommand(packageRemoveCmd)
 	packageCmd.AddCommand(packageListCmd)
@@ -277,6 +300,7 @@ func init() {
 	bindPackageFlags(v)
 	bindCreateFlags(v)
 	bindDeployFlags(v)
+	bindMirrorFlags(v)
 	bindInspectFlags(v)
 	bindRemoveFlags(v)
 	bindPublishFlags(v)
@@ -334,6 +358,27 @@ func bindDeployFlags(v *viper.Viper) {
 	deployFlags.MarkHidden("sget")
 }
 
+func bindMirrorFlags(v *viper.Viper) {
+	mirrorFlags := packageMirrorCmd.Flags()
+
+	// Always require confirm flag (no viper)
+	mirrorFlags.BoolVar(&config.CommonOptions.Confirm, "confirm", false, lang.CmdPackageDeployFlagConfirm)
+
+	mirrorFlags.BoolVar(&pkgConfig.MirrorOpts.NoImgChecksum, "no-img-checksum", false, lang.CmdPackageMirrorFlagNoChecksum)
+
+	mirrorFlags.StringVar(&pkgConfig.PkgOpts.OptionalComponents, "components", v.GetString(common.VPkgDeployComponents), lang.CmdPackageMirrorFlagComponents)
+
+	// Flags for using an external Git server
+	mirrorFlags.StringVar(&pkgConfig.InitOpts.GitServer.Address, "git-url", v.GetString(common.VInitGitURL), lang.CmdInitFlagGitURL)
+	mirrorFlags.StringVar(&pkgConfig.InitOpts.GitServer.PushUsername, "git-push-username", v.GetString(common.VInitGitPushUser), lang.CmdInitFlagGitPushUser)
+	mirrorFlags.StringVar(&pkgConfig.InitOpts.GitServer.PushPassword, "git-push-password", v.GetString(common.VInitGitPushPass), lang.CmdInitFlagGitPushPass)
+
+	// Flags for using an external registry
+	mirrorFlags.StringVar(&pkgConfig.InitOpts.RegistryInfo.Address, "registry-url", v.GetString(common.VInitRegistryURL), lang.CmdInitFlagRegURL)
+	mirrorFlags.StringVar(&pkgConfig.InitOpts.RegistryInfo.PushUsername, "registry-push-username", v.GetString(common.VInitRegistryPushUser), lang.CmdInitFlagRegPushUser)
+	mirrorFlags.StringVar(&pkgConfig.InitOpts.RegistryInfo.PushPassword, "registry-push-password", v.GetString(common.VInitRegistryPushPass), lang.CmdInitFlagRegPushPass)
+}
+
 func bindInspectFlags(v *viper.Viper) {
 	inspectFlags := packageInspectCmd.Flags()
 	inspectFlags.BoolVarP(&includeInspectSBOM, "sbom", "s", false, lang.CmdPackageInspectFlagSbom)

src/cmd/prepare.go

@@ -148,8 +148,6 @@ func init() {
 	prepareCmd.AddCommand(prepareFindImages)
 	prepareCmd.AddCommand(prepareGenerateConfigFile)
 
-	v.SetDefault(common.VPkgCreateSet, map[string]string{})
-
 	prepareFindImages.Flags().StringVarP(&repoHelmChartPath, "repo-chart-path", "p", "", lang.CmdPrepareFlagRepoChartPath)
 	// use the package create config for this and reset it here to avoid overwriting the config.CreateOptions.SetVariables
 	prepareFindImages.Flags().StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdPrepareFlagSet)

src/config/config.go

@@ -52,6 +52,7 @@ const (
 
 	ZarfDeployStage = "Deploy"
 	ZarfCreateStage = "Create"
+	ZarfMirrorStage = "Mirror"
 )
 
 // Zarf Constants for In-Cluster Services.

src/config/lang/english.go

@@ -228,6 +228,10 @@ const (
 	CmdPackageDeployLong  = "Unpacks resources and dependencies from a Zarf package archive and deploys them onto the target system.\n" +
 		"Kubernetes clusters are accessed via credentials in your current kubecontext defined in '~/.kube/config'"
 
+	CmdPackageMirrorShort = "Mirrors a Zarf package's internal resources to specified image registries and git repositories"
+	CmdPackageMirrorLong  = "Unpacks resources and dependencies from a Zarf package archive and mirrors them into the specified \n" +
+		"image registries and git repositories within the target environment"
+
 	CmdPackageInspectShort = "Displays the definition of a Zarf package (runs offline)"
 	CmdPackageInspectLong  = "Displays the 'zarf.yaml' definition for the specified package and optionally allows SBOMs to be viewed"
 
@@ -261,6 +265,9 @@ const (
 	CmdPackageDeployInvalidCLIVersionWarn              = "CLIVersion is set to '%s' which can cause issues with package creation and deployment. To avoid such issues, please set the value to the valid semantic version for this version of Zarf."
 	CmdPackageDeployErr                                = "Failed to deploy package: %s"
 
+	CmdPackageMirrorFlagComponents = "Comma-separated list of components to mirror.  This list will be respected regardless of a component's 'required' status."
+	CmdPackageMirrorFlagNoChecksum = "Turns off the addition of a checksum to image tags (as would be used by the Zarf Agent) while mirroring images."
+
 	CmdPackageInspectFlagSbom      = "View SBOM contents while inspecting the package"
 	CmdPackageInspectFlagSbomOut   = "Specify an output directory for the SBOMs from the inspected Zarf package"
 	CmdPackageInspectFlagValidate  = "Validate any checksums and signatures while inspecting the package"

src/pkg/packager/mirror.go

@@ -0,0 +1,90 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: 2021-Present The Zarf Authors
+
+// Package packager contains functions for interacting with, managing and deploying Zarf packages.
+package packager
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/defenseunicorns/zarf/src/config"
+	"github.com/defenseunicorns/zarf/src/pkg/message"
+	"github.com/defenseunicorns/zarf/src/pkg/utils/helpers"
+	"github.com/defenseunicorns/zarf/src/types"
+)
+
+// Mirror pulls resources from a package (images, git repositories, etc) and pushes them to remotes in the air gap without deploying them
+func (p *Packager) Mirror() (err error) {
+	spinner := message.NewProgressSpinner("Mirroring Zarf package %s", p.cfg.PkgOpts.PackagePath)
+	defer spinner.Stop()
+
+	if helpers.IsOCIURL(p.cfg.PkgOpts.PackagePath) {
+		err := p.SetOCIRemote(p.cfg.PkgOpts.PackagePath)
+		if err != nil {
+			return err
+		}
+	}
+
+	if err := p.loadZarfPkg(); err != nil {
+		return fmt.Errorf("unable to load the Zarf Package: %w", err)
+	}
+
+	if err := ValidatePackageSignature(p.tmp.Base, p.cfg.PkgOpts.PublicKeyPath); err != nil {
+		return err
+	}
+
+	// Confirm the overall package mirror
+	if !p.confirmAction(config.ZarfMirrorStage, p.cfg.SBOMViewFiles) {
+		return fmt.Errorf("mirror cancelled")
+	}
+
+	state := &types.ZarfState{
+		RegistryInfo: p.cfg.InitOpts.RegistryInfo,
+		GitServer:    p.cfg.InitOpts.GitServer,
+	}
+	p.cfg.State = state
+
+	// Filter out components that are not compatible with this system if we have loaded from a tarball
+	p.filterComponents(true)
+	requestedComponentNames := getRequestedComponentList(p.cfg.PkgOpts.OptionalComponents)
+
+	for _, component := range p.cfg.Pkg.Components {
+		if len(requestedComponentNames) == 0 || helpers.SliceContains(requestedComponentNames, component.Name) {
+			if err := p.mirrorComponent(component); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// mirrorComponent mirrors a Zarf Component.
+func (p *Packager) mirrorComponent(component types.ZarfComponent) error {
+
+	componentPath, err := p.createOrGetComponentPaths(component)
+	if err != nil {
+		return fmt.Errorf("unable to create the component paths: %w", err)
+	}
+
+	// All components now require a name
+	message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name))
+
+	hasImages := len(component.Images) > 0
+	hasRepos := len(component.Repos) > 0
+
+	if hasImages {
+		if err := p.pushImagesToRegistry(component.Images, p.cfg.MirrorOpts.NoImgChecksum); err != nil {
+			return fmt.Errorf("unable to push images to the registry: %w", err)
+		}
+	}
+
+	if hasRepos {
+		if err = p.pushReposToRepository(componentPath.Repos, component.Repos); err != nil {
+			return fmt.Errorf("unable to push the repos to the repository: %w", err)
+		}
+	}
+
+	return nil
+}

src/test/e2e/22_git_and_gitops_test.go

@@ -124,6 +124,7 @@ func waitFluxPodInfoDeployment(t *testing.T) {
 
 	// Tests the URL mutation for GitRepository CRD for Flux.
 	stdOut, stdErr, err = e2e.Kubectl("get", "gitrepositories", "podinfo", "-n", "flux-system", "-o", "jsonpath={.spec.url}")
+	require.NoError(t, err, stdOut, stdErr)
 	expectedMutatedRepoURL := fmt.Sprintf("%s/%s/podinfo-1646971829.git", config.ZarfInClusterGitServiceURL, config.ZarfGitPushUser)
 	require.Equal(t, expectedMutatedRepoURL, stdOut)
 

src/test/external/common.go

@@ -1,18 +1,22 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors
 
-// Package external provides a test for the external init flow.
+// Package external provides a test for interacting with external resources
 package external
 
 import (
 	"context"
+	"path"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/defenseunicorns/zarf/src/pkg/utils/exec"
+	"github.com/defenseunicorns/zarf/src/test"
 )
 
+var zarfBinPath = path.Join("../../../build", test.GetCLIName())
+
 func verifyKubectlWaitSuccess(t *testing.T, timeoutMinutes time.Duration, args []string, onTimeout string) bool {
 	return verifyWaitSuccess(t, timeoutMinutes, "kubectl", args, "condition met", onTimeout)
 }