diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_package.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_package.md index 205e46b363..1174df8caf 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_package.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_package.md @@ -6,7 +6,8 @@ Zarf package commands for creating, deploying, and inspecting packages ## Options ``` - -h, --help help for package + -h, --help help for package + --oci-concurrency int Number of concurrent layer operations to perform when interacting with a remote package. (default 3) ``` ## Options inherited from parent commands diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_create.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_create.md index 6e24d20178..3e80b73c34 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_create.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_create.md @@ -22,7 +22,7 @@ zarf package create [DIRECTORY] [flags] -k, --key string Path to private key file for signing packages --key-pass string Password to the private key file used for signing packages -m, --max-package-size int Specify the maximum size of the package in megabytes, packages larger than this will be split into multiple parts. Use 0 to disable splitting. - -o, --output-directory string Specify the output directory for the created Zarf package + -o, --output string Specify the output (either a directory or an oci:// URL) for the created Zarf package --registry-override stringToString Specify a map of domains to override on package create when pulling images (e.g. --registry-override docker.io=dockerio-reg.enterprise.intranet) (default []) -s, --sbom View SBOM contents after creating the package --sbom-out string Specify an output directory for the SBOMs from the created Zarf package @@ -38,6 +38,7 @@ zarf package create [DIRECTORY] [flags] -l, --log-level string Log level when running Zarf. Valid options are: warn, info, debug, trace (default "info") --no-log-file Disable log file creation --no-progress Disable fancy UI progress bars, spinners, logos, etc + --oci-concurrency int Number of concurrent layer operations to perform when interacting with a remote package. (default 3) --tmpdir string Specify the temporary directory to use for intermediate files --zarf-cache string Specify the location of the Zarf cache directory (default "~/.zarf-cache") ``` diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_deploy.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_deploy.md index 5763caf25b..502cc97aae 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_deploy.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_deploy.md @@ -19,7 +19,6 @@ zarf package deploy [PACKAGE] [flags] --confirm Confirms package deployment without prompting. ONLY use with packages you trust. Skips prompts to review SBOM, configure variables, select optional components and review potential breaking changes. -h, --help help for deploy -k, --key string Path to public key file for validating signed packages - --oci-concurrency int Number of concurrent layer operations to perform when interacting with a remote package. (default 3) --set stringToString Specify deployment variables to set on the command line (KEY=value) (default []) --sget string Path to public sget key file for remote packages signed via cosign --shasum string Shasum of the package to deploy. Required if deploying a remote package and "--insecure" is not provided @@ -33,6 +32,7 @@ zarf package deploy [PACKAGE] [flags] -l, --log-level string Log level when running Zarf. Valid options are: warn, info, debug, trace (default "info") --no-log-file Disable log file creation --no-progress Disable fancy UI progress bars, spinners, logos, etc + --oci-concurrency int Number of concurrent layer operations to perform when interacting with a remote package. (default 3) --tmpdir string Specify the temporary directory to use for intermediate files --zarf-cache string Specify the location of the Zarf cache directory (default "~/.zarf-cache") ``` diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_inspect.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_inspect.md index 6809be554f..4a5d0e20f5 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_inspect.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_inspect.md @@ -29,6 +29,7 @@ zarf package inspect [PACKAGE] [flags] -l, --log-level string Log level when running Zarf. Valid options are: warn, info, debug, trace (default "info") --no-log-file Disable log file creation --no-progress Disable fancy UI progress bars, spinners, logos, etc + --oci-concurrency int Number of concurrent layer operations to perform when interacting with a remote package. (default 3) --tmpdir string Specify the temporary directory to use for intermediate files --zarf-cache string Specify the location of the Zarf cache directory (default "~/.zarf-cache") ``` diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_list.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_list.md index 857f2ad883..423c8a3891 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_list.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_list.md @@ -21,6 +21,7 @@ zarf package list [flags] -l, --log-level string Log level when running Zarf. Valid options are: warn, info, debug, trace (default "info") --no-log-file Disable log file creation --no-progress Disable fancy UI progress bars, spinners, logos, etc + --oci-concurrency int Number of concurrent layer operations to perform when interacting with a remote package. (default 3) --tmpdir string Specify the temporary directory to use for intermediate files --zarf-cache string Specify the location of the Zarf cache directory (default "~/.zarf-cache") ``` diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_publish.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_publish.md index f2c5698530..143c39084f 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_publish.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_publish.md @@ -22,10 +22,9 @@ zarf package publish ./path/to/dir oci://my-registry.com/my-namespace ## Options ``` - -h, --help help for publish - -k, --key string Path to private key file for signing packages - --key-pass string Password to the private key file used for publishing packages - --oci-concurrency int Number of concurrent layer operations to perform when interacting with a remote package. (default 3) + -h, --help help for publish + -k, --key string Path to private key file for signing packages + --key-pass string Password to the private key file used for publishing packages ``` ## Options inherited from parent commands @@ -36,6 +35,7 @@ zarf package publish ./path/to/dir oci://my-registry.com/my-namespace -l, --log-level string Log level when running Zarf. Valid options are: warn, info, debug, trace (default "info") --no-log-file Disable log file creation --no-progress Disable fancy UI progress bars, spinners, logos, etc + --oci-concurrency int Number of concurrent layer operations to perform when interacting with a remote package. (default 3) --tmpdir string Specify the temporary directory to use for intermediate files --zarf-cache string Specify the location of the Zarf cache directory (default "~/.zarf-cache") ``` diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_pull.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_pull.md index c154db70fc..3007b00429 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_pull.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_pull.md @@ -18,8 +18,7 @@ zarf package pull [REFERENCE] [flags] ``` -h, --help help for pull -k, --key string Path to public key file for validating signed packages - --oci-concurrency int Number of concurrent layer operations to perform when interacting with a remote package. (default 3) - -o, --output-directory string Specify the output directory for the created Zarf package + -o, --output-directory string Specify the output directory for the pulled Zarf package ``` ## Options inherited from parent commands @@ -30,6 +29,7 @@ zarf package pull [REFERENCE] [flags] -l, --log-level string Log level when running Zarf. Valid options are: warn, info, debug, trace (default "info") --no-log-file Disable log file creation --no-progress Disable fancy UI progress bars, spinners, logos, etc + --oci-concurrency int Number of concurrent layer operations to perform when interacting with a remote package. (default 3) --tmpdir string Specify the temporary directory to use for intermediate files --zarf-cache string Specify the location of the Zarf cache directory (default "~/.zarf-cache") ``` diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_remove.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_remove.md index 4b8242d9e0..7e8ec03f7e 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_remove.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_remove.md @@ -23,6 +23,7 @@ zarf package remove {PACKAGE_NAME|PACKAGE_FILE} [flags] -l, --log-level string Log level when running Zarf. Valid options are: warn, info, debug, trace (default "info") --no-log-file Disable log file creation --no-progress Disable fancy UI progress bars, spinners, logos, etc + --oci-concurrency int Number of concurrent layer operations to perform when interacting with a remote package. (default 3) --tmpdir string Specify the temporary directory to use for intermediate files --zarf-cache string Specify the location of the Zarf cache directory (default "~/.zarf-cache") ``` diff --git a/src/cmd/package.go b/src/cmd/package.go index c283719bf3..0c3a119ca5 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -94,8 +94,6 @@ var packageDeployCmd = &cobra.Command{ pkgClient := packager.NewOrDie(&pkgConfig) defer pkgClient.ClearTempPaths() - pterm.Println() - // Deploy the package if err := pkgClient.Deploy(); err != nil { message.Fatalf(err, "Failed to deploy package: %s", err.Error()) @@ -197,7 +195,7 @@ var packageRemoveCmd = &cobra.Command{ defer pkgClient.ClearTempPaths() if err := pkgClient.Remove(pkgName); err != nil { - message.Fatalf(err, "Unable to remove the package with an error of: %#v", err) + message.Fatalf(err, "Unable to remove the package with an error of: %s", err.Error()) } }, } @@ -220,10 +218,16 @@ zarf package publish ./path/to/dir oci://my-registry.com/my-namespace message.Fatalf(nil, "Registry must be prefixed with 'oci://'") } parts := strings.Split(strings.TrimPrefix(args[1], "oci://"), "/") - pkgConfig.PublishOpts.Reference = registry.Reference{ + ref := registry.Reference{ Registry: parts[0], Repository: strings.Join(parts[1:], "/"), } + err := ref.ValidateRegistry() + if err != nil { + message.Fatalf(nil, "%s", err.Error()) + } + + pkgConfig.PublishOpts.PackageDestination = ref.String() // Configure the packager pkgClient := packager.NewOrDie(&pkgConfig) @@ -245,7 +249,8 @@ var packagePullCmd = &cobra.Command{ if !utils.IsOCIURL(args[0]) { message.Fatalf(nil, "Registry must be prefixed with 'oci://'") } - pkgConfig.DeployOpts.PackagePath = choosePackage(args) + + pkgConfig.PullOpts.PackageSource = args[0] // Configure the packager pkgClient := packager.NewOrDie(&pkgConfig) @@ -295,6 +300,7 @@ func init() { packageCmd.AddCommand(packagePublishCmd) packageCmd.AddCommand(packagePullCmd) + bindPackageFlags() bindCreateFlags() bindDeployFlags() bindInspectFlags() @@ -303,6 +309,12 @@ func init() { bindPullFlags() } +func bindPackageFlags() { + packageFlags := packageCmd.PersistentFlags() + v.SetDefault(V_PKG_OCI_CONCURRENCY, 3) + packageFlags.IntVar(&config.CommonOptions.OCIConcurrency, "oci-concurrency", v.GetInt(V_PKG_OCI_CONCURRENCY), lang.CmdPackageFlagConcurrency) +} + func bindCreateFlags() { createFlags := packageCreateCmd.Flags() @@ -310,16 +322,23 @@ func bindCreateFlags() { createFlags.BoolVar(&config.CommonOptions.Confirm, "confirm", false, lang.CmdPackageCreateFlagConfirm) v.SetDefault(V_PKG_CREATE_SET, map[string]string{}) - v.SetDefault(V_PKG_CREATE_OUTPUT_DIR, "") + v.SetDefault(V_PKG_CREATE_OUTPUT, "") v.SetDefault(V_PKG_CREATE_SBOM, false) v.SetDefault(V_PKG_CREATE_SBOM_OUTPUT, "") v.SetDefault(V_PKG_CREATE_SKIP_SBOM, false) v.SetDefault(V_PKG_CREATE_MAX_PACKAGE_SIZE, 0) v.SetDefault(V_PKG_CREATE_SIGNING_KEY, "") + outputDirectory := v.GetString("package.create.output_directory") + output := v.GetString(V_PKG_CREATE_OUTPUT) + if outputDirectory != "" && output == "" { + v.Set(V_PKG_CREATE_OUTPUT, outputDirectory) + } + createFlags.StringVar(&pkgConfig.CreateOpts.Output, "output-directory", v.GetString("package.create.output_directory"), lang.CmdPackageCreateFlagOutput) + createFlags.StringVarP(&pkgConfig.CreateOpts.Output, "output", "o", v.GetString(V_PKG_CREATE_OUTPUT), lang.CmdPackageCreateFlagOutput) + createFlags.StringVar(&pkgConfig.CreateOpts.DifferentialData.DifferentialPackagePath, "differential", v.GetString(V_PKG_CREATE_DIFFERENTIAL), lang.CmdPackageCreateFlagDifferential) createFlags.StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(V_PKG_CREATE_SET), lang.CmdPackageCreateFlagSet) - createFlags.StringVarP(&pkgConfig.CreateOpts.OutputDirectory, "output-directory", "o", v.GetString(V_PKG_CREATE_OUTPUT_DIR), lang.CmdPackageCreateFlagOutputDirectory) createFlags.BoolVarP(&pkgConfig.CreateOpts.ViewSBOM, "sbom", "s", v.GetBool(V_PKG_CREATE_SBOM), lang.CmdPackageCreateFlagSbom) createFlags.StringVar(&pkgConfig.CreateOpts.SBOMOutputDir, "sbom-out", v.GetString(V_PKG_CREATE_SBOM_OUTPUT), lang.CmdPackageCreateFlagSbomOut) createFlags.BoolVar(&pkgConfig.CreateOpts.SkipSBOM, "skip-sbom", v.GetBool(V_PKG_CREATE_SKIP_SBOM), lang.CmdPackageCreateFlagSkipSbom) @@ -327,6 +346,8 @@ func bindCreateFlags() { createFlags.StringVarP(&pkgConfig.CreateOpts.SigningKeyPath, "key", "k", v.GetString(V_PKG_CREATE_SIGNING_KEY), lang.CmdPackageCreateFlagSigningKey) createFlags.StringVar(&pkgConfig.CreateOpts.SigningKeyPassword, "key-pass", v.GetString(V_PKG_CREATE_SIGNING_KEY_PASSWORD), lang.CmdPackageCreateFlagSigningKeyPassword) createFlags.StringToStringVar(&pkgConfig.CreateOpts.RegistryOverrides, "registry-override", v.GetStringMapString(V_PKG_CREATE_REGISTRY_OVERRIDE), lang.CmdPackageCreateFlagRegistryOverride) + + createFlags.MarkHidden("output-directory") } func bindDeployFlags() { @@ -342,14 +363,12 @@ func bindDeployFlags() { v.SetDefault(V_PKG_DEPLOY_COMPONENTS, "") v.SetDefault(V_PKG_DEPLOY_SHASUM, "") v.SetDefault(V_PKG_DEPLOY_SGET, "") - v.SetDefault(V_PKG_PUBLISH_OCI_CONCURRENCY, 3) v.SetDefault(V_PKG_DEPLOY_PUBLIC_KEY, "") deployFlags.StringToStringVar(&pkgConfig.DeployOpts.SetVariables, "set", v.GetStringMapString(V_PKG_DEPLOY_SET), lang.CmdPackageDeployFlagSet) deployFlags.StringVar(&pkgConfig.DeployOpts.Components, "components", v.GetString(V_PKG_DEPLOY_COMPONENTS), lang.CmdPackageDeployFlagComponents) deployFlags.StringVar(&pkgConfig.DeployOpts.Shasum, "shasum", v.GetString(V_PKG_DEPLOY_SHASUM), lang.CmdPackageDeployFlagShasum) deployFlags.StringVar(&pkgConfig.DeployOpts.SGetKeyPath, "sget", v.GetString(V_PKG_DEPLOY_SGET), lang.CmdPackageDeployFlagSget) - deployFlags.IntVar(&pkgConfig.PublishOpts.CopyOptions.Concurrency, "oci-concurrency", v.GetInt(V_PKG_PUBLISH_OCI_CONCURRENCY), lang.CmdPackagePublishFlagConcurrency) deployFlags.StringVarP(&pkgConfig.DeployOpts.PublicKeyPath, "key", "k", v.GetString(V_PKG_DEPLOY_PUBLIC_KEY), lang.CmdPackageDeployFlagPublicKey) } @@ -369,7 +388,6 @@ func bindRemoveFlags() { func bindPublishFlags() { publishFlags := packagePublishCmd.Flags() - publishFlags.IntVar(&pkgConfig.PublishOpts.CopyOptions.Concurrency, "oci-concurrency", v.GetInt(V_PKG_PUBLISH_OCI_CONCURRENCY), lang.CmdPackagePublishFlagConcurrency) publishFlags.StringVarP(&pkgConfig.PublishOpts.SigningKeyPath, "key", "k", v.GetString(V_PKG_PUBLISH_SIGNING_KEY), lang.CmdPackagePublishFlagSigningKey) publishFlags.StringVar(&pkgConfig.PublishOpts.SigningKeyPassword, "key-pass", v.GetString(V_PKG_PUBLISH_SIGNING_KEY_PASSWORD), lang.CmdPackagePublishFlagSigningKeyPassword) } @@ -377,8 +395,6 @@ func bindPublishFlags() { func bindPullFlags() { pullFlags := packagePullCmd.Flags() v.SetDefault(V_PKG_PULL_OUTPUT_DIR, "") - v.SetDefault(V_PKG_PULL_OCI_CONCURRENCY, 3) - pullFlags.StringVarP(&pkgConfig.PullOpts.OutputDirectory, "output-directory", "o", v.GetString(V_PKG_PULL_OUTPUT_DIR), lang.CmdPackageCreateFlagOutputDirectory) - pullFlags.IntVar(&pkgConfig.PullOpts.CopyOptions.Concurrency, "oci-concurrency", v.GetInt(V_PKG_PULL_OCI_CONCURRENCY), lang.CmdPackagePublishFlagConcurrency) - pullFlags.StringVarP(&pkgConfig.PullOpts.PublicKeyPath, "key", "k", v.GetString(V_PKG_PULL_PUBLIC_KEY), lang.CmdPackagePullPublicKey) + pullFlags.StringVarP(&pkgConfig.PullOpts.OutputDirectory, "output-directory", "o", v.GetString(V_PKG_PULL_OUTPUT_DIR), lang.CmdPackagePullFlagOutputDirectory) + pullFlags.StringVarP(&pkgConfig.PullOpts.PublicKeyPath, "key", "k", v.GetString(V_PKG_PULL_PUBLIC_KEY), lang.CmdPackagePullFlagPublicKey) } diff --git a/src/cmd/tools/archiver.go b/src/cmd/tools/archiver.go index 81e8a1cfc3..432bf70243 100644 --- a/src/cmd/tools/archiver.go +++ b/src/cmd/tools/archiver.go @@ -48,7 +48,7 @@ var archiverDecompressCmd = &cobra.Command{ sourceArchive, destinationPath := args[0], args[1] err := archiver.Unarchive(sourceArchive, destinationPath) if err != nil { - message.Fatal(err, lang.CmdToolsArchiverDecompressErr) + message.Fatalf(err, lang.CmdToolsArchiverDecompressErr, err.Error()) } if unarchiveAll { diff --git a/src/cmd/viper.go b/src/cmd/viper.go index 304bb93843..fcfa868aa1 100644 --- a/src/cmd/viper.go +++ b/src/cmd/viper.go @@ -49,9 +49,12 @@ const ( V_INIT_ARTIFACT_PUSH_USER = "init.artifact.push_username" V_INIT_ARTIFACT_PUSH_TOKEN = "init.artifact.push_token" + // Package config keys + V_PKG_OCI_CONCURRENCY = "package.oci_concurrency" + // Package create config keys V_PKG_CREATE_SET = "package.create.set" - V_PKG_CREATE_OUTPUT_DIR = "package.create.output_directory" + V_PKG_CREATE_OUTPUT = "package.create.output" V_PKG_CREATE_SBOM = "package.create.sbom" V_PKG_CREATE_SBOM_OUTPUT = "package.create.sbom_output" V_PKG_CREATE_SKIP_SBOM = "package.create.skip_sbom" @@ -69,14 +72,12 @@ const ( V_PKG_DEPLOY_PUBLIC_KEY = "package.deploy.public_key" // Package publish config keys - V_PKG_PUBLISH_OCI_CONCURRENCY = "package.publish.oci_concurrency" V_PKG_PUBLISH_SIGNING_KEY = "package.publish.signing_key" V_PKG_PUBLISH_SIGNING_KEY_PASSWORD = "package.publish.signing_key_password" // Package pull config keys - V_PKG_PULL_OCI_CONCURRENCY = "package.pull.oci_concurrency" - V_PKG_PULL_OUTPUT_DIR = "package.pull.output_directory" - V_PKG_PULL_PUBLIC_KEY = "package.pull.public_key" + V_PKG_PULL_OUTPUT_DIR = "package.pull.output_directory" + V_PKG_PULL_PUBLIC_KEY = "package.pull.public_key" ) func initViper() { diff --git a/src/config/config.go b/src/config/config.go index 81440f2677..e71f0d1f9c 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -57,6 +57,8 @@ const ( ZarfSBOMTar = "sboms.tar" ZarfPackagePrefix = "zarf-package-" + ZarfComponentsDir = "components" + ZarfInClusterContainerRegistryNodePort = 31999 ZarfInClusterGitServiceURL = "http://zarf-gitea-http.zarf.svc.cluster.local:3000" @@ -77,9 +79,6 @@ var ( // ZarfSeedPort is the NodePort Zarf uses for the 'seed registry' ZarfSeedPort string - // Dirty Solution to getting the real time deployedComponents components. - deployedComponents []types.DeployedComponent - // SkipLogFile is a flag to skip logging to a file SkipLogFile bool @@ -155,24 +154,6 @@ func GetCraneAuthOption(username string, secret string) crane.Option { })) } -// GetDeployingComponents returns the list of deploying components. -// TODO: (@jeff-mccoy) this should be moved out of config. -func GetDeployingComponents() []types.DeployedComponent { - return deployedComponents -} - -// SetDeployingComponents sets the list of deploying components. -// TODO: (@jeff-mccoy) this should be moved out of config. -func SetDeployingComponents(components []types.DeployedComponent) { - deployedComponents = components -} - -// ClearDeployingComponents clears the list of deploying components. -// TODO: (@jeff-mccoy) this should be moved out of config. -func ClearDeployingComponents() { - deployedComponents = []types.DeployedComponent{} -} - // GetValidPackageExtensions returns the valid package extensions. func GetValidPackageExtensions() [3]string { return [...]string{".tar.zst", ".tar", ".zip"} diff --git a/src/config/lang/english.go b/src/config/lang/english.go index aa1206ef07..5404370e48 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -17,6 +17,7 @@ import "errors" const ( ErrLoadingConfig = "failed to load config: %w" ErrLoadState = "Failed to load the Zarf State from the Kubernetes cluster." + ErrLoadPackageSecret = "Failed to load %s's secret from the Kubernetes cluster" ErrMarshal = "failed to marshal file: %w" ErrNoClusterConnection = "Failed to connect to the Kubernetes cluster." ErrTunnelFailed = "Failed to create a tunnel to the Kubernetes cluster." @@ -184,7 +185,8 @@ zarf init --git-push-password={PASSWORD} --git-push-username={USERNAME} --git-ur CmdInternalIsValidHostnameShort = "Checks if the current machine's hostname is RFC1123 compliant" // zarf package - CmdPackageShort = "Zarf package commands for creating, deploying, and inspecting packages" + CmdPackageShort = "Zarf package commands for creating, deploying, and inspecting packages" + CmdPackageFlagConcurrency = "Number of concurrent layer operations to perform when interacting with a remote package." CmdPackageCreateShort = "Use to create a Zarf package from a given directory or the current directory" CmdPackageCreateLong = "Builds an archive of resources and dependencies defined by the 'zarf.yaml' in the active directory.\n" + @@ -209,7 +211,7 @@ zarf init --git-push-password={PASSWORD} --git-push-username={USERNAME} --git-ur CmdPackageCreateFlagConfirm = "Confirm package creation without prompting" CmdPackageCreateFlagSet = "Specify package variables to set on the command line (KEY=value)" - CmdPackageCreateFlagOutputDirectory = "Specify the output directory for the created Zarf package" + CmdPackageCreateFlagOutput = "Specify the output (either a directory or an oci:// URL) for the created Zarf package" CmdPackageCreateFlagSbom = "View SBOM contents after creating the package" CmdPackageCreateFlagSbomOut = "Specify an output directory for the SBOMs from the created Zarf package" CmdPackageCreateFlagSkipSbom = "Skip generating SBOM for this package" @@ -236,11 +238,11 @@ zarf init --git-push-password={PASSWORD} --git-push-username={USERNAME} --git-ur CmdPackageRemoveFlagConfirm = "REQUIRED. Confirm the removal action to prevent accidental deletions" CmdPackageRemoveFlagComponents = "Comma-separated list of components to uninstall" - CmdPackagePublishFlagConcurrency = "Number of concurrent layer operations to perform when interacting with a remote package." CmdPackagePublishFlagSigningKey = "Path to private key file for signing packages" CmdPackagePublishFlagSigningKeyPassword = "Password to the private key file used for publishing packages" - CmdPackagePullPublicKey = "Path to public key file for validating signed packages" + CmdPackagePullFlagOutputDirectory = "Specify the output directory for the pulled Zarf package" + CmdPackagePullFlagPublicKey = "Path to public key file for validating signed packages" // zarf prepare CmdPrepareShort = "Tools to help prepare assets for packaging" @@ -274,7 +276,7 @@ zarf init --git-push-password={PASSWORD} --git-push-username={USERNAME} --git-ur CmdToolsArchiverCompressShort = "Compress a collection of sources based off of the destination file extension." CmdToolsArchiverCompressErr = "Unable to perform compression" CmdToolsArchiverDecompressShort = "Decompress an archive or Zarf package based off of the source file extension." - CmdToolsArchiverDecompressErr = "Unable to perform decompression" + CmdToolsArchiverDecompressErr = "Unable to perform decompression: %s" CmdToolsArchiverUnarchiveAllErr = "Unable to unarchive all nested tarballs" CmdToolsRegistryShort = "Tools for working with container registries using go-containertools." diff --git a/src/internal/api/components/list.go b/src/internal/api/components/list.go index 7d217c21aa..1441961f86 100644 --- a/src/internal/api/components/list.go +++ b/src/internal/api/components/list.go @@ -7,12 +7,19 @@ package components import ( "net/http" - "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/internal/api/common" + "github.com/defenseunicorns/zarf/src/internal/cluster" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/go-chi/chi/v5" ) -// ListDeployingComponents writes a list of packages that have been deployed to the connected cluster. -func ListDeployingComponents(w http.ResponseWriter, _ *http.Request) { - deployingPackages := config.GetDeployingComponents() - common.WriteJSONResponse(w, deployingPackages, http.StatusOK) +// ListDeployedComponents writes a list of packages that have been deployed to the connected cluster. +func ListDeployedComponents(w http.ResponseWriter, r *http.Request) { + pkgName := chi.URLParam(r, "pkg") + dp, err := cluster.NewClusterOrDie().GetDeployedPackage(pkgName) + if err != nil { + message.ErrorWebf(err, w, lang.ErrLoadPackageSecret, pkgName) + } + common.WriteJSONResponse(w, dp.DeployedComponents, http.StatusOK) } diff --git a/src/internal/api/start.go b/src/internal/api/start.go index cf31c90eef..b560f4a976 100644 --- a/src/internal/api/start.go +++ b/src/internal/api/start.go @@ -94,6 +94,7 @@ func LaunchAPIServer() { r.Put("/{pkg}/connect/{name}", packages.ConnectTunnel) r.Delete("/{pkg}/disconnect/{name}", packages.DisconnectTunnel) r.Get("/{pkg}/connections", packages.ListPackageConnections) + r.Get("/{pkg}/components/deployed", components.ListDeployedComponents) r.Get("/connections", packages.ListConnections) r.Get("/sbom/{path}", packages.ExtractSBOM) r.Delete("/sbom", packages.DeleteSBOM) @@ -105,10 +106,6 @@ func LaunchAPIServer() { }) }) }) - - r.Route("/components", func(r chi.Router) { - r.Get("/deployed", components.ListDeployingComponents) - }) }) // If no dev port specified, use the server port for the URL and try to open it diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index 6c14c70cc2..fca70d1290 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -109,6 +109,13 @@ func GetLogLevel() LogLevel { return logLevel } +// ZarfCommand prints a zarf terminal command. +func ZarfCommand(format string, a ...any) { + cmd := Paragraph("$ zarf "+format, a...) + style := pterm.NewStyle(pterm.FgWhite, pterm.BgBlack) + style.Println(cmd) +} + // Debug prints a debug message. func Debug(payload ...any) { debugPrinter(2, payload...) diff --git a/src/pkg/utils/oras.go b/src/pkg/oci/common.go similarity index 58% rename from src/pkg/utils/oras.go rename to src/pkg/oci/common.go index 9be155f134..4eab41dfc3 100644 --- a/src/pkg/utils/oras.go +++ b/src/pkg/oci/common.go @@ -1,30 +1,93 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package utils provides generic helper functions. -package utils +// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +package oci import ( "context" "errors" "fmt" "net/http" + "strings" zarfconfig "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/docker/cli/cli/config" "github.com/docker/cli/cli/config/configfile" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2" "oras.land/oras-go/v2/registry" "oras.land/oras-go/v2/registry/remote" "oras.land/oras-go/v2/registry/remote/auth" ) +const ( + // ZarfLayerMediaTypeBlob is the media type for all Zarf layers due to the range of possible content + ZarfLayerMediaTypeBlob = "application/vnd.zarf.layer.v1.blob" + // SkeletonSuffix is the reference suffix used for skeleton packages + SkeletonSuffix = "skeleton" +) + // OrasRemote is a wrapper around the Oras remote repository that includes a progress bar for interactive feedback. type OrasRemote struct { *remote.Repository context.Context - Transport *Transport + Transport *utils.Transport + CopyOpts oras.CopyOptions + client *auth.Client +} + +// NewOrasRemote returns an oras remote repository client and context for the given url. +// +// Registry auth is handled by the Docker CLI's credential store and checked before returning the client +func NewOrasRemote(url string) (*OrasRemote, error) { + ref, err := registry.ParseReference(strings.TrimPrefix(url, utils.OCIURLPrefix)) + if err != nil { + return nil, fmt.Errorf("failed to parse OCI reference: %w", err) + } + o := &OrasRemote{} + o.Context = context.TODO() + + err = o.WithRepository(ref) + if err != nil { + return nil, err + } + + err = o.CheckAuth() + if err != nil { + return nil, fmt.Errorf("unable to authenticate to %s: %s", ref.Registry, err.Error()) + } + + copyOpts := oras.DefaultCopyOptions + copyOpts.OnCopySkipped = o.printLayerSuccess + copyOpts.PostCopy = o.printLayerSuccess + o.CopyOpts = copyOpts + + return o, nil +} + +// WithRepository sets the repository for the remote as well as the auth client. +func (o *OrasRemote) WithRepository(ref registry.Reference) error { + // patch docker.io to registry-1.docker.io + // this allows end users to use docker.io as an alias for registry-1.docker.io + if ref.Registry == "docker.io" { + ref.Registry = "registry-1.docker.io" + } + client, err := o.withAuthClient(ref) + if err != nil { + return err + } + o.client = client + + repo, err := remote.NewRepository(ref.String()) + if err != nil { + return err + } + repo.PlainHTTP = zarfconfig.CommonOptions.Insecure + repo.Client = o.client + o.Repository = repo + return nil } // withScopes returns a context with the given scopes. @@ -45,10 +108,10 @@ func (o *OrasRemote) withAuthClient(ref registry.Reference) (*auth.Client, error message.Debugf("Loading docker config file from default config location: %s", config.Dir()) cfg, err := config.Load(config.Dir()) if err != nil { - return &auth.Client{}, err + return nil, err } if !cfg.ContainsAuth() { - return &auth.Client{}, errors.New("no docker config file found, run 'zarf tools registry login --help'") + return nil, errors.New("no docker config file found, run 'zarf tools registry login --help'") } configs := []*configfile.ConfigFile{cfg} @@ -61,7 +124,7 @@ func (o *OrasRemote) withAuthClient(ref registry.Reference) (*auth.Client, error authConf, err := configs[0].GetCredentialsStore(key).Get(key) if err != nil { - return &auth.Client{}, fmt.Errorf("unable to get credentials for %s: %w", key, err) + return nil, fmt.Errorf("unable to get credentials for %s: %w", key, err) } if authConf.ServerAddress != "" { @@ -78,7 +141,7 @@ func (o *OrasRemote) withAuthClient(ref registry.Reference) (*auth.Client, error transport := http.DefaultTransport.(*http.Transport).Clone() transport.TLSClientConfig.InsecureSkipVerify = zarfconfig.CommonOptions.Insecure - o.Transport = NewTransport(transport, nil) + o.Transport = utils.NewTransport(transport, nil) client := &auth.Client{ Credential: auth.StaticCredential(ref.Registry, cred), @@ -92,38 +155,13 @@ func (o *OrasRemote) withAuthClient(ref registry.Reference) (*auth.Client, error return client, nil } -// NewOrasRemote returns an oras remote repository client and context for the given reference. -func NewOrasRemote(ref registry.Reference) (*OrasRemote, error) { - o := &OrasRemote{} - o.Context = context.TODO() - // patch docker.io to registry-1.docker.io - // this allows end users to use docker.io as an alias for registry-1.docker.io - if ref.Registry == "docker.io" { - ref.Registry = "registry-1.docker.io" - } - repo, err := remote.NewRepository(ref.String()) - if err != nil { - return &OrasRemote{}, err - } - repo.PlainHTTP = zarfconfig.CommonOptions.Insecure - authClient, err := o.withAuthClient(ref) +// CheckAuth checks if the user is authenticated to the remote registry. +func (o *OrasRemote) CheckAuth() error { + reg, err := remote.NewRegistry(o.Repository.Reference.Registry) if err != nil { - return &OrasRemote{}, err + return err } - repo.Client = authClient - o.Repository = repo - return o, nil -} - -// PrintLayerExists prints a success message to the console when a layer has been successfully published to a registry. -func PrintLayerExists(_ context.Context, desc ocispec.Descriptor) error { - title := desc.Annotations[ocispec.AnnotationTitle] - var format string - if title != "" { - format = fmt.Sprintf("%s %s", desc.Digest.Encoded()[:12], First30last30(title)) - } else { - format = fmt.Sprintf("%s [%s]", desc.Digest.Encoded()[:12], desc.MediaType) - } - message.Successf(format) - return nil + reg.PlainHTTP = zarfconfig.CommonOptions.Insecure + reg.Client = o.client + return reg.Ping(o.Context) } diff --git a/src/pkg/oci/manifest.go b/src/pkg/oci/manifest.go new file mode 100644 index 0000000000..803bf15d34 --- /dev/null +++ b/src/pkg/oci/manifest.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +package oci + +import ( + "path/filepath" + + "github.com/defenseunicorns/zarf/src/pkg/utils" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// ZarfOCIManifest is a wrapper around the OCI manifest +// +// it includes the path to the index.json, oci-layout, and image blobs. +// as well as a few helper functions for locating layers and calculating the size of the layers. +type ZarfOCIManifest struct { + ocispec.Manifest + indexPath string + ociLayoutPath string + imagesBlobsDir string +} + +// NewZarfOCIManifest returns a new ZarfOCIManifest. +func NewZarfOCIManifest(manifest *ocispec.Manifest) *ZarfOCIManifest { + return &ZarfOCIManifest{ + Manifest: *manifest, + indexPath: filepath.Join("images", "index.json"), + ociLayoutPath: filepath.Join("images", "oci-layout"), + imagesBlobsDir: filepath.Join("images", "blobs", "sha256"), + } +} + +// Locate returns the descriptor for the layer with the given path. +func (m *ZarfOCIManifest) Locate(path string) ocispec.Descriptor { + return utils.Find(m.Layers, func(layer ocispec.Descriptor) bool { + return layer.Annotations[ocispec.AnnotationTitle] == path + }) +} + +// SumLayersSize returns the sum of the size of all the layers in the manifest. +func (m *ZarfOCIManifest) SumLayersSize() int64 { + var sum int64 + for _, layer := range m.Layers { + sum += layer.Size + } + return sum +} diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go new file mode 100644 index 0000000000..8491596211 --- /dev/null +++ b/src/pkg/oci/pull.go @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +package oci + +import ( + "context" + "fmt" + "path/filepath" + "sync" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/types" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pterm/pterm" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content" + "oras.land/oras-go/v2/content/file" +) + +var ( + // AlwaysPull is a list of paths that will always be pulled from the remote repository. + AlwaysPull = []string{config.ZarfYAML, config.ZarfChecksumsTxt, config.ZarfYAMLSignature} +) + +// LayersFromPaths returns the descriptors for the given paths from the root manifest. +func (o *OrasRemote) LayersFromPaths(requestedPaths []string) (layers []ocispec.Descriptor, err error) { + manifest, err := o.FetchRoot() + if err != nil { + return nil, err + } + for _, path := range requestedPaths { + layer := manifest.Locate(path) + if o.isEmptyDescriptor(layer) { + return nil, fmt.Errorf("path %s does not exist in this package", path) + } + layers = append(layers, layer) + } + return layers, nil +} + +// LayersFromRequestedComponents returns the descriptors for the given components from the root manifest. +// +// It also retrieves the descriptors for all image layers that are required by the components. +// +// It also respects the `required` flag on components, and will retrieve all necessary layers for required components. +func (o *OrasRemote) LayersFromRequestedComponents(requestedComponents []string) (layers []ocispec.Descriptor, err error) { + root, err := o.FetchRoot() + if err != nil { + return nil, err + } + + pkg, err := o.FetchZarfYAML(root) + if err != nil { + return nil, err + } + images := map[string]bool{} + tarballFormat := "%s.tar" + for _, name := range requestedComponents { + component := utils.Find(pkg.Components, func(component types.ZarfComponent) bool { + return component.Name == name + }) + if component.Name == "" { + return nil, fmt.Errorf("component %s does not exist in this package", name) + } + } + for _, component := range pkg.Components { + // If we requested this component, or it is required, we need to pull its images and tarball + if utils.SliceContains(requestedComponents, component.Name) || component.Required { + for _, image := range component.Images { + images[image] = true + } + layers = append(layers, root.Locate(filepath.Join(config.ZarfComponentsDir, fmt.Sprintf(tarballFormat, component.Name)))) + } + } + // Append the sboms.tar layer if it exists + // + // Since sboms.tar is not a heavy addition 99% of the time, we'll just always pull it + sbomsDescriptor := root.Locate(config.ZarfSBOMTar) + if !o.isEmptyDescriptor(sbomsDescriptor) { + layers = append(layers, sbomsDescriptor) + } + if len(images) > 0 { + // Add the image index and the oci-layout layers + layers = append(layers, root.Locate(root.indexPath), root.Locate(root.ociLayoutPath)) + index, err := o.FetchImagesIndex(root) + if err != nil { + return nil, err + } + for image := range images { + manifestDescriptor := utils.Find(index.Manifests, func(layer ocispec.Descriptor) bool { + return layer.Annotations[ocispec.AnnotationBaseImageName] == image + }) + manifest, err := o.FetchManifest(manifestDescriptor) + if err != nil { + return nil, err + } + // Add the manifest and the manifest config layers + layers = append(layers, root.Locate(filepath.Join(root.imagesBlobsDir, manifestDescriptor.Digest.Encoded()))) + layers = append(layers, root.Locate(filepath.Join(root.imagesBlobsDir, manifest.Config.Digest.Encoded()))) + + // Add all the layers from the manifest + for _, layer := range manifest.Layers { + layerPath := filepath.Join(root.imagesBlobsDir, layer.Digest.Encoded()) + layers = append(layers, root.Locate(layerPath)) + } + } + } + return layers, nil +} + +// PullPackage pulls the package from the remote repository and saves it to the given path. +// +// layersToPull is an optional parameter that allows the caller to specify which layers to pull. +// +// The following layers will ALWAYS be pulled if they exist: +// - zarf.yaml +// - checksums.txt +// - zarf.yaml.sig +func (o *OrasRemote) PullPackage(destinationDir string, concurrency int, layersToPull ...ocispec.Descriptor) error { + isPartialPull := len(layersToPull) > 0 + ref := o.Reference + pterm.Println() + message.Debugf("Pulling %s", ref.String()) + message.Infof("Pulling Zarf package from %s", ref) + + manifest, err := o.FetchRoot() + if err != nil { + return err + } + + estimatedBytes := int64(0) + if isPartialPull { + for _, path := range AlwaysPull { + desc := manifest.Locate(path) + layersToPull = append(layersToPull, desc) + } + for _, desc := range layersToPull { + estimatedBytes += desc.Size + } + } else { + estimatedBytes = manifest.SumLayersSize() + } + estimatedBytes += manifest.Config.Size + + dst, err := file.New(destinationDir) + if err != nil { + return err + } + defer dst.Close() + + copyOpts := o.CopyOpts + copyOpts.Concurrency = concurrency + if isPartialPull { + paths := []string{} + for _, layer := range layersToPull { + paths = append(paths, layer.Annotations[ocispec.AnnotationTitle]) + } + copyOpts.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + nodes, err := content.Successors(ctx, fetcher, desc) + if err != nil { + return nil, err + } + var ret []ocispec.Descriptor + for _, node := range nodes { + if utils.SliceContains(paths, node.Annotations[ocispec.AnnotationTitle]) { + ret = append(ret, node) + } + } + return ret, nil + } + } + + // Create a thread to update a progress bar as we save the package to disk + doneSaving := make(chan int) + var wg sync.WaitGroup + wg.Add(1) + go utils.RenderProgressBarForLocalDirWrite(destinationDir, estimatedBytes, &wg, doneSaving, "Pulling Zarf package data") + _, err = oras.Copy(o.Context, o.Repository, ref.String(), dst, ref.String(), copyOpts) + if err != nil { + return err + } + + // Send a signal to the progress bar that we're done and wait for it to finish + doneSaving <- 1 + wg.Wait() + + message.Debugf("Pulled %s", ref.String()) + message.Successf("Pulled %s", ref.String()) + + layersToCheck := []string{} + if isPartialPull { + for _, layer := range layersToPull { + if layer.Annotations[ocispec.AnnotationTitle] != "" { + layersToCheck = append(layersToCheck, layer.Annotations[ocispec.AnnotationTitle]) + } + } + } + return utils.ValidatePackageChecksums(destinationDir, layersToCheck) +} + +// PullLayer pulls a layer from the remote repository and saves it to `destinationDir/annotationTitle`. +func (o *OrasRemote) PullLayer(desc ocispec.Descriptor, destinationDir string) error { + if desc.MediaType != ZarfLayerMediaTypeBlob { + return fmt.Errorf("invalid media type for file layer: %s", desc.MediaType) + } + b, err := o.FetchLayer(desc) + if err != nil { + return err + } + return utils.WriteFile(filepath.Join(destinationDir, desc.Annotations[ocispec.AnnotationTitle]), b) +} + +// PullPackageMetadata pulls the package metadata from the remote repository and saves it to `destinationDir`. +func (o *OrasRemote) PullPackageMetadata(destinationDir string) error { + root, err := o.FetchRoot() + if err != nil { + return err + } + for _, path := range AlwaysPull { + desc := root.Locate(path) + if !o.isEmptyDescriptor(desc) { + err = o.PullLayer(desc, destinationDir) + if err != nil { + return err + } + } + } + var pkg types.ZarfPackage + err = utils.ReadYaml(filepath.Join(destinationDir, config.ZarfYAML), &pkg) + if err != nil { + return err + } + return utils.ValidatePackageChecksums(destinationDir, AlwaysPull) +} diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go new file mode 100644 index 0000000000..1b259c0d5b --- /dev/null +++ b/src/pkg/oci/push.go @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +package oci + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/types" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content" + "oras.land/oras-go/v2/content/file" +) + +// ConfigPartial is a partial OCI config that is used to create the manifest config. +// +// Unless specified, an empty manifest config will be used: `{}` +// which causes an error on Google Artifact Registry +// +// to negate this, we create a simple manifest config with some build metadata +// +// the contents of this file are not used by Zarf +type ConfigPartial struct { + Architecture string `json:"architecture"` + OCIVersion string `json:"ociVersion"` + Annotations map[string]string `json:"annotations,omitempty"` +} + +// PushFile pushes the file at the given path to the remote repository. +func (o *OrasRemote) PushFile(path string) (*ocispec.Descriptor, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + return o.PushBytes(b, ZarfLayerMediaTypeBlob) +} + +// PushBytes pushes the given bytes to the remote repository. +func (o *OrasRemote) PushBytes(b []byte, mediaType string) (*ocispec.Descriptor, error) { + desc := content.NewDescriptorFromBytes(mediaType, b) + return &desc, o.Push(o.Context, desc, bytes.NewReader(b)) +} + +func (o *OrasRemote) pushManifestConfigFromMetadata(metadata *types.ZarfMetadata, build *types.ZarfBuildData) (*ocispec.Descriptor, error) { + annotations := map[string]string{ + ocispec.AnnotationTitle: metadata.Name, + ocispec.AnnotationDescription: metadata.Description, + } + manifestConfig := ConfigPartial{ + Architecture: build.Architecture, + OCIVersion: "1.0.1", + Annotations: annotations, + } + manifestConfigBytes, err := json.Marshal(manifestConfig) + if err != nil { + return nil, err + } + return o.PushBytes(manifestConfigBytes, ocispec.MediaTypeImageConfig) +} + +func (o *OrasRemote) manifestAnnotationsFromMetadata(metadata *types.ZarfMetadata) map[string]string { + annotations := map[string]string{ + ocispec.AnnotationDescription: metadata.Description, + } + + if url := metadata.URL; url != "" { + annotations[ocispec.AnnotationURL] = url + } + if authors := metadata.Authors; authors != "" { + annotations[ocispec.AnnotationAuthors] = authors + } + if documentation := metadata.Documentation; documentation != "" { + annotations[ocispec.AnnotationDocumentation] = documentation + } + if source := metadata.Source; source != "" { + annotations[ocispec.AnnotationSource] = source + } + if vendor := metadata.Vendor; vendor != "" { + annotations[ocispec.AnnotationVendor] = vendor + } + + return annotations +} + +func (o *OrasRemote) generatePackManifest(src *file.Store, descs []ocispec.Descriptor, configDesc *ocispec.Descriptor, metadata *types.ZarfMetadata) (ocispec.Descriptor, error) { + packOpts := oras.PackOptions{} + packOpts.ConfigDescriptor = configDesc + packOpts.PackImageManifest = true + packOpts.ManifestAnnotations = o.manifestAnnotationsFromMetadata(metadata) + + root, err := oras.Pack(o.Context, src, ocispec.MediaTypeImageManifest, descs, packOpts) + if err != nil { + return ocispec.Descriptor{}, err + } + if err = src.Tag(o.Context, root, root.Digest.String()); err != nil { + return ocispec.Descriptor{}, err + } + + return root, nil +} + +// PublishPackage publishes the package to the remote repository. +func (o *OrasRemote) PublishPackage(pkg *types.ZarfPackage, sourceDir string, concurrency int) error { + ctx := o.Context + // source file store + src, err := file.New(sourceDir) + if err != nil { + return err + } + defer src.Close() + + message.Infof("Publishing package to %s", o.Reference.String()) + spinner := message.NewProgressSpinner("") + defer spinner.Stop() + + // Get all of the layers in the package + paths := []string{} + err = filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error { + // Catch any errors that happened during the walk + if err != nil { + return err + } + + // Add any resource that is not a directory to the paths of objects we will include into the package + if !info.IsDir() { + paths = append(paths, path) + } + return nil + }) + if err != nil { + return fmt.Errorf("unable to get the layers in the package to publish: %w", err) + } + + var descs []ocispec.Descriptor + for idx, path := range paths { + name, err := filepath.Rel(sourceDir, path) + if err != nil { + return err + } + spinner.Updatef("Preparing layer %d/%d: %s", idx+1, len(paths), name) + + mediaType := ZarfLayerMediaTypeBlob + + desc, err := src.Add(ctx, name, mediaType, path) + if err != nil { + return err + } + descs = append(descs, desc) + } + spinner.Successf("Prepared %d layers", len(descs)) + + copyOpts := o.CopyOpts + copyOpts.Concurrency = concurrency + var total int64 + for _, desc := range descs { + total += desc.Size + } + // assumes referrers API is not supported since OCI artifact + // media type is not supported + o.SetReferrersCapability(false) + + // push the manifest config + // since this config is so tiny, and the content is not used again + // it is not logged to the progress, but will error if it fails + manifestConfigDesc, err := o.pushManifestConfigFromMetadata(&pkg.Metadata, &pkg.Build) + if err != nil { + return err + } + root, err := o.generatePackManifest(src, descs, manifestConfigDesc, &pkg.Metadata) + if err != nil { + return err + } + total += root.Size + manifestConfigDesc.Size + + o.Transport.ProgressBar = message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", o.Reference.Repository, o.Reference.Reference)) + defer o.Transport.ProgressBar.Stop() + // attempt to push the image manifest + _, err = oras.Copy(ctx, src, root.Digest.String(), o, o.Reference.Reference, copyOpts) + if err != nil { + return err + } + + o.Transport.ProgressBar.Successf("Published %s [%s]", o.Reference, root.MediaType) + message.HorizontalRule() + if strings.HasSuffix(o.Reference.String(), SkeletonSuffix) { + message.Title("How to import components from this skeleton:", "") + ex := []types.ZarfComponent{} + for _, c := range pkg.Components { + ex = append(ex, types.ZarfComponent{ + Name: fmt.Sprintf("import-%s", c.Name), + Import: types.ZarfComponentImport{ + ComponentName: c.Name, + URL: fmt.Sprintf("oci://%s", o.Reference), + }, + }) + } + utils.ColorPrintYAML(ex) + } else { + flags := "" + if config.CommonOptions.Insecure { + flags = "--insecure" + } + message.Title("To inspect/deploy/pull:", "") + message.ZarfCommand("package inspect oci://%s %s", o.Reference, flags) + message.ZarfCommand("package deploy oci://%s %s", o.Reference, flags) + message.ZarfCommand("package pull oci://%s %s", o.Reference, flags) + } + + return nil +} diff --git a/src/pkg/oci/utils.go b/src/pkg/oci/utils.go new file mode 100644 index 0000000000..53073a3e83 --- /dev/null +++ b/src/pkg/oci/utils.go @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +package oci + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + config "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/types" + goyaml "github.com/goccy/go-yaml" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content" + "oras.land/oras-go/v2/registry" +) + +// ReferenceFromMetadata returns a reference for the given metadata. +// +// prepending the provided prefix +// +// appending the provided suffix to the version +func ReferenceFromMetadata(registryLocation string, metadata *types.ZarfMetadata, suffix string) (*registry.Reference, error) { + ver := metadata.Version + if len(ver) == 0 { + return nil, errors.New("version is required for publishing") + } + + if !strings.HasSuffix(registryLocation, "/") { + registryLocation = registryLocation + "/" + } + + format := "%s%s:%s-%s" + + raw := fmt.Sprintf(format, registryLocation, metadata.Name, ver, suffix) + + ref, err := registry.ParseReference(raw) + if err != nil { + return nil, err + } + + return &ref, nil +} + +// FetchRoot fetches the root manifest from the remote repository. +func (o *OrasRemote) FetchRoot() (*ZarfOCIManifest, error) { + // get the manifest descriptor + descriptor, err := o.Resolve(o.Context, o.Reference.Reference) + if err != nil { + return nil, err + } + + // get the manifest itself + bytes, err := o.FetchLayer(descriptor) + if err != nil { + return nil, err + } + manifest := ocispec.Manifest{} + + if err = json.Unmarshal(bytes, &manifest); err != nil { + return nil, err + } + return NewZarfOCIManifest(&manifest), nil +} + +// FetchManifest fetches the manifest with the given descriptor from the remote repository. +func (o *OrasRemote) FetchManifest(desc ocispec.Descriptor) (manifest *ZarfOCIManifest, err error) { + bytes, err := o.FetchLayer(desc) + if err != nil { + return manifest, err + } + err = json.Unmarshal(bytes, &manifest) + if err != nil { + return manifest, err + } + return manifest, nil +} + +// FetchLayer fetches the layer with the given descriptor from the remote repository. +func (o *OrasRemote) FetchLayer(desc ocispec.Descriptor) (bytes []byte, err error) { + return content.FetchAll(o.Context, o, desc) +} + +// FetchZarfYAML fetches the zarf.yaml file from the remote repository. +func (o *OrasRemote) FetchZarfYAML(manifest *ZarfOCIManifest) (pkg types.ZarfPackage, err error) { + zarfYamlDescriptor := manifest.Locate(config.ZarfYAML) + if zarfYamlDescriptor.Digest == "" { + return pkg, fmt.Errorf("unable to find %s in the manifest", config.ZarfYAML) + } + zarfYamlBytes, err := o.FetchLayer(zarfYamlDescriptor) + if err != nil { + return pkg, err + } + err = goyaml.Unmarshal(zarfYamlBytes, &pkg) + if err != nil { + return pkg, err + } + return pkg, nil +} + +// FetchImagesIndex fetches the images/index.json file from the remote repository. +func (o *OrasRemote) FetchImagesIndex(manifest *ZarfOCIManifest) (index *ocispec.Index, err error) { + indexDescriptor := manifest.Locate(manifest.indexPath) + indexBytes, err := o.FetchLayer(indexDescriptor) + if err != nil { + return nil, err + } + err = json.Unmarshal(indexBytes, &index) + if err != nil { + return nil, err + } + return index, nil +} + +// printLayerSuccess prints a success message to the console when a layer has been successfully published/pulled to/from a registry. +func (o *OrasRemote) printLayerSuccess(_ context.Context, desc ocispec.Descriptor) error { + title := desc.Annotations[ocispec.AnnotationTitle] + var format string + if title != "" { + format = fmt.Sprintf("%s %s", desc.Digest.Encoded()[:12], utils.First30last30(title)) + } else { + format = fmt.Sprintf("%s [%s]", desc.Digest.Encoded()[:12], desc.MediaType) + } + message.Successf(format) + return nil +} + +func (o *OrasRemote) isEmptyDescriptor(desc ocispec.Descriptor) bool { + return desc.Digest == "" && desc.Size == 0 +} diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index d8e36df618..dfb430db39 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -5,9 +5,9 @@ package packager import ( - "bufio" "crypto" "encoding/json" + "errors" "fmt" "io" "os" @@ -24,6 +24,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/utils" ) @@ -32,6 +33,7 @@ import ( type Packager struct { cfg *types.PackagerConfig cluster *cluster.Cluster + remote *oci.OrasRemote tmp types.TempPaths arch string warnings []string @@ -43,8 +45,6 @@ New creates a new package instance with the provided config. Note: This function creates a tmp directory that should be cleaned up with p.ClearTempPaths(). */ func New(cfg *types.PackagerConfig) (*Packager, error) { - message.Debugf("packager.New(%s)", message.JSONValue(cfg)) - if cfg == nil { return nil, fmt.Errorf("no config provided") } @@ -224,7 +224,7 @@ func createPaths() (paths types.TempPaths, err error) { InjectBinary: filepath.Join(basePath, "zarf-injector"), SeedImages: filepath.Join(basePath, "seed-images"), Images: filepath.Join(basePath, "images"), - Components: filepath.Join(basePath, "components"), + Components: filepath.Join(basePath, config.ZarfComponentsDir), SbomTar: filepath.Join(basePath, config.ZarfSBOMTar), Sboms: filepath.Join(basePath, "sboms"), Checksums: filepath.Join(basePath, config.ZarfChecksumsTxt), @@ -237,7 +237,11 @@ func createPaths() (paths types.TempPaths, err error) { func getRequestedComponentList(requestedComponents string) []string { if requestedComponents != "" { - return strings.Split(requestedComponents, ",") + split := strings.Split(requestedComponents, ",") + for idx, component := range split { + split[idx] = strings.ToLower(strings.TrimSpace(component)) + } + return split } return []string{} @@ -249,6 +253,8 @@ func (p *Packager) loadZarfPkg() error { return fmt.Errorf("unable to handle the provided package path: %w", err) } + extractedToTmp := p.cfg.DeployOpts.PackagePath == p.tmp.Base + spinner := message.NewProgressSpinner("Loading Zarf Package %s", p.cfg.DeployOpts.PackagePath) defer spinner.Stop() @@ -263,7 +269,7 @@ func (p *Packager) loadZarfPkg() error { } // If the package was pulled from OCI, there is no need to extract it since it is unpacked already - if p.cfg.DeployOpts.PackagePath != p.tmp.Base { + if !extractedToTmp { // Extract the archive spinner.Updatef("Extracting the package, this may take a few moments") if err := archiver.Unarchive(p.cfg.DeployOpts.PackagePath, p.tmp.Base); err != nil { @@ -272,15 +278,16 @@ func (p *Packager) loadZarfPkg() error { } // Load the config from the extracted archive zarf.yaml - spinner.Updatef("Loading the zarf package config") + spinner.Updatef("Loading the Zarf package config") configPath := p.tmp.ZarfYaml if err := p.readYaml(configPath); err != nil { return fmt.Errorf("unable to read the zarf.yaml in %s: %w", p.tmp.Base, err) } // Validate the checksums of all the things!!! - if p.cfg.Pkg.Metadata.AggregateChecksum != "" { - if err := p.validatePackageChecksums(); err != nil { + // validation is skipped here if the package is from OCI as the checksums are validated after the pull + if !extractedToTmp { + if err := utils.ValidatePackageChecksums(p.tmp.Base, nil); err != nil { return fmt.Errorf("unable to validate the package checksums: %w", err) } } @@ -421,81 +428,6 @@ func (p *Packager) handleIfPartialPkg() error { return nil } -func (p *Packager) validatePackageChecksums() error { - - // Run pre-checks to make sure we have what we need to validate the checksums - _, err := os.Stat(p.tmp.Checksums) - if err != nil { - return fmt.Errorf("unable to validate checksums as we are unable to find checksums.txt file within the package") - } - if p.cfg.Pkg.Metadata.AggregateChecksum == "" { - return fmt.Errorf("unable to validate checksums because of missing metadata checksum signature") - } - - // Create a map of all the files in the package so we can track which files we have processed - filepathMap := make(map[string]bool) - err = filepath.Walk(p.tmp.Base, func(path string, info os.FileInfo, err error) error { - if !info.IsDir() { - filepathMap[path] = false - } - return nil - }) - if err != nil { - return err - } - - // Verify the that checksums.txt file matches the aggregated checksum provided - actualAggregateChecksum, err := utils.GetSHA256OfFile(p.tmp.Checksums) - if err != nil { - return fmt.Errorf("unable to get the checksum of the checksums.txt file: %w", err) - } - if actualAggregateChecksum != p.cfg.Pkg.Metadata.AggregateChecksum { - return fmt.Errorf("mismatch on the checksum of the checksums.txt file, the checksums.txt file might have been tampered with") - } - - // Check off all the files that we can trust given the checksum and signing checks - filepathMap[p.tmp.Checksums] = true - filepathMap[p.tmp.ZarfYaml] = true - filepathMap[p.tmp.ZarfSig] = true - - // Load the contents of the checksums file - checksumsFile, err := os.Open(filepath.Join(p.tmp.Base, config.ZarfChecksumsTxt)) - if err != nil { - return err - } - defer checksumsFile.Close() - - /* Process every line in the checksums file */ - scanner := bufio.NewScanner(checksumsFile) - scanner.Split(bufio.ScanLines) - for scanner.Scan() { - // Separate the checksum from the file path - strs := strings.Split(scanner.Text(), " ") - itemPath := strs[1] - expectedShasum := strs[0] - - actualShasum, err := utils.GetSHA256OfFile(filepath.Join(p.tmp.Base, itemPath)) - if err != nil { - return err - } - - if expectedShasum != actualShasum { - return fmt.Errorf("mismatch on the checksum of the %s file (expected: %s, actual: %s)", itemPath, expectedShasum, actualShasum) - } - - filepathMap[filepath.Join(p.tmp.Base, itemPath)] = true - } - - for path, processed := range filepathMap { - if !processed { - return fmt.Errorf("the file %s was present in the Zarf package but not specified in the checksums.txt, the package might have been tampered with", path) - } - } - - message.Successf("All of the checksums matched!") - return nil -} - // validatePackageArchitecture validates that the package architecture matches the target cluster architecture. func (p *Packager) validatePackageArchitecture() error { // Ignore this check if the architecture is explicitly "multi" @@ -517,28 +449,31 @@ func (p *Packager) validatePackageArchitecture() error { return nil } +var ( + // ErrPkgKeyButNoSig is returned when a key was provided but the package is not signed + ErrPkgKeyButNoSig = errors.New("a key was provided but the package is not signed - remove the --key flag and run the command again") + // ErrPkgSigButNoKey is returned when a package is signed but no key was provided + ErrPkgSigButNoKey = errors.New("package is signed but no key was provided - add a key with the --key flag or use the --insecure flag and run the command again") +) + func (p *Packager) validatePackageSignature(publicKeyPath string) error { - // If the insecure flag was provided, ignore the signature validation - if config.CommonOptions.Insecure { + // If the insecure flag was provided, or there is no aggregate checksum, ignore the signature validation + if config.CommonOptions.Insecure || p.cfg.Pkg.Metadata.AggregateChecksum == "" { return nil } // Handle situations where there is no signature within the package - _, sigCheckErr := os.Stat(p.tmp.ZarfSig) - if sigCheckErr != nil { + sigExist := !utils.InvalidPath(p.tmp.ZarfSig) + if !sigExist && publicKeyPath == "" { // Nobody was expecting a signature, so we can just return - if publicKeyPath == "" { - return nil - } - - // We were expecting a signature, but there wasn't one.. - return fmt.Errorf("package is not signed but a key was provided") - } - - // Validate the signature of the package - if publicKeyPath == "" { - return fmt.Errorf("package is signed but no key was provided - add a key with the --key flag or use the --insecure flag and run the command again") + return nil + } else if sigExist && publicKeyPath == "" { + // The package is signed but no key was provided + return ErrPkgSigButNoKey + } else if !sigExist && publicKeyPath != "" { + // A key was provided but there is no signature + return ErrPkgKeyButNoSig } // Validate the signature with the key we were provided @@ -603,3 +538,74 @@ func (p *Packager) archiveComponent(component types.ZarfComponent) error { } return os.RemoveAll(componentPath) } + +func (p *Packager) archivePackage(sourceDir string, destinationTarball string) error { + spinner := message.NewProgressSpinner("Writing %s to %s", sourceDir, destinationTarball) + defer spinner.Stop() + // Make the archive + archiveSrc := []string{sourceDir + string(os.PathSeparator)} + if err := archiver.Archive(archiveSrc, destinationTarball); err != nil { + return fmt.Errorf("unable to create package: %w", err) + } + spinner.Updatef("Wrote %s to %s", sourceDir, destinationTarball) + + f, err := os.Stat(destinationTarball) + if err != nil { + return fmt.Errorf("unable to read the package archive: %w", err) + } + + // Convert Megabytes to bytes. + chunkSize := p.cfg.CreateOpts.MaxPackageSizeMB * 1000 * 1000 + + // If a chunk size was specified and the package is larger than the chunk size, split it into chunks. + if p.cfg.CreateOpts.MaxPackageSizeMB > 0 && f.Size() > int64(chunkSize) { + spinner.Updatef("Package is larger than %dMB, splitting into multiple files", p.cfg.CreateOpts.MaxPackageSizeMB) + chunks, sha256sum, err := utils.SplitFile(destinationTarball, chunkSize) + if err != nil { + return fmt.Errorf("unable to split the package archive into multiple files: %w", err) + } + if len(chunks) > 999 { + return fmt.Errorf("unable to split the package archive into multiple files: must be less than 1,000 files") + } + + status := fmt.Sprintf("Package split into %d files, original sha256sum is %s", len(chunks)+1, sha256sum) + spinner.Updatef(status) + message.Debug(status) + _ = os.RemoveAll(destinationTarball) + + // Marshal the data into a json file. + jsonData, err := json.Marshal(types.ZarfPartialPackageData{ + Count: len(chunks), + Bytes: f.Size(), + Sha256Sum: sha256sum, + }) + if err != nil { + return fmt.Errorf("unable to marshal the partial package data: %w", err) + } + + // Prepend the json data to the first chunk. + chunks = append([][]byte{jsonData}, chunks...) + + for idx, chunk := range chunks { + path := fmt.Sprintf("%s.part%03d", destinationTarball, idx) + status := fmt.Sprintf("Writing %s", path) + spinner.Updatef(status) + message.Debug(status) + if err := os.WriteFile(path, chunk, 0644); err != nil { + return fmt.Errorf("unable to write the file %s: %w", path, err) + } + } + } + spinner.Successf("Package tarball successfully written") + return nil +} + +// SetOCIRemote sets the remote OCI client for the package. +func (p *Packager) SetOCIRemote(url string) error { + remote, err := oci.NewOrasRemote(url) + if err != nil { + return err + } + p.remote = remote + return nil +} diff --git a/src/pkg/packager/components.go b/src/pkg/packager/components.go index 42c970621b..90b9f1af90 100644 --- a/src/pkg/packager/components.go +++ b/src/pkg/packager/components.go @@ -7,7 +7,6 @@ package packager import ( "fmt" "runtime" - "strings" "github.com/AlecAivazis/survey/v2" "github.com/defenseunicorns/zarf/src/config" @@ -161,7 +160,7 @@ func (p *Packager) isRequiredOrRequested(component types.ZarfComponent, requeste if len(requestedComponentNames) > 0 || config.CommonOptions.Confirm { for _, requestedComponent := range requestedComponentNames { // If the component name matches one of the requested components, then return true - if strings.ToLower(requestedComponent) == component.Name { + if requestedComponent == component.Name { return true } } diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index 93f9a9f706..40663216f5 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -13,6 +13,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" @@ -88,7 +89,7 @@ func (p *Packager) getChildComponent(parent types.ZarfComponent, pathAncestry st var cachePath string if parent.Import.URL != "" { - if !strings.HasSuffix(parent.Import.URL, skeletonSuffix) { + if !strings.HasSuffix(parent.Import.URL, oci.SkeletonSuffix) { return child, fmt.Errorf("import URL must be a 'skeleton' package: %s", parent.Import.URL) } @@ -102,8 +103,16 @@ func (p *Packager) getChildComponent(parent types.ZarfComponent, pathAncestry st return child, fmt.Errorf("unable to create cache path %s: %w", cachePath, err) } - componentLayer := filepath.Join("components", fmt.Sprintf("%s.tar", childComponentName)) - err = p.handleOciPackage(skelURL, cachePath, 3, componentLayer) + componentLayer := filepath.Join(config.ZarfComponentsDir, fmt.Sprintf("%s.tar", childComponentName)) + err = p.SetOCIRemote(parent.Import.URL) + if err != nil { + return child, err + } + manifest, err := p.remote.FetchRoot() + if err != nil { + return child, err + } + err = p.remote.PullPackage(cachePath, 3, manifest.Locate(componentLayer)) if err != nil { return child, fmt.Errorf("unable to pull skeleton from %s: %w", skelURL, err) } @@ -149,9 +158,9 @@ func (p *Packager) getChildComponent(parent types.ZarfComponent, pathAncestry st // If it's OCI, we need to unpack the component tarball if parent.Import.URL != "" { - dir := filepath.Join(cachePath, "components", child.Name) + dir := filepath.Join(cachePath, config.ZarfComponentsDir, child.Name) componentTarball := fmt.Sprintf("%s.tar", dir) - parent.Import.Path = filepath.Join(parent.Import.Path, "components", child.Name) + parent.Import.Path = filepath.Join(parent.Import.Path, config.ZarfComponentsDir, child.Name) if !utils.InvalidPath(componentTarball) { if !utils.InvalidPath(dir) { err = os.RemoveAll(dir) @@ -159,7 +168,7 @@ func (p *Packager) getChildComponent(parent types.ZarfComponent, pathAncestry st return child, fmt.Errorf("unable to remove composed component cache path %s: %w", cachePath, err) } } - err = archiver.Unarchive(componentTarball, filepath.Join(cachePath, "components")) + err = archiver.Unarchive(componentTarball, filepath.Join(cachePath, config.ZarfComponentsDir)) if err != nil { return child, fmt.Errorf("unable to unpack composed component tarball: %w", err) } diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 62be1ddd68..a96a0ac599 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -6,7 +6,6 @@ package packager import ( "crypto" - "encoding/json" "errors" "fmt" "os" @@ -25,6 +24,7 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/sbom" "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" @@ -41,6 +41,17 @@ func (p *Packager) Create(baseDir string) error { return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) } + if utils.IsOCIURL(p.cfg.CreateOpts.Output) { + ref, err := oci.ReferenceFromMetadata(p.cfg.CreateOpts.Output, &p.cfg.Pkg.Metadata, p.cfg.Pkg.Build.Architecture) + if err != nil { + return err + } + err = p.SetOCIRemote(ref.String()) + if err != nil { + return err + } + } + // Load the images and repos from the 'reference' package if err := p.loadDifferentialData(); err != nil { return err @@ -150,7 +161,7 @@ func (p *Packager) Create(baseDir string) error { // Images are handled separately from other component assets. if len(imgList) > 0 { - message.HeaderInfof("📦 COMPONENT IMAGES") + message.HeaderInfof("📦 PACKAGE IMAGES") doPull := func() error { imgConfig := images.ImgConfig{ @@ -213,57 +224,21 @@ func (p *Packager) Create(baseDir string) error { } } - // Use the output path if the user specified it. - packageName := filepath.Join(p.cfg.CreateOpts.OutputDirectory, p.GetPackageName()) - - // Try to remove the package if it already exists. - _ = os.Remove(packageName) - - // Make the archive - archiveSrc := []string{p.tmp.Base + string(os.PathSeparator)} - if err := archiver.Archive(archiveSrc, packageName); err != nil { - return fmt.Errorf("unable to create package: %w", err) - } - - f, err := os.Stat(packageName) - if err != nil { - return fmt.Errorf("unable to read the package archive: %w", err) - } - - // Convert Megabytes to bytes. - chunkSize := p.cfg.CreateOpts.MaxPackageSizeMB * 1000 * 1000 - - // If a chunk size was specified and the package is larger than the chunk size, split it into chunks. - if p.cfg.CreateOpts.MaxPackageSizeMB > 0 && f.Size() > int64(chunkSize) { - chunks, sha256sum, err := utils.SplitFile(packageName, chunkSize) - if err != nil { - return fmt.Errorf("unable to split the package archive into multiple files: %w", err) - } - if len(chunks) > 999 { - return fmt.Errorf("unable to split the package archive into multiple files: must be less than 1,000 files") - } - - message.Infof("Package split into %d files, original sha256sum is %s", len(chunks)+1, sha256sum) - _ = os.RemoveAll(packageName) - - // Marshal the data into a json file. - jsonData, err := json.Marshal(types.ZarfPartialPackageData{ - Count: len(chunks), - Bytes: f.Size(), - Sha256Sum: sha256sum, - }) + if utils.IsOCIURL(p.cfg.CreateOpts.Output) { + err := p.remote.PublishPackage(&p.cfg.Pkg, p.tmp.Base, config.CommonOptions.OCIConcurrency) if err != nil { - return fmt.Errorf("unable to marshal the partial package data: %w", err) + return fmt.Errorf("unable to publish package: %w", err) } + } else { + // Use the output path if the user specified it. + packageName := filepath.Join(p.cfg.CreateOpts.Output, p.GetPackageName()) - // Prepend the json data to the first chunk. - chunks = append([][]byte{jsonData}, chunks...) + // Try to remove the package if it already exists. + _ = os.Remove(packageName) - for idx, chunk := range chunks { - path := fmt.Sprintf("%s.part%03d", packageName, idx) - if err := os.WriteFile(path, chunk, 0644); err != nil { - return fmt.Errorf("unable to write the file %s: %w", path, err) - } + // Create the package tarball. + if err := p.archivePackage(p.tmp.Base, packageName); err != nil { + return fmt.Errorf("unable to archive package: %w", err) } } @@ -601,8 +576,21 @@ func (p *Packager) loadDifferentialData() error { // Load the package spec of the package we're using as a 'reference' for the differential build if utils.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) { - if err := p.pullPackageLayers(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, tmpDir, []string{config.ZarfYAML}); err != nil { - return fmt.Errorf("unable to pull the differential zarf package spec: %s", err.Error()) + err := p.SetOCIRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) + if err != nil { + return err + } + manifest, err := p.remote.FetchRoot() + if err != nil { + return err + } + pkg, err := p.remote.FetchZarfYAML(manifest) + if err != nil { + return err + } + err = utils.WriteYaml(filepath.Join(tmpDir, config.ZarfYAML), pkg, 0600) + if err != nil { + return err } } else { if err := archiver.Extract(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, config.ZarfYAML, tmpDir); err != nil { diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 46809709e1..9440eb5bf2 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -40,6 +40,13 @@ var ( func (p *Packager) Deploy() error { message.Debug("packager.Deploy()") + if utils.IsOCIURL(p.cfg.DeployOpts.PackagePath) { + err := p.SetOCIRemote(p.cfg.DeployOpts.PackagePath) + if err != nil { + return err + } + } + if err := p.loadZarfPkg(); err != nil { return fmt.Errorf("unable to load the Zarf Package: %w", err) } @@ -100,7 +107,6 @@ func (p *Packager) Deploy() error { // deployComponents loops through a list of ZarfComponents and deploys them. func (p *Packager) deployComponents() (deployedComponents []types.DeployedComponent, err error) { componentsToDeploy := p.getValidComponents() - config.SetDeployingComponents(deployedComponents) // Generate a value template if valueTemplate, err = template.Generate(p.cfg); err != nil { @@ -132,7 +138,6 @@ func (p *Packager) deployComponents() (deployedComponents []types.DeployedCompon // Deploy the component deployedComponent.InstalledCharts = charts deployedComponents = append(deployedComponents, deployedComponent) - config.SetDeployingComponents(deployedComponents) // Save deployed package information to k8s // Note: Not all packages need k8s; check if k8s is being used before saving the secret @@ -149,7 +154,6 @@ func (p *Packager) deployComponents() (deployedComponents []types.DeployedCompon } } - config.ClearDeployingComponents() return deployedComponents, nil } diff --git a/src/pkg/packager/inspect.go b/src/pkg/packager/inspect.go index ee8bfa0a26..df623c7376 100644 --- a/src/pkg/packager/inspect.go +++ b/src/pkg/packager/inspect.go @@ -6,83 +6,84 @@ package packager import ( "fmt" - "os" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/sbom" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/mholt/archiver/v3" - "github.com/pterm/pterm" ) // Inspect list the contents of a package. func (p *Packager) Inspect(includeSBOM bool, outputSBOM string, inspectPublicKey string) error { + wantSBOM := includeSBOM || outputSBOM != "" + + requestedFiles := []string{config.ZarfYAML} + if wantSBOM { + requestedFiles = append(requestedFiles, config.ZarfSBOMTar) + } + // Handle OCI packages that have been published to a registry if utils.IsOCIURL(p.cfg.DeployOpts.PackagePath) { - // Download all the layers we need - pullSBOM := includeSBOM || outputSBOM != "" - pullZarfSig := inspectPublicKey != "" + message.Debugf("Pulling layers %v from %s", requestedFiles, p.cfg.DeployOpts.PackagePath) - layersToPull := []string{config.ZarfYAML} - if pullSBOM { - layersToPull = append(layersToPull, config.ZarfSBOMTar) + err := p.SetOCIRemote(p.cfg.DeployOpts.PackagePath) + if err != nil { + return err } - if pullZarfSig { - layersToPull = append(layersToPull, config.ZarfYAMLSignature) + layersToPull, err := p.remote.LayersFromPaths(requestedFiles) + if err != nil { + return err } - - message.Debugf("Pulling layers %v from %s", layersToPull, p.cfg.DeployOpts.PackagePath) - if err := p.pullPackageLayers(p.cfg.DeployOpts.PackagePath, p.tmp.Base, layersToPull); err != nil { - return fmt.Errorf("unable to pull layers for inspect: %w", err) + if err := p.remote.PullPackage(p.tmp.Base, config.CommonOptions.OCIConcurrency, layersToPull...); err != nil { + return fmt.Errorf("unable to pull the package: %w", err) } if err := p.readYaml(p.tmp.ZarfYaml); err != nil { return fmt.Errorf("unable to read the zarf.yaml in %s: %w", p.tmp.Base, err) } } else { // This package exists on the local file system - extract the first layer of the tarball - if err := archiver.Unarchive(p.cfg.DeployOpts.PackagePath, p.tmp.Base); err != nil { - return fmt.Errorf("unable to extract the package: %w", err) + if err := archiver.Extract(p.cfg.DeployOpts.PackagePath, config.ZarfChecksumsTxt, p.tmp.Base); err != nil { + return fmt.Errorf("unable to extract %s: %w", config.ZarfChecksumsTxt, err) + } + if err := archiver.Extract(p.cfg.DeployOpts.PackagePath, config.ZarfYAML, p.tmp.Base); err != nil { + return fmt.Errorf("unable to extract %s: %w", config.ZarfYAML, err) + } + if err := archiver.Extract(p.cfg.DeployOpts.PackagePath, config.ZarfYAMLSignature, p.tmp.Base); err != nil { + return fmt.Errorf("unable to extract %s: %w", config.ZarfYAMLSignature, err) } if err := p.readYaml(p.tmp.ZarfYaml); err != nil { return fmt.Errorf("unable to read the zarf.yaml in %s: %w", p.tmp.Base, err) } - + if wantSBOM { + if err := archiver.Extract(p.cfg.DeployOpts.PackagePath, config.ZarfSBOMTar, p.tmp.Base); err != nil { + return fmt.Errorf("unable to extract %s: %w", config.ZarfSBOMTar, err) + } + } } - pterm.Println() - pterm.Println() - utils.ColorPrintYAML(p.cfg.Pkg) - // Attempt to validate the checksums, or explain why we cannot validate them if !utils.IsOCIURL(p.cfg.DeployOpts.PackagePath) { - // If the package is not a remote OCI package, we can validate the checksums - if err := p.validatePackageChecksums(); err != nil { - message.Warnf("Unable to validate the package checksums, the package may have been tampered with: %s", err.Error()) + if err := utils.ValidatePackageChecksums(p.tmp.Base, requestedFiles); err != nil { + return fmt.Errorf("unable to validate the package checksums, the package may have been tampered with: %s", err.Error()) } - } else { - message.Warnf("Zarf is unable to validate the checksums of remote OCI packages. We are unable to determine the integrity of the package without downloading the entire package.") } // Validate the package checksums and signatures if specified, and warn if the package was signed but a key was not provided - _, sigExistErr := os.Stat(p.tmp.ZarfSig) - if inspectPublicKey != "" { - if err := p.validatePackageSignature(inspectPublicKey); err != nil { + if err := p.validatePackageSignature(inspectPublicKey); err != nil { + if err == ErrPkgSigButNoKey { + message.Warn("The package was signed but no public key was provided, skipping signature validation") + } else { return fmt.Errorf("unable to validate the package signature: %w", err) } - } else if sigExistErr == nil { - message.Warnf("The package you are inspecting has been signed but a public key was not provided.") } - if includeSBOM || outputSBOM != "" { + if wantSBOM { // Extract the SBOM files from the sboms.tar file - _, tarErr := os.Stat(p.tmp.SbomTar) - if tarErr == nil { - if err := archiver.Unarchive(p.tmp.SbomTar, p.tmp.Sboms); err != nil { - return fmt.Errorf("unable to extract the SBOM files: %w", err) - } + if err := archiver.Unarchive(p.tmp.SbomTar, p.tmp.Sboms); err != nil { + return fmt.Errorf("unable to extract the SBOM files: %w", err) } } diff --git a/src/pkg/packager/network.go b/src/pkg/packager/network.go index ad1a0be75e..d2f091785d 100644 --- a/src/pkg/packager/network.go +++ b/src/pkg/packager/network.go @@ -4,7 +4,6 @@ import ( "context" "crypto/sha256" "encoding/hex" - "encoding/json" "fmt" "io" "net/http" @@ -12,16 +11,11 @@ import ( "os" "path/filepath" "strings" - "sync" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "oras.land/oras-go/v2" - "oras.land/oras-go/v2/content" - "oras.land/oras-go/v2/content/file" - "oras.land/oras-go/v2/registry" ) // handlePackagePath If provided package is a URL download it to a temp directory. @@ -39,9 +33,18 @@ func (p *Packager) handlePackagePath() error { // Handle case where deploying remote package stored in an OCI registry if utils.IsOCIURL(opts.PackagePath) { - ociURL := opts.PackagePath p.cfg.DeployOpts.PackagePath = p.tmp.Base - return p.handleOciPackage(ociURL, p.tmp.Base, p.cfg.PublishOpts.CopyOptions.Concurrency) + requestedComponents := getRequestedComponentList(p.cfg.DeployOpts.Components) + layersToPull := []ocispec.Descriptor{} + // only pull specified components and their images if --components AND --confirm are set + if len(requestedComponents) > 0 && config.CommonOptions.Confirm { + layers, err := p.remote.LayersFromRequestedComponents(requestedComponents) + if err != nil { + return fmt.Errorf("unable to get published component image layers: %s", err.Error()) + } + layersToPull = append(layersToPull, layers...) + } + return p.remote.PullPackage(p.tmp.Base, config.CommonOptions.OCIConcurrency, layersToPull...) } // Handle case where deploying remote package validated via sget @@ -130,145 +133,3 @@ func (p *Packager) handleSgetPackage() error { spinner.Success() return nil } - -func (p *Packager) handleOciPackage(url string, out string, concurrency int, layers ...string) error { - message.Debugf("packager.handleOciPackage(%s, %s, %d, %s)", url, out, concurrency, layers) - ref, err := registry.ParseReference(strings.TrimPrefix(url, utils.OCIURLPrefix)) - if err != nil { - return fmt.Errorf("failed to parse OCI reference: %w", err) - } - - message.Debugf("Pulling %s", ref.String()) - message.Infof("Pulling Zarf package from %s", ref) - - src, err := utils.NewOrasRemote(ref) - if err != nil { - return err - } - - estimatedBytes, err := getOCIPackageSize(src, layers...) - if err != nil { - return err - } - - dst, err := file.New(out) - if err != nil { - return err - } - defer dst.Close() - - copyOpts := oras.DefaultCopyOptions - copyOpts.Concurrency = concurrency - copyOpts.OnCopySkipped = func(ctx context.Context, desc ocispec.Descriptor) error { - title := desc.Annotations[ocispec.AnnotationTitle] - var format string - if title != "" { - format = fmt.Sprintf("%s %s", desc.Digest.Encoded()[:12], utils.First30last30(title)) - } else { - format = fmt.Sprintf("%s [%s]", desc.Digest.Encoded()[:12], desc.MediaType) - } - message.Successf(format) - return nil - } - copyOpts.PostCopy = copyOpts.OnCopySkipped - isPartialPull := len(layers) > 0 - if isPartialPull { - alwaysPull := []string{config.ZarfYAML, config.ZarfChecksumsTxt, config.ZarfYAMLSignature} - layers = append(layers, alwaysPull...) - copyOpts.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - nodes, err := content.Successors(ctx, fetcher, desc) - if err != nil { - return nil, err - } - var ret []ocispec.Descriptor - for _, node := range nodes { - if utils.SliceContains(layers, node.Annotations[ocispec.AnnotationTitle]) { - ret = append(ret, node) - } - } - return ret, nil - } - } - - // Create a thread to update a progress bar as we save the package to disk - doneSaving := make(chan int) - var wg sync.WaitGroup - wg.Add(1) - go utils.RenderProgressBarForLocalDirWrite(out, estimatedBytes, &wg, doneSaving, "Pulling Zarf package data") - _, err = oras.Copy(src.Context, src.Repository, ref.Reference, dst, ref.Reference, copyOpts) - if err != nil { - return err - } - - // Send a signal to the progress bar that we're done and wait for it to finish - doneSaving <- 1 - wg.Wait() - - message.Debugf("Pulled %s", ref.String()) - message.Successf("Pulled %s", ref.String()) - - return nil -} - -func getOCIPackageSize(src *utils.OrasRemote, layers ...string) (int64, error) { - var total int64 - - manifest, err := getManifest(src) - if err != nil { - return 0, err - } - - manifestLayers := manifest.Layers - - processedLayers := make(map[string]bool) - for _, layer := range manifestLayers { - // Only include this layer's size if we haven't already processed it - hasBeenProcessed := processedLayers[layer.Digest.String()] - if !hasBeenProcessed { - if len(layers) > 0 { - // If we're only pulling a subset of layers, only include the size of the layers we're pulling - if utils.SliceContains(layers, layer.Annotations[ocispec.AnnotationTitle]) { - total += layer.Size - processedLayers[layer.Digest.String()] = true - continue - } - } - total += layer.Size - processedLayers[layer.Digest.String()] = true - } - } - - return total, nil -} - -// getManifest fetches the manifest from a Zarf OCI package -func getManifest(dst *utils.OrasRemote) (*ocispec.Manifest, error) { - // get the manifest descriptor - // ref.Reference can be a tag or a digest - descriptor, err := dst.Resolve(dst.Context, dst.Reference.Reference) - if err != nil { - return nil, err - } - - // get the manifest itself - pulled, err := content.FetchAll(dst.Context, dst, descriptor) - if err != nil { - return nil, err - } - manifest := ocispec.Manifest{} - - if err = json.Unmarshal(pulled, &manifest); err != nil { - return nil, err - } - return &manifest, nil -} - -// pullLayer fetches a single layer from a Zarf OCI package -func pullLayer(dst *utils.OrasRemote, desc ocispec.Descriptor, out string) error { - bytes, err := content.FetchAll(dst.Context, dst, desc) - if err != nil { - return err - } - err = utils.WriteFile(out, bytes) - return err -} diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 20bffa76c1..9befea343d 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -5,32 +5,15 @@ package packager import ( - "bytes" - "context" - "encoding/json" - "errors" "fmt" "os" "path/filepath" - "strings" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/types" "github.com/mholt/archiver/v3" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - - "oras.land/oras-go/v2" - "oras.land/oras-go/v2/content" - "oras.land/oras-go/v2/content/file" - "oras.land/oras-go/v2/registry" -) - -// ZarfLayerMediaTypeBlob is the media type for all Zarf layers due to the range of possible content -const ( - ZarfLayerMediaTypeBlob = "application/vnd.zarf.layer.v1.blob" - skeletonSuffix = "skeleton" ) // Publish publishes the package to a registry @@ -40,18 +23,16 @@ const ( // // Authentication is handled via the Docker config file created w/ `zarf tools registry login` func (p *Packager) Publish() error { - p.cfg.DeployOpts.PackagePath = p.cfg.PublishOpts.PackagePath - var ref registry.Reference - referenceSuffix := "" + var referenceSuffix string if utils.IsDir(p.cfg.PublishOpts.PackagePath) { - referenceSuffix = skeletonSuffix + referenceSuffix = oci.SkeletonSuffix err := p.loadSkeleton() if err != nil { return err } } else { // Extract the first layer of the tarball - if err := archiver.Unarchive(p.cfg.DeployOpts.PackagePath, p.tmp.Base); err != nil { + if err := archiver.Unarchive(p.cfg.PublishOpts.PackagePath, p.tmp.Base); err != nil { return fmt.Errorf("unable to extract the package: %w", err) } @@ -63,12 +44,17 @@ func (p *Packager) Publish() error { } // Get a reference to the registry for this package - ref, err := p.ref(referenceSuffix) + ref, err := oci.ReferenceFromMetadata(p.cfg.PublishOpts.PackageDestination, &p.cfg.Pkg.Metadata, referenceSuffix) + if err != nil { + return err + } + + err = p.SetOCIRemote(ref.String()) if err != nil { return err } - if err := p.validatePackageChecksums(); err != nil { + if err := utils.ValidatePackageChecksums(p.tmp.Base, nil); err != nil { return fmt.Errorf("unable to publish package because checksums do not match: %w", err) } @@ -80,200 +66,10 @@ func (p *Packager) Publish() error { } } - message.HeaderInfof("📦 PACKAGE PUBLISH %s:%s", p.cfg.Pkg.Metadata.Name, ref.Reference) - return p.publish(ref) -} - -func (p *Packager) publish(ref registry.Reference) error { - message.Infof("Publishing package to %s", ref) - spinner := message.NewProgressSpinner("") - defer spinner.Stop() - - // Get all of the layers in the package - paths := []string{} - err := filepath.Walk(p.tmp.Base, func(path string, info os.FileInfo, err error) error { - // Catch any errors that happened during the walk - if err != nil { - return err - } - - // Add any resource that is not a directory to the paths of objects we will include into the package - if !info.IsDir() { - paths = append(paths, path) - } - return nil - }) - if err != nil { - return fmt.Errorf("unable to get the layers in the package to publish: %w", err) - } - - // destination remote - dst, err := utils.NewOrasRemote(ref) - if err != nil { - return err - } - ctx := dst.Context - - // source file store - src, err := file.New(p.tmp.Base) - if err != nil { - return err - } - defer src.Close() - - var descs []ocispec.Descriptor - - for idx, path := range paths { - name, err := filepath.Rel(p.tmp.Base, path) - if err != nil { - return err - } - spinner.Updatef("Preparing layer %d/%d: %s", idx+1, len(paths), name) - - mediaType := ZarfLayerMediaTypeBlob - - desc, err := src.Add(ctx, name, mediaType, path) - if err != nil { - return err - } - descs = append(descs, desc) - } - spinner.Successf("Prepared %d layers", len(descs)) - - copyOpts := oras.DefaultCopyOptions - copyOpts.Concurrency = p.cfg.PublishOpts.CopyOptions.Concurrency - copyOpts.OnCopySkipped = utils.PrintLayerExists - copyOpts.PostCopy = utils.PrintLayerExists - - root, err := p.publishImage(dst, src, descs, copyOpts) - if err != nil { - return err - } - - dst.Transport.ProgressBar.Successf("Published %s [%s]", ref, root.MediaType) - fmt.Println() - if strings.HasSuffix(ref.Reference, skeletonSuffix) { - message.Info("Example of importing components from this package:") - fmt.Println() - ex := []types.ZarfComponent{} - for _, c := range p.cfg.Pkg.Components { - ex = append(ex, types.ZarfComponent{ - Name: fmt.Sprintf("import-%s", c.Name), - Import: types.ZarfComponentImport{ - ComponentName: c.Name, - URL: fmt.Sprintf("oci://%s", ref.String()), - }, - }) - } - utils.ColorPrintYAML(ex) - fmt.Println() - } else { - flags := "" - if config.CommonOptions.Insecure { - flags = "--insecure" - } - message.Info("To inspect/deploy/pull:") - message.Infof("zarf package inspect oci://%s %s", ref, flags) - message.Infof("zarf package deploy oci://%s %s", ref, flags) - message.Infof("zarf package pull oci://%s %s", ref, flags) - } + message.HeaderInfof("📦 PACKAGE PUBLISH %s:%s", p.cfg.Pkg.Metadata.Name, p.remote.Reference.Reference) - return nil -} - -func (p *Packager) publishImage(dst *utils.OrasRemote, src *file.Store, descs []ocispec.Descriptor, copyOpts oras.CopyOptions) (root ocispec.Descriptor, err error) { - var total int64 - for _, desc := range descs { - total += desc.Size - } - // assumes referrers API is not supported since OCI artifact - // media type is not supported - dst.SetReferrersCapability(false) - - // fallback to an ImageManifest push - manifestConfigDesc, manifestConfigContent, err := p.generateManifestConfigFile() - if err != nil { - return root, err - } - // push the manifest config - // since this config is so tiny, and the content is not used again - // it is not logged to the progress, but will error if it fails - err = dst.Push(dst.Context, manifestConfigDesc, bytes.NewReader(manifestConfigContent)) - if err != nil { - return root, err - } - packOpts := p.cfg.PublishOpts.PackOptions - packOpts.ConfigDescriptor = &manifestConfigDesc - packOpts.PackImageManifest = true - root, err = p.pack(dst.Context, ocispec.MediaTypeImageManifest, descs, src, packOpts) - if err != nil { - return root, err - } - total += root.Size + manifestConfigDesc.Size - - dst.Transport.ProgressBar = message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", dst.Reference.Repository, dst.Reference.Reference)) - defer dst.Transport.ProgressBar.Stop() - // attempt to push the image manifest - _, err = oras.Copy(dst.Context, src, root.Digest.String(), dst, dst.Reference.Reference, copyOpts) - if err != nil { - return root, err - } - - return root, nil -} - -func (p *Packager) generateAnnotations() map[string]string { - annotations := map[string]string{ - ocispec.AnnotationDescription: p.cfg.Pkg.Metadata.Description, - } - - if url := p.cfg.Pkg.Metadata.URL; url != "" { - annotations[ocispec.AnnotationURL] = url - } - if authors := p.cfg.Pkg.Metadata.Authors; authors != "" { - annotations[ocispec.AnnotationAuthors] = authors - } - if documentation := p.cfg.Pkg.Metadata.Documentation; documentation != "" { - annotations[ocispec.AnnotationDocumentation] = documentation - } - if source := p.cfg.Pkg.Metadata.Source; source != "" { - annotations[ocispec.AnnotationSource] = source - } - if vendor := p.cfg.Pkg.Metadata.Vendor; vendor != "" { - annotations[ocispec.AnnotationVendor] = vendor - } - - return annotations -} - -func (p *Packager) generateManifestConfigFile() (ocispec.Descriptor, []byte, error) { - // Unless specified, an empty manifest config will be used: `{}` - // which causes an error on Google Artifact Registry - // to negate this, we create a simple manifest config with some build metadata - // the contents of this file are not used by Zarf - type OCIConfigPartial struct { - Architecture string `json:"architecture"` - OCIVersion string `json:"ociVersion"` - Annotations map[string]string `json:"annotations,omitempty"` - } - - annotations := map[string]string{ - ocispec.AnnotationTitle: p.cfg.Pkg.Metadata.Name, - ocispec.AnnotationDescription: p.cfg.Pkg.Metadata.Description, - } - - manifestConfig := OCIConfigPartial{ - Architecture: p.cfg.Pkg.Build.Architecture, - OCIVersion: "1.0.1", - Annotations: annotations, - } - manifestConfigBytes, err := json.Marshal(manifestConfig) - if err != nil { - return ocispec.Descriptor{}, nil, err - } - manifestConfigDesc := content.NewDescriptorFromBytes("application/vnd.unknown.config.v1+json", manifestConfigBytes) - - return manifestConfigDesc, manifestConfigBytes, nil + // Publish the package/skeleton to the registry + return p.remote.PublishPackage(&p.cfg.Pkg, p.tmp.Base, config.CommonOptions.OCIConcurrency) } func (p *Packager) loadSkeleton() error { @@ -323,41 +119,3 @@ func (p *Packager) loadSkeleton() error { return p.writeYaml() } - -// pack creates an artifact/image manifest from the provided descriptors and pushes it to the store -func (p *Packager) pack(ctx context.Context, artifactType string, descs []ocispec.Descriptor, src *file.Store, packOpts oras.PackOptions) (ocispec.Descriptor, error) { - packOpts.ManifestAnnotations = p.generateAnnotations() - root, err := oras.Pack(ctx, src, artifactType, descs, packOpts) - if err != nil { - return ocispec.Descriptor{}, err - } - if err = src.Tag(ctx, root, root.Digest.String()); err != nil { - return ocispec.Descriptor{}, err - } - - return root, nil -} - -// ref returns a registry.Reference using metadata from the package's build config and the PublishOpts -// -// if suffix is not empty, the architecture will be replaced with the suffix string -func (p *Packager) ref(suffix string) (registry.Reference, error) { - ver := p.cfg.Pkg.Metadata.Version - if len(ver) == 0 { - return registry.Reference{}, errors.New("version is required for publishing") - } - - ref := registry.Reference{ - Registry: p.cfg.PublishOpts.Reference.Registry, - Repository: fmt.Sprintf("%s/%s", p.cfg.PublishOpts.Reference.Repository, p.cfg.Pkg.Metadata.Name), - Reference: fmt.Sprintf("%s-%s", ver, suffix), - } - if len(p.cfg.PublishOpts.Reference.Repository) == 0 { - ref.Repository = p.cfg.Pkg.Metadata.Name - } - err := ref.Validate() - if err != nil { - return registry.Reference{}, err - } - return ref, nil -} diff --git a/src/pkg/packager/pull.go b/src/pkg/packager/pull.go index ce3181b07f..6c65cd92c1 100644 --- a/src/pkg/packager/pull.go +++ b/src/pkg/packager/pull.go @@ -12,15 +12,18 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/mholt/archiver/v3" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "oras.land/oras-go/v2/registry" ) // Pull pulls a Zarf package and saves it as a compressed tarball. func (p *Packager) Pull() error { - err := p.handleOciPackage(p.cfg.DeployOpts.PackagePath, p.tmp.Base, p.cfg.PullOpts.CopyOptions.Concurrency) + err := p.SetOCIRemote(p.cfg.PullOpts.PackageSource) + if err != nil { + return err + } + err = p.remote.PullPackage(p.tmp.Base, config.CommonOptions.OCIConcurrency) if err != nil { return err } @@ -35,10 +38,8 @@ func (p *Packager) Pull() error { message.Successf("Package signature is valid") } - if p.cfg.Pkg.Metadata.AggregateChecksum != "" { - if err = p.validatePackageChecksums(); err != nil { - return fmt.Errorf("unable to validate the package checksums: %w", err) - } + if err = utils.ValidatePackageChecksums(p.tmp.Base, nil); err != nil { + return fmt.Errorf("unable to validate the package checksums: %w", err) } // Get all the layers from within the temp directory @@ -48,7 +49,7 @@ func (p *Packager) Pull() error { } var name string - if strings.HasSuffix(p.cfg.DeployOpts.PackagePath, skeletonSuffix) { + if strings.HasSuffix(p.cfg.PullOpts.PackageSource, oci.SkeletonSuffix) { name = fmt.Sprintf("zarf-package-%s-skeleton-%s.tar.zst", p.cfg.Pkg.Metadata.Name, p.cfg.Pkg.Metadata.Version) } else { name = fmt.Sprintf("zarf-package-%s-%s-%s.tar.zst", p.cfg.Pkg.Metadata.Name, p.cfg.Pkg.Build.Architecture, p.cfg.Pkg.Metadata.Version) @@ -61,36 +62,3 @@ func (p *Packager) Pull() error { } return nil } - -// pullPackageSpecLayer pulls the `zarf.yaml` and `zarf.yaml.sig` (if it exists) layers from the published package -func (p *Packager) pullPackageLayers(packagePath string, targetDir string, layersToPull []string) error { - ref, err := registry.ParseReference(strings.TrimPrefix(packagePath, utils.OCIURLPrefix)) - if err != nil { - return err - } - - dst, err := utils.NewOrasRemote(ref) - if err != nil { - return err - } - - // get the manifest - manifest, err := getManifest(dst) - if err != nil { - return err - } - layers := manifest.Layers - - for _, layerToPull := range layersToPull { - layerDesc := utils.Find(layers, func(d ocispec.Descriptor) bool { - return d.Annotations[ocispec.AnnotationTitle] == layerToPull - }) - if len(layerDesc.Digest) == 0 { - return fmt.Errorf("unable to find layer (%s) from the OCI package %s", layerToPull, packagePath) - } - if err := pullLayer(dst, layerDesc, filepath.Join(targetDir, layerToPull)); err != nil { - return fmt.Errorf("unable to pull the layer (%s) from the OCI package %s", layerToPull, packagePath) - } - } - return nil -} diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index 4013cfb43b..314ee5b520 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -7,7 +7,6 @@ package packager import ( "encoding/json" "fmt" - "strings" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/cluster" @@ -23,8 +22,24 @@ func (p *Packager) Remove(packageName string) (err error) { spinner := message.NewProgressSpinner("Removing zarf package %s", packageName) defer spinner.Stop() + if utils.IsOCIURL(packageName) { + err := p.SetOCIRemote(packageName) + if err != nil { + message.Fatalf(err, "Unable to set OCI remote: %s", err.Error()) + } + + // pull the package from the remote + if err = p.remote.PullPackageMetadata(p.tmp.Base); err != nil { + return fmt.Errorf("unable to pull the package from the remote: %w", err) + } + if err := p.readYaml(p.tmp.ZarfYaml); err != nil { + return err + } + packageName = p.cfg.Pkg.Metadata.Name + } + // If components were provided; just remove the things we were asked to remove - requestedComponents := strings.Split(p.cfg.DeployOpts.Components, ",") + requestedComponents := getRequestedComponentList(p.cfg.DeployOpts.Components) partialRemove := len(requestedComponents) > 0 && requestedComponents[0] != "" // Determine if we need the cluster @@ -71,7 +86,7 @@ func (p *Packager) Remove(packageName string) (err error) { deployedPackage, err = p.cluster.GetDeployedPackage(packageName) if err != nil { - return fmt.Errorf("unable to load the secret for the package we are attempting to remove: %w", err) + return fmt.Errorf("unable to load the secret for the package we are attempting to remove: %s", err.Error()) } } else { // If we do not need the cluster, create a deployed components object based on the info we have diff --git a/src/pkg/utils/checksum.go b/src/pkg/utils/checksum.go new file mode 100644 index 0000000000..a6e3592500 --- /dev/null +++ b/src/pkg/utils/checksum.go @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package utils provides generic helper functions. +package utils + +import ( + "bufio" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/types" +) + +// ValidatePackageChecksums validates the checksums of a Zarf package. +func ValidatePackageChecksums(baseDir string, pathsToCheck []string) error { + spinner := message.NewProgressSpinner("Validating package checksums") + defer spinner.Stop() + + // Run pre-checks to make sure we have what we need to validate the checksums + if InvalidPath(baseDir) { + return fmt.Errorf("invalid base directory: %s", baseDir) + } + var pkg types.ZarfPackage + err := ReadYaml(filepath.Join(baseDir, config.ZarfYAML), &pkg) + if err != nil { + return err + } + aggregateChecksum := pkg.Metadata.AggregateChecksum + if aggregateChecksum == "" { + spinner.Updatef("Package does not have an aggregate checksum, skipping checksums validation") + return nil + } + if len(aggregateChecksum) != 64 { + return fmt.Errorf("invalid aggregate checksum: %s", aggregateChecksum) + } + isPartial := false + if len(pathsToCheck) > 0 { + pathsToCheck = Unique(pathsToCheck) + isPartial = true + message.Debugf("Validating checksums for a subset of files in the package - %v", pathsToCheck) + for idx, path := range pathsToCheck { + pathsToCheck[idx] = filepath.Join(baseDir, path) + } + } + + checkedMap, err := PathCheckMap(baseDir) + if err != nil { + return err + } + + checksumPath := filepath.Join(baseDir, config.ZarfChecksumsTxt) + actualAggregateChecksum, err := GetSHA256OfFile(checksumPath) + if err != nil { + return fmt.Errorf("unable to get checksum of: %s", err.Error()) + } + if actualAggregateChecksum != aggregateChecksum { + return fmt.Errorf("invalid aggregate checksum: (expected: %s, received: %s)", aggregateChecksum, actualAggregateChecksum) + } + + checkedMap[filepath.Join(baseDir, config.ZarfChecksumsTxt)] = true + checkedMap[filepath.Join(baseDir, config.ZarfYAML)] = true + checkedMap[filepath.Join(baseDir, config.ZarfYAMLSignature)] = true + + err = LineByLine(checksumPath, func(line string) error { + split := strings.Split(line, " ") + sha := split[0] + rel := split[1] + if sha == "" || rel == "" { + return fmt.Errorf("invalid checksum line: %s", line) + } + path := filepath.Join(baseDir, rel) + + status := fmt.Sprintf("Validating checksum of %s", rel) + spinner.Updatef(message.Truncate(status, message.TermWidth, false)) + + if InvalidPath(path) { + if !isPartial && !checkedMap[path] { + return fmt.Errorf("unable to validate checksums - missing file: %s", rel) + } else if SliceContains(pathsToCheck, path) { + return fmt.Errorf("unable to validate partial checksums - missing file: %s", rel) + } + // it's okay if we're doing a partial check and the file isn't there as long as the path isn't in the list of paths to check + return nil + } + + actualSHA, err := GetSHA256OfFile(path) + if err != nil { + return fmt.Errorf("unable to get checksum of: %s", err.Error()) + } + + if sha != actualSHA { + return fmt.Errorf("invalid checksum for %s: (expected: %s, received: %s)", path, sha, actualSHA) + } + + checkedMap[path] = true + + return nil + }) + if err != nil { + return err + } + + // If we're doing a partial check, make sure we've checked all the files we were asked to check + if isPartial { + for _, path := range pathsToCheck { + if !checkedMap[path] { + return fmt.Errorf("unable to validate partial checksums, %s did not get checked", path) + } + } + } else { + // Otherwise, make sure we've checked all the files in the package + for path, checked := range checkedMap { + if !checked { + return fmt.Errorf("unable to validate checksums, %s did not get checked", path) + } + } + } + + spinner.Successf("Checksums validated!") + return nil +} + +// PathCheckMap returns a map of all the files in a directory and a boolean to use for checking status. +func PathCheckMap(dir string) (map[string]bool, error) { + filepathMap := make(map[string]bool) + err := filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error { + if info.IsDir() { + return nil + } + filepathMap[path] = false + return nil + }) + return filepathMap, err +} + +// LineByLine reads a file line by line and calls a callback function for each line. +func LineByLine(path string, cb func(line string) error) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + // Read line by line + scanner := bufio.NewScanner(file) + scanner.Split(bufio.ScanLines) + for scanner.Scan() { + err := cb(scanner.Text()) + if err != nil { + return err + } + } + return nil +} diff --git a/src/pkg/utils/credentials.go b/src/pkg/utils/credentials.go index 64da77c89c..e9854e71e4 100644 --- a/src/pkg/utils/credentials.go +++ b/src/pkg/utils/credentials.go @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package utils provides generic helper functions package utils import ( diff --git a/src/pkg/utils/client.go b/src/pkg/utils/transport.go similarity index 100% rename from src/pkg/utils/client.go rename to src/pkg/utils/transport.go diff --git a/src/test/common.go b/src/test/common.go index 80a22856c6..5b0598a172 100644 --- a/src/test/common.go +++ b/src/test/common.go @@ -10,6 +10,7 @@ import ( "os" "regexp" "runtime" + "strings" "testing" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -25,6 +26,7 @@ type ZarfE2ETest struct { Arch string ApplianceMode bool RunClusterTests bool + CommandLog []string } var logRegex = regexp.MustCompile(`Saving log file to (?P.*?\.log)`) @@ -58,6 +60,7 @@ func (e2e *ZarfE2ETest) SetupWithCluster(t *testing.T) { // Zarf executes a Zarf command. func (e2e *ZarfE2ETest) Zarf(args ...string) (string, string, error) { + e2e.CommandLog = append(e2e.CommandLog, strings.Join(args, " ")) return exec.CmdWithContext(context.TODO(), exec.PrintCfg(), e2e.ZarfBinPath, args...) } diff --git a/src/test/e2e/30_config_file_test.go b/src/test/e2e/30_config_file_test.go index 7cbba5fb38..4ed4b4e45a 100644 --- a/src/test/e2e/30_config_file_test.go +++ b/src/test/e2e/30_config_file_test.go @@ -123,7 +123,7 @@ func configFileDefaultTests(t *testing.T) { } packageCreateFlags := []string{ - "create.output_directory: 52d061d5", + "create.output: 52d061d5", "Skip generating SBOM for this package (default true)", "[thing1=1a2b3c4d]", "Specify the maximum size of the package in megabytes, packages larger than this will be split into multiple parts. Use 0 to disable splitting. (default 42)", diff --git a/src/test/e2e/50_oci_package_test.go b/src/test/e2e/50_oci_package_test.go index 8e5bf98f41..c088fd58b7 100644 --- a/src/test/e2e/50_oci_package_test.go +++ b/src/test/e2e/50_oci_package_test.go @@ -9,7 +9,6 @@ import ( "path/filepath" "testing" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/exec" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -19,7 +18,6 @@ import ( type RegistryClientTestSuite struct { suite.Suite *require.Assertions - Remote *utils.OrasRemote Reference registry.Reference PackagesDir string } @@ -42,10 +40,7 @@ func (suite *RegistryClientTestSuite) TearDownSuite() { local := fmt.Sprintf("zarf-package-helm-charts-%s-0.0.1.tar.zst", e2e.Arch) e2e.CleanFiles(local) - stdOut, stdErr, err := e2e.Zarf("package", "remove", "helm-charts", "--confirm") - suite.NoError(err, stdOut, stdErr) - - _, _, err = exec.Cmd("docker", "rm", "-f", "registry") + _, _, err := exec.Cmd("docker", "rm", "-f", "registry") suite.NoError(err) } @@ -101,6 +96,10 @@ func (suite *RegistryClientTestSuite) Test_2_Deploy() { suite.NoError(err, stdOut, stdErr) suite.Contains(stdErr, "Pulled "+ref) + // Remove the package via OCI. + stdOut, stdErr, err = e2e.Zarf("package", "remove", "oci://"+ref, "--insecure", "--confirm") + suite.NoError(err, stdOut, stdErr) + // Test deploy w/ bad ref. _, stdErr, err = e2e.Zarf("package", "deploy", "oci://"+badRef.String(), "--insecure", "--confirm") suite.Error(err, stdErr) @@ -114,7 +113,6 @@ func (suite *RegistryClientTestSuite) Test_3_Inspect() { ref := suite.Reference.String() stdOut, stdErr, err := e2e.Zarf("package", "inspect", "oci://"+ref, "--insecure") suite.NoError(err, stdOut, stdErr) - suite.Contains(stdErr, "without downloading the entire package.") // Test inspect w/ bad ref. _, stdErr, err = e2e.Zarf("package", "inspect", "oci://"+badRef.String(), "--insecure") diff --git a/src/test/e2e/51_oci_compose_test.go b/src/test/e2e/51_oci_compose_test.go index 5e7a077695..3707952026 100644 --- a/src/test/e2e/51_oci_compose_test.go +++ b/src/test/e2e/51_oci_compose_test.go @@ -25,7 +25,6 @@ import ( type SkeletonSuite struct { suite.Suite *require.Assertions - Remote *utils.OrasRemote Reference registry.Reference } diff --git a/src/test/e2e/52_oci_compose_differential_test.go b/src/test/e2e/52_oci_compose_differential_test.go index 1d1682cd2a..60ec3d4108 100644 --- a/src/test/e2e/52_oci_compose_differential_test.go +++ b/src/test/e2e/52_oci_compose_differential_test.go @@ -23,7 +23,6 @@ import ( type OCIDifferentialSuite struct { suite.Suite *require.Assertions - Remote *utils.OrasRemote Reference registry.Reference tmpdir string } diff --git a/src/test/e2e/main_test.go b/src/test/e2e/main_test.go index 2b0aa7dba5..ee73c4c4bd 100644 --- a/src/test/e2e/main_test.go +++ b/src/test/e2e/main_test.go @@ -11,7 +11,9 @@ import ( "testing" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/test" + "github.com/pterm/pterm" ) var ( @@ -35,6 +37,7 @@ func TestMain(m *testing.M) { if err != nil { fmt.Println(err) //nolint:forbidigo } + os.Exit(retCode) } @@ -63,5 +66,16 @@ func doAllTheThings(m *testing.M) (int, error) { // Run the tests, with the cluster cleanup being deferred to the end of the function call returnCode := m.Run() + isCi := os.Getenv("CI") == "true" + if isCi { + pterm.Println("::notice::Zarf Command Log") + // Print out the command history + pterm.Println("::group::Zarf Command Log") + for _, cmd := range e2e.CommandLog { + message.ZarfCommand(cmd) + } + pterm.Println("::endgroup::") + } + return returnCode, nil } diff --git a/src/test/zarf-config-test.toml b/src/test/zarf-config-test.toml index 55bcca0fb4..a81de98fe9 100644 --- a/src/test/zarf-config-test.toml +++ b/src/test/zarf-config-test.toml @@ -28,7 +28,7 @@ url = 'registry.url: c0ac2e47' [package] [package.create] -output_directory = 'create.output_directory: 52d061d5' +output = 'create.output: 52d061d5' skip_sbom = true max_package_size = 42 diff --git a/src/types/runtime.go b/src/types/runtime.go index 92ee38361a..a13fdc7d4f 100644 --- a/src/types/runtime.go +++ b/src/types/runtime.go @@ -4,11 +4,6 @@ // Package types contains all the types used by Zarf. package types -import ( - "oras.land/oras-go/v2" - "oras.land/oras-go/v2/registry" -) - // Constants to keep track of folders within components const ( TempFolder = "temp" @@ -22,10 +17,11 @@ const ( // ZarfCommonOptions tracks the user-defined preferences used across commands. type ZarfCommonOptions struct { - Confirm bool `json:"confirm" jsonschema:"description=Verify that Zarf should perform an action"` - Insecure bool `json:"insecure" jsonschema:"description=Allow insecure connections for remote packages"` - CachePath string `json:"cachePath" jsonschema:"description=Path to use to cache images and git repos on package create"` - TempDirectory string `json:"tempDirectory" jsonschema:"description=Location Zarf should use as a staging ground when managing files and images for package creation and deployment"` + Confirm bool `json:"confirm" jsonschema:"description=Verify that Zarf should perform an action"` + Insecure bool `json:"insecure" jsonschema:"description=Allow insecure connections for remote packages"` + CachePath string `json:"cachePath" jsonschema:"description=Path to use to cache images and git repos on package create"` + TempDirectory string `json:"tempDirectory" jsonschema:"description=Location Zarf should use as a staging ground when managing files and images for package creation and deployment"` + OCIConcurrency int `jsonschema:"description=Number of concurrent layer operations to perform when interacting with a remote package"` } // ZarfDeployOptions tracks the user-defined preferences during a package deployment. @@ -41,22 +37,17 @@ type ZarfDeployOptions struct { // ZarfPublishOptions tracks the user-defined preferences during a package publish. type ZarfPublishOptions struct { - Reference registry.Reference `jsonschema:"description=Remote registry reference"` - CopyOptions oras.CopyOptions `jsonschema:"description=Options for the copy operation"` - PackOptions oras.PackOptions `jsonschema:"description=Options for the pack operation"` - PackagePath string `json:"packagePath" jsonschema:"description=Location where a Zarf package to publish can be found"` - SigningKeyPassword string `json:"signingKeyPassword" jsonschema:"description=Password to the private key signature file that will be used to sign the published package"` - SigningKeyPath string `json:"signingKeyPath" jsonschema:"description=Location where the private key component of a cosign key-pair can be found"` + PackageDestination string `json:"packageDestination" jsonschema:"description=Location where the Zarf package will be published to"` + PackagePath string `json:"packagePath" jsonschema:"description=Location where a Zarf package to publish can be found"` + SigningKeyPassword string `json:"signingKeyPassword" jsonschema:"description=Password to the private key signature file that will be used to sign the published package"` + SigningKeyPath string `json:"signingKeyPath" jsonschema:"description=Location where the private key component of a cosign key-pair can be found"` } // ZarfPullOptions tracks the user-defined preferences during a package pull. type ZarfPullOptions struct { - Reference registry.Reference `jsonschema:"description=Remote registry reference"` - CopyOptions oras.CopyOptions `jsonschema:"description=Options for the copy operation"` - PackOptions oras.PackOptions `jsonschema:"description=Options for the pack operation"` - PackagePath string `json:"packagePath" jsonschema:"description=Location where a Zarf package to publish can be found"` - OutputDirectory string `json:"outputDirectory" jsonschema:"description=Location where the pulled Zarf package will be placed"` - PublicKeyPath string `json:"publicKeyPath" jsonschema:"description=Location where the public key component of a cosign key-pair can be found"` + PackageSource string `json:"packageSource" jsonschema:"description=Location where the Zarf package will be pulled from"` + OutputDirectory string `json:"outputDirectory" jsonschema:"description=Location where the pulled Zarf package will be placed"` + PublicKeyPath string `json:"publicKeyPath" jsonschema:"description=Location where the public key component of a cosign key-pair can be found"` } // ZarfInitOptions tracks the user-defined options during cluster initialization. @@ -75,7 +66,7 @@ type ZarfInitOptions struct { // ZarfCreateOptions tracks the user-defined options used to create the package. type ZarfCreateOptions struct { SkipSBOM bool `json:"skipSBOM" jsonschema:"description=Disable the generation of SBOM materials during package creation"` - OutputDirectory string `json:"outputDirectory" jsonschema:"description=Location where the finalized Zarf package will be placed"` + Output string `json:"output" jsonschema:"description=Location where the finalized Zarf package will be placed"` ViewSBOM bool `json:"sbom" jsonschema:"description=Whether to pause to allow for viewing the SBOM post-creation"` SBOMOutputDir string `json:"sbomOutput" jsonschema:"description=Location to output an SBOM into after package creation"` SetVariables map[string]string `json:"setVariables" jsonschema:"description=Key-Value map of variable names and their corresponding values that will be used to template against the Zarf package being used"` diff --git a/src/ui/lib/api-types.ts b/src/ui/lib/api-types.ts index ac8b3a815d..cfd90dc07f 100644 --- a/src/ui/lib/api-types.ts +++ b/src/ui/lib/api-types.ts @@ -1128,6 +1128,10 @@ export interface ZarfCommonOptions { * Allow insecure connections for remote packages */ insecure: boolean; + /** + * Number of concurrent layer operations to perform when interacting with a remote package + */ + OCIConcurrency: number; /** * Location Zarf should use as a staging ground when managing files and images for package * creation and deployment @@ -1148,7 +1152,7 @@ export interface ZarfCreateOptions { /** * Location where the finalized Zarf package will be placed */ - outputDirectory: string; + output: string; /** * A map of domains to override on package create when pulling images */ @@ -1740,12 +1744,13 @@ const typeMap: any = { { json: "cachePath", js: "cachePath", typ: "" }, { json: "confirm", js: "confirm", typ: true }, { json: "insecure", js: "insecure", typ: true }, + { json: "OCIConcurrency", js: "OCIConcurrency", typ: 0 }, { json: "tempDirectory", js: "tempDirectory", typ: "" }, ], false), "ZarfCreateOptions": o([ { json: "differential", js: "differential", typ: r("DifferentialData") }, { json: "maxPackageSizeMB", js: "maxPackageSizeMB", typ: 0 }, - { json: "outputDirectory", js: "outputDirectory", typ: "" }, + { json: "output", js: "output", typ: "" }, { json: "registryOverrides", js: "registryOverrides", typ: m("") }, { json: "sbom", js: "sbom", typ: true }, { json: "sbomOutput", js: "sbomOutput", typ: "" }, diff --git a/src/ui/lib/api.ts b/src/ui/lib/api.ts index dcf22c7df8..406259e266 100644 --- a/src/ui/lib/api.ts +++ b/src/ui/lib/api.ts @@ -41,6 +41,10 @@ const Packages = { deploy: (options: APIZarfDeployPayload) => http.put(`/packages/deploy`, options), deployStream: (eventParams: EventParams) => http.eventStream('/packages/deploy-stream', eventParams), + deployingComponents: { + list: (pkgName: string) => + http.get(`/packages/${encodeURIComponent(pkgName)}/components/deployed`), + }, remove: (name: string) => http.del(`/packages/remove/${encodeURIComponent(name)}`), listPkgConnections: (name: string) => http.get(`/packages/${encodeURIComponent(name)}/connections`), @@ -63,8 +67,4 @@ const Packages = { http.eventStream(`/packages/find/stream/home?init=${init}`, eventParams), }; -const DeployingComponents = { - list: () => http.get('/components/deployed'), -}; - -export { Auth, Cluster, Packages, DeployingComponents }; +export { Auth, Cluster, Packages }; diff --git a/src/ui/routes/packages/[name]/deploy/+page.svelte b/src/ui/routes/packages/[name]/deploy/+page.svelte index 50739c21f0..52b509460b 100644 --- a/src/ui/routes/packages/[name]/deploy/+page.svelte +++ b/src/ui/routes/packages/[name]/deploy/+page.svelte @@ -1,9 +1,9 @@ -