From 1ec88490121cf8ec1ae0ce402691fc3c8f663681 Mon Sep 17 00:00:00 2001 From: Lucas Rodriguez Date: Wed, 13 Mar 2024 11:02:54 -0500 Subject: [PATCH] fix: refactor create stages into separate lib (#2223) ## Description refactor create stages into separate lib ## Related Issue Fixes https://github.com/defenseunicorns/zarf/issues/2188 ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [x] Other (security config, docs update, etc) ## Checklist before merging - [x] Test, docs, adr added or updated as needed - [x] [Contributor Guide Steps](https://github.com/defenseunicorns/zarf/blob/main/CONTRIBUTING.md#developer-workflow) followed --------- Signed-off-by: razzle Co-authored-by: razzle Co-authored-by: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> --- docs/3-create-a-zarf-package/4-zarf-schema.md | 34 +- .../5-package-create-lifecycle.md | 69 +- src/cmd/common/utils.go | 13 +- src/cmd/dev.go | 8 +- src/cmd/initialize.go | 2 +- src/cmd/package.go | 4 +- src/config/lang/english.go | 9 +- src/internal/packager/sbom/tools.go | 30 - src/pkg/layout/package.go | 116 ++- src/pkg/layout/package_test.go | 261 +++--- src/pkg/layout/sbom.go | 41 +- src/pkg/packager/{ => actions}/actions.go | 37 +- src/pkg/packager/common.go | 124 +-- src/pkg/packager/common_test.go | 2 +- src/pkg/packager/components.go | 32 + src/pkg/packager/composer/list.go | 3 + src/pkg/packager/create.go | 18 +- src/pkg/packager/create_stages.go | 782 ------------------ src/pkg/packager/{ => creator}/compose.go | 40 +- src/pkg/packager/creator/creator.go | 17 + src/pkg/packager/creator/differential.go | 108 +++ src/pkg/packager/creator/normal.go | 560 +++++++++++++ src/pkg/packager/creator/skeleton.go | 329 ++++++++ src/pkg/packager/creator/template.go | 99 +++ src/pkg/packager/creator/utils.go | 61 ++ src/pkg/packager/deploy.go | 28 +- src/pkg/packager/dev.go | 15 +- src/pkg/packager/extensions.go | 68 -- src/pkg/packager/generate.go | 1 - src/pkg/packager/inspect.go | 5 +- src/pkg/packager/interactive.go | 3 +- src/pkg/packager/lint/lint.go | 5 +- src/pkg/packager/mirror.go | 9 +- src/pkg/packager/prepare.go | 29 +- src/pkg/packager/publish.go | 40 +- src/pkg/packager/remove.go | 14 +- src/pkg/packager/sources/oci.go | 11 +- src/pkg/packager/sources/utils.go | 21 +- src/pkg/packager/variables.go | 160 ---- src/pkg/packager/variables/variables.go | 54 ++ src/pkg/packager/yaml.go | 123 --- src/pkg/utils/helpers/io.go | 3 +- src/pkg/utils/io.go | 8 +- src/pkg/utils/yaml.go | 6 +- src/test/e2e/03_deprecations_test.go | 2 +- src/test/e2e/08_create_differential_test.go | 9 +- src/test/e2e/20_zarf_init_test.go | 2 +- src/test/e2e/24_variables_test.go | 2 +- src/test/e2e/51_oci_compose_test.go | 28 + .../packages/09-composable-packages/zarf.yaml | 1 + src/types/package.go | 38 +- src/types/packager.go | 24 + src/types/runtime.go | 31 +- zarf.schema.json | 12 +- 54 files changed, 1914 insertions(+), 1637 deletions(-) rename src/pkg/packager/{ => actions}/actions.go (86%) delete mode 100644 src/pkg/packager/create_stages.go rename src/pkg/packager/{ => creator}/compose.go (50%) create mode 100644 src/pkg/packager/creator/creator.go create mode 100644 src/pkg/packager/creator/differential.go create mode 100644 src/pkg/packager/creator/normal.go create mode 100644 src/pkg/packager/creator/skeleton.go create mode 100644 src/pkg/packager/creator/template.go create mode 100644 src/pkg/packager/creator/utils.go delete mode 100644 src/pkg/packager/extensions.go delete mode 100644 src/pkg/packager/variables.go create mode 100644 src/pkg/packager/variables/variables.go delete mode 100644 src/pkg/packager/yaml.go diff --git a/docs/3-create-a-zarf-package/4-zarf-schema.md b/docs/3-create-a-zarf-package/4-zarf-schema.md index 80a989a03b..050539669f 100644 --- a/docs/3-create-a-zarf-package/4-zarf-schema.md +++ b/docs/3-create-a-zarf-package/4-zarf-schema.md @@ -409,6 +409,25 @@ Must be one of: +
+ + registryOverrides + +  +
+ + ## build > registryOverrides + +**Description:** Any registry domains that were overridden on package create when pulling images + +| | | +| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| **Type** | `object` | +| **Additional properties** | [![Any type: allowed](https://img.shields.io/badge/Any%20type-allowed-green)](# "Additional Properties of any type are allowed.") | + +
+
+
differential @@ -425,21 +444,18 @@ Must be one of:
-
+
- registryOverrides + differentialPackageVersion  
- ## build > registryOverrides - -**Description:** Any registry domains that were overridden on package create when pulling images +**Description:** Version of a previously built package used as the basis for creating this differential package -| | | -| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| **Type** | `object` | -| **Additional properties** | [![Any type: allowed](https://img.shields.io/badge/Any%20type-allowed-green)](# "Additional Properties of any type are allowed.") | +| | | +| -------- | -------- | +| **Type** | `string` |
diff --git a/docs/3-create-a-zarf-package/5-package-create-lifecycle.md b/docs/3-create-a-zarf-package/5-package-create-lifecycle.md index 58da969df8..fa1a0a8d6d 100644 --- a/docs/3-create-a-zarf-package/5-package-create-lifecycle.md +++ b/docs/3-create-a-zarf-package/5-package-create-lifecycle.md @@ -6,50 +6,51 @@ The following diagram shows the order of operations for the `zarf package create ```mermaid graph TD - A1(set working directory)-->A2 - A2(parse zarf.yaml)-->A3 - A3(filter components by architecture)-->A4 - A4(detect init package)-->A5 - A5(handle deprecations)-->A6 + A1(cd to directory with zarf.yaml)-->A2 + A2(load zarf.yaml into memory)-->A3 + A3(set package architecture if not provided)-->A4 + A4(filter components by architecture and flavor)-->A5 + A5(migrate deprecated component configs)-->A6 A6(parse component imports)-->A7 A7(process create-time variables)-->A8 - A8(write build data and zarf.yaml)-->A9 - - A9(run validations)-->A10 - A10(confirm package create):::prompt-->A11 - A11{Init package?} - A11 -->|Yes| A12(add seed image)-->A13 - A11 -->|No| A13 + A8(process extensions)-->A9 + A9(remove duplicate images/repos if --differential flag used)-->A10 + A10(run validations)-->A11 + A11(confirm package create):::prompt-->A12 subgraph - A13(add each component)-->A13 - A13 --> A14(run each '.actions.onCreate.before'):::action-->A14 - A14 --> A15(load '.charts')-->A16 - A16(load '.files')-->A17 - A17(load '.dataInjections')-->A18 - A18(load '.manifests')-->A19 - A19(load '.repos')-->A20 - A20(run each '.actions.onCreate.after'):::action-->A20 - A20-->A21{Success?} - A21-->|Yes|A22(run each\n'.actions.onCreate.success'):::action-->A22 - A21-->|No|A23(run each\n'.actions.onCreate.failure'):::action-->A23-->A999 + A12(run each '.actions.onCreate.before'):::action-->A13(load '.charts') + A13-->A14(load '.files') + A14-->A15(load '.dataInjections') + A15-->A16(load '.manifests') + A16-->A17(load '.repos') + A17-->A18(run each '.actions.onCreate.after'):::action + A18-->A19{Success?} + A19-->|Yes|A20(run each\n'.actions.onCreate.success'):::action + A19-->|No|A999 end - A22-->A24(load all '.images') - A24-->A25{Skip SBOM?} - A25-->|Yes|A27 - A25-->|No|A26 - A26(generate SBOM)-->A27 - A27(reset working directory)-->A28 - A28(create package archive)-->A29 - A29{Is multipart?} - A29-->|Yes|A30(split package archive)-->A31 - A29-->|No|A31 - A31(handle sbom view/out flags) + A20-->A21(load all '.images') + A21-->A22(generate SBOMs unless --skip-sbom flag was used) + A22-->A23(cd back to original working directory) + A23-->A24(archive components into tarballs) + A24-->A25(generate checksums for all package files) + A25-->A26(record package build metadata) + A26-->A27(write the zarf.yaml to disk) + A27-->A28(sign the package if a key was provided) + A28-->A29{Output to OCI?} + A29-->|Yes|A30(publish package to OCI registry) + A29-->|No|A31(archive package into a tarball and write to disk) + A30-->A32 + A31-->A32 + A32(write SBOM files to disk if --sbom or --sbom-out flags used)-->A33 + A33(view SBOMs if --sbom flag used)-->A34 + A34[Zarf Package Create Successful]:::success A999[Abort]:::fail classDef prompt fill:#4adede,color:#000000 classDef action fill:#bd93f9,color:#000000 classDef fail fill:#aa0000 + classDef success fill:#008000,color:#fff; ``` diff --git a/src/cmd/common/utils.go b/src/cmd/common/utils.go index 01a6b104d6..1da7e456ee 100644 --- a/src/cmd/common/utils.go +++ b/src/cmd/common/utils.go @@ -4,15 +4,10 @@ // Package common handles command configuration across all commands package common -import ( - "github.com/defenseunicorns/zarf/src/types" -) - -// SetBaseDirectory sets base directory on package config when given in args -func SetBaseDirectory(args []string, pkgConfig *types.PackagerConfig) { +// SetBaseDirectory sets the base directory. This is a directory with a zarf.yaml. +func SetBaseDirectory(args []string) string { if len(args) > 0 { - pkgConfig.CreateOpts.BaseDir = args[0] - } else { - pkgConfig.CreateOpts.BaseDir = "." + return args[0] } + return "." } diff --git a/src/cmd/dev.go b/src/cmd/dev.go index 0ded3ac401..9bc12a2b1c 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -41,7 +41,7 @@ var devDeployCmd = &cobra.Command{ Short: lang.CmdDevDeployShort, Long: lang.CmdDevDeployLong, Run: func(_ *cobra.Command, args []string) { - common.SetBaseDirectory(args, &pkgConfig) + pkgConfig.CreateOpts.BaseDir = common.SetBaseDirectory(args) v := common.GetViper() pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( @@ -116,7 +116,7 @@ var devTransformGitLinksCmd = &cobra.Command{ if confirm { // Overwrite the file - err = os.WriteFile(fileName, []byte(processedText), 0640) + err = os.WriteFile(fileName, []byte(processedText), helpers.ReadAllWriteUser) if err != nil { message.Fatal(err, lang.CmdDevPatchGitFileWriteErr) } @@ -207,7 +207,7 @@ var devFindImagesCmd = &cobra.Command{ Short: lang.CmdDevFindImagesShort, Long: lang.CmdDevFindImagesLong, Run: func(_ *cobra.Command, args []string) { - common.SetBaseDirectory(args, &pkgConfig) + pkgConfig.CreateOpts.BaseDir = common.SetBaseDirectory(args) // Ensure uppercase keys from viper v := common.GetViper() @@ -256,7 +256,7 @@ var devLintCmd = &cobra.Command{ Short: lang.CmdDevLintShort, Long: lang.CmdDevLintLong, Run: func(_ *cobra.Command, args []string) { - common.SetBaseDirectory(args, &pkgConfig) + pkgConfig.CreateOpts.BaseDir = common.SetBaseDirectory(args) v := common.GetViper() pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( v.GetStringMapString(common.VPkgCreateSet), pkgConfig.CreateOpts.SetVariables, strings.ToUpper) diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index e12eccd665..40f633ed41 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -44,7 +44,7 @@ var initCmd = &cobra.Command{ } // Continue running package deploy for all components like any other package - initPackageName := packager.GetInitPackageName("") + initPackageName := sources.GetInitPackageName() pkgConfig.PkgOpts.PackageSource = initPackageName // Try to use an init-package in the executable directory if none exist in current working directory diff --git a/src/cmd/package.go b/src/cmd/package.go index f5a1c57804..f296684e7a 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -41,7 +41,7 @@ var packageCreateCmd = &cobra.Command{ Short: lang.CmdPackageCreateShort, Long: lang.CmdPackageCreateLong, Run: func(_ *cobra.Command, args []string) { - common.SetBaseDirectory(args, &pkgConfig) + pkgConfig.CreateOpts.BaseDir = common.SetBaseDirectory(args) var isCleanPathRegex = regexp.MustCompile(`^[a-zA-Z0-9\_\-\/\.\~\\:]+$`) if !isCleanPathRegex.MatchString(config.CommonOptions.CachePath) { @@ -350,7 +350,7 @@ func bindCreateFlags(v *viper.Viper) { createFlags.StringVar(&pkgConfig.CreateOpts.Output, "output-directory", v.GetString("package.create.output_directory"), lang.CmdPackageCreateFlagOutput) createFlags.StringVarP(&pkgConfig.CreateOpts.Output, "output", "o", v.GetString(common.VPkgCreateOutput), lang.CmdPackageCreateFlagOutput) - createFlags.StringVar(&pkgConfig.CreateOpts.DifferentialData.DifferentialPackagePath, "differential", v.GetString(common.VPkgCreateDifferential), lang.CmdPackageCreateFlagDifferential) + createFlags.StringVar(&pkgConfig.CreateOpts.DifferentialPackagePath, "differential", v.GetString(common.VPkgCreateDifferential), lang.CmdPackageCreateFlagDifferential) createFlags.StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdPackageCreateFlagSet) createFlags.BoolVarP(&pkgConfig.CreateOpts.ViewSBOM, "sbom", "s", v.GetBool(common.VPkgCreateSbom), lang.CmdPackageCreateFlagSbom) createFlags.StringVar(&pkgConfig.CreateOpts.SBOMOutputDir, "sbom-out", v.GetString(common.VPkgCreateSbomOutput), lang.CmdPackageCreateFlagSbomOut) diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 9ff198e8e1..d919da0f9b 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -633,12 +633,13 @@ const ( AgentErrUnableTransform = "unable to transform the provided request; see zarf http proxy logs for more details" ) -// src/internal/packager/create +// Package create const ( - PkgCreateErrDifferentialSameVersion = "unable to create a differential package with the same version as the package you are using as a reference; the package version must be incremented" + PkgCreateErrDifferentialSameVersion = "unable to create differential package. Please ensure the differential package version and reference package version are not the same. The package version must be incremented" + PkgCreateErrDifferentialNoVersion = "unable to create differential package. Please ensure both package versions are set" ) -// src/internal/packager/deploy. +// Package deploy const ( PkgDeployErrMultipleComponentsSameGroup = "You cannot specify multiple components (%q, %q) within the same group (%q) when using the --components flag." PkgDeployErrNoDefaultOrSelection = "You must make a selection from %q with the --components flag as there is no default in their group." @@ -646,7 +647,7 @@ const ( PkgDeployErrComponentSelectionCanceled = "Component selection canceled: %s" ) -// src/internal/packager/validate. +// Package validate const ( PkgValidateTemplateDeprecation = "Package template %q is using the deprecated syntax ###ZARF_PKG_VAR_%s###. This will be removed in Zarf v1.0.0. Please update to ###ZARF_PKG_TMPL_%s###." PkgValidateMustBeUppercase = "variable name %q must be all uppercase and contain no special characters except _" diff --git a/src/internal/packager/sbom/tools.go b/src/internal/packager/sbom/tools.go index e964f85ece..860b50e879 100644 --- a/src/internal/packager/sbom/tools.go +++ b/src/internal/packager/sbom/tools.go @@ -6,15 +6,11 @@ package sbom import ( "fmt" - "os" "path/filepath" "github.com/AlecAivazis/survey/v2" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/exec" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" - "github.com/defenseunicorns/zarf/src/types" ) // ViewSBOMFiles opens a browser to view the SBOM files and pauses for user input. @@ -41,29 +37,3 @@ func ViewSBOMFiles(directory string) { message.Note("There were no images with software bill-of-materials (SBOM) included.") } } - -// OutputSBOMFiles outputs the sbom files into a specified directory. -func OutputSBOMFiles(sourceDir, outputDir, packageName string) (string, error) { - packagePath := filepath.Join(outputDir, packageName) - - if err := os.RemoveAll(packagePath); err != nil { - return "", err - } - - if err := utils.CreateDirectory(packagePath, helpers.ReadWriteExecuteUser); err != nil { - return "", err - } - - return packagePath, utils.CreatePathAndCopy(sourceDir, packagePath) -} - -// IsSBOMAble checks if a package has contents that an SBOM can be created on (i.e. images, files, or data injections) -func IsSBOMAble(pkg types.ZarfPackage) bool { - for _, c := range pkg.Components { - if len(c.Images) > 0 || len(c.Files) > 0 || len(c.DataInjections) > 0 { - return true - } - } - - return false -} diff --git a/src/pkg/layout/package.go b/src/pkg/layout/package.go index 4e7bcff63a..2683ebf603 100644 --- a/src/pkg/layout/package.go +++ b/src/pkg/layout/package.go @@ -5,15 +5,21 @@ package layout import ( + "fmt" "os" "path/filepath" + "slices" "strings" "github.com/Masterminds/semver/v3" + "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/google/go-containerregistry/pkg/crane" + "github.com/mholt/archiver/v3" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -51,6 +57,29 @@ func New(baseDir string) *PackagePaths { } } +// ReadZarfYAML reads a zarf.yaml file into memory, +// checks if it's using the legacy layout, and migrates deprecated component configs. +func (pp *PackagePaths) ReadZarfYAML(path string) (pkg types.ZarfPackage, warnings []string, err error) { + if err := utils.ReadYaml(path, &pkg); err != nil { + return types.ZarfPackage{}, nil, fmt.Errorf("unable to read zarf.yaml file") + } + + if pp.IsLegacyLayout() { + warnings = append(warnings, "Detected deprecated package layout, migrating to new layout - support for this package will be dropped in v1.0.0") + } + + if len(pkg.Build.Migrations) > 0 { + var componentWarnings []string + for idx, component := range pkg.Components { + // Handle component configuration deprecations + pkg.Components[idx], componentWarnings = deprecated.MigrateComponent(pkg.Build, component) + warnings = append(warnings, componentWarnings...) + } + } + + return pkg, warnings, nil +} + // MigrateLegacy migrates a legacy package layout to the new layout. func (pp *PackagePaths) MigrateLegacy() (err error) { var pkg types.ZarfPackage @@ -139,12 +168,87 @@ func (pp *PackagePaths) IsLegacyLayout() bool { return pp.isLegacyLayout } -// AddSignature sets the signature path if the keyPath is not empty. -func (pp *PackagePaths) AddSignature(keyPath string) *PackagePaths { - if keyPath != "" { - pp.Signature = filepath.Join(pp.Base, Signature) +// SignPackage signs the zarf.yaml in a Zarf package. +func (pp *PackagePaths) SignPackage(signingKeyPath, signingKeyPassword string) error { + pp.Signature = filepath.Join(pp.Base, Signature) + + passwordFunc := func(_ bool) ([]byte, error) { + if signingKeyPassword != "" { + return []byte(signingKeyPassword), nil + } + return interactive.PromptSigPassword() } - return pp + _, err := utils.CosignSignBlob(pp.ZarfYAML, pp.Signature, signingKeyPath, passwordFunc) + if err != nil { + return fmt.Errorf("unable to sign the package: %w", err) + } + + return nil +} + +// GenerateChecksums walks through all of the files starting at the base path and generates a checksum file. +// +// Each file within the basePath represents a layer within the Zarf package. +// +// Returns a SHA256 checksum of the checksums.txt file. +func (pp *PackagePaths) GenerateChecksums() (string, error) { + var checksumsData = []string{} + + for rel, abs := range pp.Files() { + if rel == ZarfYAML || rel == Checksums { + continue + } + + sum, err := utils.GetSHA256OfFile(abs) + if err != nil { + return "", err + } + checksumsData = append(checksumsData, fmt.Sprintf("%s %s", sum, rel)) + } + slices.Sort(checksumsData) + + // Create the checksums file + if err := os.WriteFile(pp.Checksums, []byte(strings.Join(checksumsData, "\n")+"\n"), helpers.ReadWriteUser); err != nil { + return "", err + } + + // Calculate the checksum of the checksum file + return utils.GetSHA256OfFile(pp.Checksums) +} + +// ArchivePackage creates an archive for a Zarf package. +func (pp *PackagePaths) ArchivePackage(destinationTarball string, maxPackageSizeMB int) error { + spinner := message.NewProgressSpinner("Writing %s to %s", pp.Base, destinationTarball) + defer spinner.Stop() + + // Make the archive + archiveSrc := []string{pp.Base + string(os.PathSeparator)} + if err := archiver.Archive(archiveSrc, destinationTarball); err != nil { + return fmt.Errorf("unable to create package: %w", err) + } + spinner.Updatef("Wrote %s to %s", pp.Base, destinationTarball) + + fi, err := os.Stat(destinationTarball) + if err != nil { + return fmt.Errorf("unable to read the package archive: %w", err) + } + spinner.Successf("Package saved to %q", destinationTarball) + + // Convert Megabytes to bytes. + chunkSize := maxPackageSizeMB * 1000 * 1000 + + // If a chunk size was specified and the package is larger than the chunk size, split it into chunks. + if maxPackageSizeMB > 0 && fi.Size() > int64(chunkSize) { + if fi.Size()/int64(chunkSize) > 999 { + return fmt.Errorf("unable to split the package archive into multiple files: must be less than 1,000 files") + } + message.Notef("Package is larger than %dMB, splitting into multiple files", maxPackageSizeMB) + err := utils.SplitFile(destinationTarball, chunkSize) + if err != nil { + return fmt.Errorf("unable to split the package archive into multiple files: %w", err) + } + } + return nil } // AddImages sets the default image paths. @@ -242,7 +346,7 @@ func (pp *PackagePaths) Files() map[string]string { add(tarball) } - if filepath.Ext(pp.SBOMs.Path) == ".tar" { + if pp.SBOMs.IsTarball() { add(pp.SBOMs.Path) } return pathMap diff --git a/src/pkg/layout/package_test.go b/src/pkg/layout/package_test.go index 55c3aeb695..8e7036f066 100644 --- a/src/pkg/layout/package_test.go +++ b/src/pkg/layout/package_test.go @@ -5,6 +5,7 @@ package layout import ( + "path/filepath" "runtime" "strings" "testing" @@ -14,128 +15,154 @@ import ( ) func TestPackageFiles(t *testing.T) { - pp := New("test") - - raw := &PackagePaths{ - Base: "test", - ZarfYAML: normalizePath("test/zarf.yaml"), - Checksums: normalizePath("test/checksums.txt"), - Components: Components{ - Base: normalizePath("test/components"), - }, - } - - require.Equal(t, raw, pp) - - files := pp.Files() - - expected := map[string]string{ - "zarf.yaml": normalizePath("test/zarf.yaml"), - "checksums.txt": normalizePath("test/checksums.txt"), - } - - require.Equal(t, expected, files) - - pp = pp.AddSignature("") - - files = pp.Files() - - // AddSignature will only add the signature if it is not empty - require.Equal(t, expected, files) - - pp = pp.AddSignature("key.priv") - - files = pp.Files() - - expected = map[string]string{ - "zarf.yaml": normalizePath("test/zarf.yaml"), - "checksums.txt": normalizePath("test/checksums.txt"), - "zarf.yaml.sig": normalizePath("test/zarf.yaml.sig"), - } - - require.Equal(t, expected, files) - pp = pp.AddImages() - - files = pp.Files() - - // Note that the map key will always be the forward "Slash" (/) version of the file path (never \) - expected = map[string]string{ - "zarf.yaml": normalizePath("test/zarf.yaml"), - "checksums.txt": normalizePath("test/checksums.txt"), - "zarf.yaml.sig": normalizePath("test/zarf.yaml.sig"), - "images/index.json": normalizePath("test/images/index.json"), - "images/oci-layout": normalizePath("test/images/oci-layout"), - } - - require.Equal(t, expected, files) + t.Parallel() - pp = pp.AddSBOMs() + t.Run("Verify New()", func(t *testing.T) { + t.Parallel() - files = pp.Files() + pp := New("test") - // AddSBOMs adds the SBOMs directory, and files will only cares about files - require.Equal(t, expected, files) - - paths := []string{ - "zarf.yaml", - "checksums.txt", - "sboms.tar", - normalizePath("components/c1.tar"), - normalizePath("images/index.json"), - normalizePath("images/oci-layout"), - normalizePath("images/blobs/sha256/" + strings.Repeat("1", 64)), - } - - pp = New("test") - - pp.SetFromPaths(paths) - - files = pp.Files() - - expected = map[string]string{ - "zarf.yaml": normalizePath("test/zarf.yaml"), - "checksums.txt": normalizePath("test/checksums.txt"), - "sboms.tar": normalizePath("test/sboms.tar"), - "components/c1.tar": normalizePath("test/components/c1.tar"), - "images/index.json": normalizePath("test/images/index.json"), - "images/oci-layout": normalizePath("test/images/oci-layout"), - "images/blobs/sha256/" + strings.Repeat("1", 64): normalizePath("test/images/blobs/sha256/" + strings.Repeat("1", 64)), - } - - require.Len(t, pp.Images.Blobs, 1) - - require.Equal(t, expected, files) - - descs := []ocispec.Descriptor{ - { - Annotations: map[string]string{ - ocispec.AnnotationTitle: "components/c2.tar", + raw := &PackagePaths{ + Base: "test", + ZarfYAML: normalizePath("test/zarf.yaml"), + Checksums: normalizePath("test/checksums.txt"), + Components: Components{ + Base: normalizePath("test/components"), }, - }, - { - Annotations: map[string]string{ - ocispec.AnnotationTitle: "images/blobs/sha256/" + strings.Repeat("2", 64), + } + require.Equal(t, raw, pp) + }) + + t.Run("Verify Files()", func(t *testing.T) { + t.Parallel() + + pp := New("test") + + files := pp.Files() + expected := map[string]string{ + "zarf.yaml": normalizePath("test/zarf.yaml"), + "checksums.txt": normalizePath("test/checksums.txt"), + } + require.Equal(t, expected, files) + }) + + t.Run("Verify Files() with signature", func(t *testing.T) { + t.Parallel() + + pp := New("test") + pp.Signature = filepath.Join(pp.Base, Signature) + + files := pp.Files() + expected := map[string]string{ + "zarf.yaml": normalizePath("test/zarf.yaml"), + "checksums.txt": normalizePath("test/checksums.txt"), + "zarf.yaml.sig": normalizePath("test/zarf.yaml.sig"), + } + require.Equal(t, expected, files) + }) + + t.Run("Verify Files() with images", func(t *testing.T) { + t.Parallel() + + pp := New("test") + pp = pp.AddImages() + + files := pp.Files() + expected := map[string]string{ + "zarf.yaml": normalizePath("test/zarf.yaml"), + "checksums.txt": normalizePath("test/checksums.txt"), + "images/index.json": normalizePath("test/images/index.json"), + "images/oci-layout": normalizePath("test/images/oci-layout"), + } + require.Equal(t, expected, files) + }) + + // AddSBOMs sets the SBOMs path, so Files() should not return new files. + t.Run("Verify Files() with SBOMs", func(t *testing.T) { + t.Parallel() + + pp := New("test") + pp = pp.AddSBOMs() + + files := pp.Files() + expected := map[string]string{ + "zarf.yaml": normalizePath("test/zarf.yaml"), + "checksums.txt": normalizePath("test/checksums.txt"), + } + require.Equal(t, expected, files) + + pp.SBOMs.Path = normalizePath("test/sboms.tar") + files = pp.Files() + expected = map[string]string{ + "zarf.yaml": normalizePath("test/zarf.yaml"), + "checksums.txt": normalizePath("test/checksums.txt"), + "sboms.tar": normalizePath("test/sboms.tar"), + } + require.Equal(t, expected, files) + }) + + t.Run("Verify Files() with paths mapped to package paths", func(t *testing.T) { + t.Parallel() + + pp := New("test") + + paths := []string{ + "zarf.yaml", + "checksums.txt", + "sboms.tar", + normalizePath("components/c1.tar"), + normalizePath("images/index.json"), + normalizePath("images/oci-layout"), + normalizePath("images/blobs/sha256/" + strings.Repeat("1", 64)), + } + pp.SetFromPaths(paths) + + files := pp.Files() + expected := map[string]string{ + "zarf.yaml": normalizePath("test/zarf.yaml"), + "checksums.txt": normalizePath("test/checksums.txt"), + "sboms.tar": normalizePath("test/sboms.tar"), + "components/c1.tar": normalizePath("test/components/c1.tar"), + "images/index.json": normalizePath("test/images/index.json"), + "images/oci-layout": normalizePath("test/images/oci-layout"), + "images/blobs/sha256/" + strings.Repeat("1", 64): normalizePath("test/images/blobs/sha256/" + strings.Repeat("1", 64)), + } + + require.Len(t, pp.Images.Blobs, 1) + require.Equal(t, expected, files) + }) + + t.Run("Verify Files() with image layers mapped to package paths", func(t *testing.T) { + t.Parallel() + + pp := New("test") + + descs := []ocispec.Descriptor{ + { + Annotations: map[string]string{ + ocispec.AnnotationTitle: "components/c2.tar", + }, }, - }, - } - - pp.SetFromLayers(descs) - - files = pp.Files() - - expected = map[string]string{ - "zarf.yaml": normalizePath("test/zarf.yaml"), - "checksums.txt": normalizePath("test/checksums.txt"), - "sboms.tar": normalizePath("test/sboms.tar"), - "components/c1.tar": normalizePath("test/components/c1.tar"), - "components/c2.tar": normalizePath("test/components/c2.tar"), - "images/index.json": normalizePath("test/images/index.json"), - "images/oci-layout": normalizePath("test/images/oci-layout"), - "images/blobs/sha256/" + strings.Repeat("1", 64): normalizePath("test/images/blobs/sha256/" + strings.Repeat("1", 64)), - "images/blobs/sha256/" + strings.Repeat("2", 64): normalizePath("test/images/blobs/sha256/" + strings.Repeat("2", 64)), - } - - require.Equal(t, expected, files) + { + Annotations: map[string]string{ + ocispec.AnnotationTitle: "images/blobs/sha256/" + strings.Repeat("1", 64), + }, + }, + } + pp.AddImages() + pp.SetFromLayers(descs) + + files := pp.Files() + expected := map[string]string{ + "zarf.yaml": normalizePath("test/zarf.yaml"), + "checksums.txt": normalizePath("test/checksums.txt"), + "components/c2.tar": normalizePath("test/components/c2.tar"), + "images/index.json": normalizePath("test/images/index.json"), + "images/oci-layout": normalizePath("test/images/oci-layout"), + "images/blobs/sha256/" + strings.Repeat("1", 64): normalizePath("test/images/blobs/sha256/" + strings.Repeat("1", 64)), + } + require.Equal(t, expected, files) + }) } // normalizePath ensures that the filepaths being generated are normalized to the host OS. diff --git a/src/pkg/layout/sbom.go b/src/pkg/layout/sbom.go index 13f7ee0fc1..fe7dcefd02 100644 --- a/src/pkg/layout/sbom.go +++ b/src/pkg/layout/sbom.go @@ -5,6 +5,7 @@ package layout import ( + "fmt" "io/fs" "os" "path/filepath" @@ -67,12 +68,44 @@ func (s *SBOMs) Archive() (err error) { return os.RemoveAll(dir) } -// IsDir returns true if the SBOMs are a directory. -func (s SBOMs) IsDir() bool { - return utils.IsDir(s.Path) +// StageSBOMViewFiles copies SBOM viewer HTML files to the Zarf SBOM directory. +func (s *SBOMs) StageSBOMViewFiles() (warnings []string, err error) { + if s.IsTarball() { + return nil, fmt.Errorf("unable to process the SBOM files for this package: %s is a tarball", s.Path) + } + + // If SBOMs were loaded, temporarily place them in the deploy directory + if !utils.InvalidPath(s.Path) { + if _, err := filepath.Glob(filepath.Join(s.Path, "sbom-viewer-*")); err != nil { + return nil, err + } + + if _, err := s.OutputSBOMFiles(SBOMDir, ""); err != nil { + // Don't stop the deployment, let the user decide if they want to continue the deployment + warning := fmt.Sprintf("Unable to process the SBOM files for this package: %s", err.Error()) + warnings = append(warnings, warning) + } + } + + return warnings, nil +} + +// OutputSBOMFiles outputs SBOM files into outputDir. +func (s *SBOMs) OutputSBOMFiles(outputDir, packageName string) (string, error) { + packagePath := filepath.Join(outputDir, packageName) + + if err := os.RemoveAll(packagePath); err != nil { + return "", err + } + + if err := utils.CreateDirectory(packagePath, 0700); err != nil { + return "", err + } + + return packagePath, utils.CreatePathAndCopy(s.Path, packagePath) } // IsTarball returns true if the SBOMs are a tarball. func (s SBOMs) IsTarball() bool { - return !s.IsDir() && filepath.Ext(s.Path) == ".tar" + return !utils.IsDir(s.Path) && filepath.Ext(s.Path) == ".tar" } diff --git a/src/pkg/packager/actions.go b/src/pkg/packager/actions/actions.go similarity index 86% rename from src/pkg/packager/actions.go rename to src/pkg/packager/actions/actions.go index 1a49af931d..c6df1abc8d 100644 --- a/src/pkg/packager/actions.go +++ b/src/pkg/packager/actions/actions.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package packager contains functions for interacting with, managing and deploying zarf packages. -package packager +// Package actions contains functions for running component actions within Zarf packages. +package actions import ( "context" @@ -20,9 +20,10 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -func (p *Packager) runActions(defaultCfg types.ZarfComponentActionDefaults, actions []types.ZarfComponentAction, valueTemplate *template.Values) error { +// Run runs all provided actions. +func Run(cfg *types.PackagerConfig, defaultCfg types.ZarfComponentActionDefaults, actions []types.ZarfComponentAction, valueTemplate *template.Values) error { for _, a := range actions { - if err := p.runAction(defaultCfg, a, valueTemplate); err != nil { + if err := runAction(cfg, defaultCfg, a, valueTemplate); err != nil { return err } } @@ -30,7 +31,7 @@ func (p *Packager) runActions(defaultCfg types.ZarfComponentActionDefaults, acti } // Run commands that a component has provided. -func (p *Packager) runAction(defaultCfg types.ZarfComponentActionDefaults, action types.ZarfComponentAction, valueTemplate *template.Values) error { +func runAction(cfg *types.PackagerConfig, defaultCfg types.ZarfComponentActionDefaults, action types.ZarfComponentAction, valueTemplate *template.Values) error { var ( ctx context.Context cancel context.CancelFunc @@ -87,23 +88,23 @@ func (p *Packager) runAction(defaultCfg types.ZarfComponentActionDefaults, actio vars, _ = valueTemplate.GetVariables(types.ZarfComponent{}) } - cfg := actionGetCfg(defaultCfg, action, vars) + actionDefaults := actionGetCfg(defaultCfg, action, vars) - if cmd, err = actionCmdMutation(cmd, cfg.Shell); err != nil { + if cmd, err = actionCmdMutation(cmd, actionDefaults.Shell); err != nil { spinner.Errorf(err, "Error mutating command: %s", cmdEscaped) } - duration := time.Duration(cfg.MaxTotalSeconds) * time.Second + duration := time.Duration(actionDefaults.MaxTotalSeconds) * time.Second timeout := time.After(duration) // Keep trying until the max retries is reached. retryCmd: - for remaining := cfg.MaxRetries + 1; remaining > 0; remaining-- { + for remaining := actionDefaults.MaxRetries + 1; remaining > 0; remaining-- { // Perform the action run. tryCmd := func(ctx context.Context) error { // Try running the command and continue the retry loop if it fails. - if out, err = actionRun(ctx, cfg, cmd, cfg.Shell, spinner); err != nil { + if out, err = actionRun(ctx, actionDefaults, cmd, actionDefaults.Shell, spinner); err != nil { return err } @@ -111,8 +112,8 @@ retryCmd: // If an output variable is defined, set it. for _, v := range action.SetVariables { - p.setVariableInConfig(v.Name, out, v.Sensitive, v.AutoIndent, v.Type) - if err := p.checkVariablePattern(v.Name, v.Pattern); err != nil { + cfg.SetVariable(v.Name, out, v.Sensitive, v.AutoIndent, v.Type) + if err := cfg.CheckVariablePattern(v.Name, v.Pattern); err != nil { message.WarnErr(err, err.Error()) return err } @@ -130,7 +131,7 @@ retryCmd: } // If no timeout is set, run the command and return or continue retrying. - if cfg.MaxTotalSeconds < 1 { + if actionDefaults.MaxTotalSeconds < 1 { spinner.Updatef("Waiting for \"%s\" (no timeout)", cmdEscaped) if err := tryCmd(context.TODO()); err != nil { continue retryCmd @@ -140,7 +141,7 @@ retryCmd: } // Run the command on repeat until success or timeout. - spinner.Updatef("Waiting for \"%s\" (timeout: %ds)", cmdEscaped, cfg.MaxTotalSeconds) + spinner.Updatef("Waiting for \"%s\" (timeout: %ds)", cmdEscaped, actionDefaults.MaxTotalSeconds) select { // On timeout break the loop to abort. case <-timeout: @@ -161,14 +162,14 @@ retryCmd: select { case <-timeout: // If we reached this point, the timeout was reached or command failed with no retries. - if cfg.MaxTotalSeconds < 1 { - return fmt.Errorf("command %q failed after %d retries", cmdEscaped, cfg.MaxRetries) + if actionDefaults.MaxTotalSeconds < 1 { + return fmt.Errorf("command %q failed after %d retries", cmdEscaped, actionDefaults.MaxRetries) } else { - return fmt.Errorf("command %q timed out after %d seconds", cmdEscaped, cfg.MaxTotalSeconds) + return fmt.Errorf("command %q timed out after %d seconds", cmdEscaped, actionDefaults.MaxTotalSeconds) } default: // If we reached this point, the retry limit was reached. - return fmt.Errorf("command %q failed after %d retries", cmdEscaped, cfg.MaxRetries) + return fmt.Errorf("command %q failed after %d retries", cmdEscaped, actionDefaults.MaxRetries) } } diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index b2286f7fd4..11101ecec2 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -8,8 +8,6 @@ import ( "errors" "fmt" "os" - "path/filepath" - "regexp" "strings" "time" @@ -17,14 +15,11 @@ import ( "github.com/Masterminds/semver/v3" "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/internal/packager/sbom" "github.com/defenseunicorns/zarf/src/internal/packager/template" "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/types" - "github.com/mholt/archiver/v3" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" @@ -37,7 +32,6 @@ type Packager struct { cfg *types.PackagerConfig cluster *cluster.Cluster layout *layout.PackagePaths - arch string warnings []string valueTemplate *template.Values hpaModified bool @@ -47,15 +41,6 @@ type Packager struct { generation int } -// Zarf Packager Variables. -var ( - // Find zarf-packages on the local system (https://regex101.com/r/TUUftK/1) - ZarfPackagePattern = regexp.MustCompile(`zarf-package[^\s\\\/]*\.tar(\.zst)?$`) - - // Find zarf-init packages on the local system - ZarfInitPattern = regexp.MustCompile(GetInitPackageName("") + "$") -) - // Modifier is a function that modifies the packager. type Modifier func(*Packager) @@ -161,37 +146,6 @@ func (p *Packager) setTempDirectory(path string) error { return nil } -// GetInitPackageName returns the formatted name of the init package. -func GetInitPackageName(arch string) string { - if arch == "" { - // No package has been loaded yet so lookup GetArch() with no package info - arch = config.GetArch() - } - return fmt.Sprintf("zarf-init-%s-%s.tar.zst", arch, config.CLIVersion) -} - -// GetPackageName returns the formatted name of the package. -func (p *Packager) GetPackageName() string { - if p.isInitConfig() { - return GetInitPackageName(p.arch) - } - - packageName := p.cfg.Pkg.Metadata.Name - suffix := "tar.zst" - if p.cfg.Pkg.Metadata.Uncompressed { - suffix = "tar" - } - - packageFileName := fmt.Sprintf("%s%s-%s", config.ZarfPackagePrefix, packageName, p.arch) - if p.cfg.Pkg.Build.Differential { - packageFileName = fmt.Sprintf("%s-%s-differential-%s", packageFileName, p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion, p.cfg.Pkg.Metadata.Version) - } else if p.cfg.Pkg.Metadata.Version != "" { - packageFileName = fmt.Sprintf("%s-%s", packageFileName, p.cfg.Pkg.Metadata.Version) - } - - return fmt.Sprintf("%s.%s", packageFileName, suffix) -} - // ClearTempPaths removes the temp directory and any files within it. func (p *Packager) ClearTempPaths() { // Remove the temp directory, but don't throw an error if it fails @@ -218,11 +172,6 @@ func (p *Packager) isConnectedToCluster() bool { return p.cluster != nil } -// isInitConfig returns whether the current packager instance is deploying an init config -func (p *Packager) isInitConfig() bool { - return p.cfg.Pkg.Kind == types.ZarfInitConfig -} - // hasImages returns whether the current package contains images func (p *Packager) hasImages() bool { for _, component := range p.cfg.Pkg.Components { @@ -269,7 +218,7 @@ func (p *Packager) attemptClusterChecks() (err error) { // validatePackageArchitecture validates that the package architecture matches the target cluster architecture. func (p *Packager) validatePackageArchitecture() error { // Ignore this check if the architecture is explicitly "multi", we don't have a cluster connection, or the package contains no images - if p.arch == "multi" || !p.isConnectedToCluster() || !p.hasImages() { + if p.cfg.Pkg.Metadata.Architecture == "multi" || !p.isConnectedToCluster() || !p.hasImages() { return nil } @@ -279,8 +228,8 @@ func (p *Packager) validatePackageArchitecture() error { } // Check if the package architecture and the cluster architecture are the same. - if !slices.Contains(clusterArchitectures, p.arch) { - return fmt.Errorf(lang.CmdPackageDeployValidateArchitectureErr, p.arch, strings.Join(clusterArchitectures, ", ")) + if !slices.Contains(clusterArchitectures, p.cfg.Pkg.Metadata.Architecture) { + return fmt.Errorf(lang.CmdPackageDeployValidateArchitectureErr, p.cfg.Pkg.Metadata.Architecture, strings.Join(clusterArchitectures, ", ")) } return nil @@ -319,70 +268,3 @@ func (p *Packager) validateLastNonBreakingVersion() (err error) { return nil } - -func (p *Packager) archivePackage(destinationTarball string) error { - spinner := message.NewProgressSpinner("Writing %s to %s", p.layout.Base, destinationTarball) - defer spinner.Stop() - - // Make the archive - archiveSrc := []string{p.layout.Base + string(os.PathSeparator)} - if err := archiver.Archive(archiveSrc, destinationTarball); err != nil { - return fmt.Errorf("unable to create package: %w", err) - } - spinner.Updatef("Wrote %s to %s", p.layout.Base, destinationTarball) - - fi, err := os.Stat(destinationTarball) - if err != nil { - return fmt.Errorf("unable to read the package archive: %w", err) - } - spinner.Successf("Package saved to %q", destinationTarball) - - // Convert Megabytes to bytes. - chunkSize := p.cfg.CreateOpts.MaxPackageSizeMB * 1000 * 1000 - - // If a chunk size was specified and the package is larger than the chunk size, split it into chunks. - if p.cfg.CreateOpts.MaxPackageSizeMB > 0 && fi.Size() > int64(chunkSize) { - if fi.Size()/int64(chunkSize) > 999 { - return fmt.Errorf("unable to split the package archive into multiple files: must be less than 1,000 files") - } - message.Notef("Package is larger than %dMB, splitting into multiple files", p.cfg.CreateOpts.MaxPackageSizeMB) - err := utils.SplitFile(destinationTarball, chunkSize) - if err != nil { - return fmt.Errorf("unable to split the package archive into multiple files: %w", err) - } - } - return nil -} - -func (p *Packager) signPackage(signingKeyPath, signingKeyPassword string) error { - p.layout = p.layout.AddSignature(signingKeyPath) - passwordFunc := func(_ bool) ([]byte, error) { - if signingKeyPassword != "" { - return []byte(signingKeyPassword), nil - } - return interactive.PromptSigPassword() - } - _, err := utils.CosignSignBlob(p.layout.ZarfYAML, p.layout.Signature, signingKeyPath, passwordFunc) - if err != nil { - return fmt.Errorf("unable to sign the package: %w", err) - } - return nil -} - -func (p *Packager) stageSBOMViewFiles() error { - if p.layout.SBOMs.IsTarball() { - return fmt.Errorf("unable to process the SBOM files for this package: %s is a tarball", p.layout.SBOMs.Path) - } - // If SBOMs were loaded, temporarily place them in the deploy directory - sbomDir := p.layout.SBOMs.Path - if !utils.InvalidPath(sbomDir) { - p.sbomViewFiles, _ = filepath.Glob(filepath.Join(sbomDir, "sbom-viewer-*")) - _, err := sbom.OutputSBOMFiles(sbomDir, layout.SBOMDir, "") - if err != nil { - // Don't stop the deployment, let the user decide if they want to continue the deployment - warning := fmt.Sprintf("Unable to process the SBOM files for this package: %s", err.Error()) - p.warnings = append(p.warnings, warning) - } - } - return nil -} diff --git a/src/pkg/packager/common_test.go b/src/pkg/packager/common_test.go index 105daf0242..1f497415be 100644 --- a/src/pkg/packager/common_test.go +++ b/src/pkg/packager/common_test.go @@ -85,7 +85,6 @@ func TestValidatePackageArchitecture(t *testing.T) { // Create a Packager instance with package architecture set and a mock Kubernetes client. p := &Packager{ - arch: testCase.pkgArch, cluster: &cluster.Cluster{ K8s: &k8s.K8s{ Clientset: mockClient, @@ -94,6 +93,7 @@ func TestValidatePackageArchitecture(t *testing.T) { }, cfg: &types.PackagerConfig{ Pkg: types.ZarfPackage{ + Metadata: types.ZarfMetadata{Architecture: testCase.pkgArch}, Components: []types.ZarfComponent{ { Images: testCase.images, diff --git a/src/pkg/packager/components.go b/src/pkg/packager/components.go index d393eb3e59..84e0b59d04 100644 --- a/src/pkg/packager/components.go +++ b/src/pkg/packager/components.go @@ -6,6 +6,7 @@ package packager import ( "path" + "runtime" "slices" "strings" @@ -24,6 +25,37 @@ const ( excluded ) +// filterComponents removes components not matching the current OS if filterByOS is set. +func (p *Packager) filterComponents() { + // Filter each component to only compatible platforms. + filteredComponents := []types.ZarfComponent{} + for _, component := range p.cfg.Pkg.Components { + // Ignore only filters that are empty + var validArch, validOS bool + + // Test for valid architecture + if component.Only.Cluster.Architecture == "" || component.Only.Cluster.Architecture == p.cfg.Pkg.Metadata.Architecture { + validArch = true + } else { + message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.Cluster.Architecture, p.cfg.Pkg.Metadata.Architecture) + } + + // Test for a valid OS + if component.Only.LocalOS == "" || component.Only.LocalOS == runtime.GOOS { + validOS = true + } else { + message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.LocalOS, runtime.GOOS) + } + + // If both the OS and architecture are valid, add the component to the filtered list + if validArch && validOS { + filteredComponents = append(filteredComponents, component) + } + } + // Update the active package with the filtered components. + p.cfg.Pkg.Components = filteredComponents +} + func (p *Packager) getSelectedComponents() []types.ZarfComponent { var selectedComponents []types.ZarfComponent groupedComponents := map[string][]types.ZarfComponent{} diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index a037812101..9ab104b8b1 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -347,6 +347,9 @@ func (ic *ImportChain) MergeConstants(existing []types.ZarfPackageConstant) (mer // CompatibleComponent determines if this component is compatible with the given create options func CompatibleComponent(c types.ZarfComponent, arch, flavor string) bool { + if arch == zoci.SkeletonArch { + return true + } satisfiesArch := c.Only.Cluster.Architecture == "" || c.Only.Cluster.Architecture == arch satisfiesFlavor := c.Only.Flavor == "" || c.Only.Flavor == flavor return satisfiesArch && satisfiesFlavor diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 02f13b240b..9b01c6fefe 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -10,21 +10,27 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/creator" ) // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. func (p *Packager) Create() (err error) { - cwd, err := os.Getwd() if err != nil { return err } - if err := p.cdToBaseDir(p.cfg.CreateOpts.BaseDir, cwd); err != nil { - return err + if err := os.Chdir(p.cfg.CreateOpts.BaseDir); err != nil { + return fmt.Errorf("unable to access directory %q: %w", p.cfg.CreateOpts.BaseDir, err) } - if err := p.load(); err != nil { + message.Note(fmt.Sprintf("Using build directory %s", p.cfg.CreateOpts.BaseDir)) + + pc := creator.NewPackageCreator(p.cfg.CreateOpts, p.cfg, cwd) + + p.cfg.Pkg, p.warnings, err = pc.LoadPackageDefinition(p.layout) + if err != nil { return err } @@ -37,7 +43,7 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("package creation canceled") } - if err := p.assemble(); err != nil { + if err := pc.Assemble(p.layout, p.cfg.Pkg.Components, p.cfg.Pkg.Metadata.Architecture); err != nil { return err } @@ -46,5 +52,5 @@ func (p *Packager) Create() (err error) { return err } - return p.output() + return pc.Output(p.layout, &p.cfg.Pkg) } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go deleted file mode 100644 index a4c0aa7e58..0000000000 --- a/src/pkg/packager/create_stages.go +++ /dev/null @@ -1,782 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager - -import ( - "context" - "errors" - "fmt" - "os" - "path/filepath" - "slices" - "strconv" - "strings" - "time" - - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/internal/packager/git" - "github.com/defenseunicorns/zarf/src/internal/packager/helm" - "github.com/defenseunicorns/zarf/src/internal/packager/images" - "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" - "github.com/defenseunicorns/zarf/src/internal/packager/sbom" - "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/oci" - "github.com/defenseunicorns/zarf/src/pkg/transform" - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" - "github.com/defenseunicorns/zarf/src/pkg/zoci" - "github.com/defenseunicorns/zarf/src/types" - "github.com/go-git/go-git/v5/plumbing" - "github.com/mholt/archiver/v3" -) - -func (p *Packager) cdToBaseDir(base string, cwd string) error { - if err := os.Chdir(base); err != nil { - return fmt.Errorf("unable to access directory %q: %w", base, err) - } - message.Note(fmt.Sprintf("Using build directory %s", base)) - - // differentials are relative to the current working directory - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { - p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) - } - return nil -} - -func (p *Packager) load() error { - if err := p.readZarfYAML(layout.ZarfYAML); err != nil { - return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) - } - if p.isInitConfig() { - p.cfg.Pkg.Metadata.Version = config.CLIVersion - } - - // Compose components into a single zarf.yaml file - if err := p.composeComponents(); err != nil { - return err - } - - if p.cfg.CreateOpts.IsSkeleton { - return nil - } - - // After components are composed, template the active package. - if err := p.fillActiveTemplate(); err != nil { - return fmt.Errorf("unable to fill values in template: %s", err.Error()) - } - - // After templates are filled process any create extensions - if err := p.processExtensions(); err != nil { - return err - } - - // After we have a full zarf.yaml remove unnecessary repos and images if we are building a differential package - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { - // Load the images and repos from the 'reference' package - if err := p.loadDifferentialData(); err != nil { - return err - } - // Verify the package version of the package we're using as a 'reference' for the differential build is different than the package we're building - // If the package versions are the same return an error - if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == p.cfg.Pkg.Metadata.Version { - return errors.New(lang.PkgCreateErrDifferentialSameVersion) - } - if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || p.cfg.Pkg.Metadata.Version == "" { - return fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") - } - - // Handle any potential differential images/repos before going forward - if err := p.removeCopiesFromDifferentialPackage(); err != nil { - return err - } - } - - return nil -} - -func (p *Packager) assemble() error { - componentSBOMs := map[string]*layout.ComponentSBOM{} - var imageList []transform.Image - for idx, component := range p.cfg.Pkg.Components { - onCreate := component.Actions.OnCreate - onFailure := func() { - if err := p.runActions(onCreate.Defaults, onCreate.OnFailure, nil); err != nil { - message.Debugf("unable to run component failure action: %s", err.Error()) - } - } - if err := p.addComponent(idx, component); err != nil { - onFailure() - return fmt.Errorf("unable to add component %q: %w", component.Name, err) - } - - if err := p.runActions(onCreate.Defaults, onCreate.OnSuccess, nil); err != nil { - onFailure() - return fmt.Errorf("unable to run component success action: %w", err) - } - - if !p.cfg.CreateOpts.SkipSBOM { - componentSBOM, err := p.getFilesToSBOM(component) - if err != nil { - return fmt.Errorf("unable to create component SBOM: %w", err) - } - if componentSBOM != nil && len(componentSBOM.Files) > 0 { - componentSBOMs[component.Name] = componentSBOM - } - } - - // Combine all component images into a single entry for efficient layer reuse. - for _, src := range component.Images { - refInfo, err := transform.ParseImageRef(src) - if err != nil { - return fmt.Errorf("failed to create ref for image %s: %w", src, err) - } - imageList = append(imageList, refInfo) - } - } - - imageList = helpers.Unique(imageList) - var sbomImageList []transform.Image - - // Images are handled separately from other component assets. - if len(imageList) > 0 { - message.HeaderInfof("📦 PACKAGE IMAGES") - - p.layout = p.layout.AddImages() - - var pulled []images.ImgInfo - var err error - - doPull := func() error { - imgConfig := images.ImageConfig{ - ImagesPath: p.layout.Images.Base, - ImageList: imageList, - Insecure: config.CommonOptions.Insecure, - Architectures: []string{p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture}, - RegistryOverrides: p.cfg.CreateOpts.RegistryOverrides, - } - - pulled, err = imgConfig.PullAll() - return err - } - - if err := helpers.Retry(doPull, p.cfg.PkgOpts.Retries, 5*time.Second, message.Warnf); err != nil { - return fmt.Errorf("unable to pull images after %d attempts: %w", p.cfg.PkgOpts.Retries, err) - } - - for _, imgInfo := range pulled { - if err := p.layout.Images.AddV1Image(imgInfo.Img); err != nil { - return err - } - if imgInfo.HasImageLayers { - sbomImageList = append(sbomImageList, imgInfo.RefInfo) - } - } - } - - // Ignore SBOM creation if the flag is set. - if p.cfg.CreateOpts.SkipSBOM { - message.Debug("Skipping image SBOM processing per --skip-sbom flag") - } else { - p.layout = p.layout.AddSBOMs() - if err := sbom.Catalog(componentSBOMs, sbomImageList, p.layout); err != nil { - return fmt.Errorf("unable to create an SBOM catalog for the package: %w", err) - } - } - - return nil -} - -func (p *Packager) assembleSkeleton() error { - if err := p.skeletonizeExtensions(); err != nil { - return err - } - for _, warning := range p.warnings { - message.Warn(warning) - } - for idx, component := range p.cfg.Pkg.Components { - if err := p.addComponent(idx, component); err != nil { - return err - } - - if err := p.layout.Components.Archive(component, false); err != nil { - return err - } - } - checksumChecksum, err := p.generatePackageChecksums() - if err != nil { - return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) - } - p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum - - return p.writeYaml() -} - -// output assumes it is running from cwd, not the build directory -func (p *Packager) output() error { - // Process the component directories into compressed tarballs - // NOTE: This is purposefully being done after the SBOM cataloging - for _, component := range p.cfg.Pkg.Components { - // Make the component a tar archive - if err := p.layout.Components.Archive(component, true); err != nil { - return fmt.Errorf("unable to archive component: %s", err.Error()) - } - } - - // Calculate all the checksums - checksumChecksum, err := p.generatePackageChecksums() - if err != nil { - return fmt.Errorf("unable to generate checksums for the package: %w", err) - } - p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum - - // Save the transformed config. - if err := p.writeYaml(); err != nil { - return fmt.Errorf("unable to write zarf.yaml: %w", err) - } - - // Sign the config file if a key was provided - if p.cfg.CreateOpts.SigningKeyPath != "" { - if err := p.signPackage(p.cfg.CreateOpts.SigningKeyPath, p.cfg.CreateOpts.SigningKeyPassword); err != nil { - return err - } - } - - // Create a remote ref + client for the package (if output is OCI) - // then publish the package to the remote. - if helpers.IsOCIURL(p.cfg.CreateOpts.Output) { - ref, err := zoci.ReferenceFromMetadata(p.cfg.CreateOpts.Output, &p.cfg.Pkg.Metadata, &p.cfg.Pkg.Build) - if err != nil { - return err - } - remote, err := zoci.NewRemote(ref, oci.PlatformForArch(config.GetArch())) - if err != nil { - return err - } - - ctx := context.TODO() - err = remote.PublishPackage(ctx, &p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency) - if err != nil { - return fmt.Errorf("unable to publish package: %w", err) - } - message.HorizontalRule() - flags := "" - if config.CommonOptions.Insecure { - flags = "--insecure" - } - message.Title("To inspect/deploy/pull:", "") - message.ZarfCommand("package inspect %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) - message.ZarfCommand("package deploy %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) - message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) - } else { - // Use the output path if the user specified it. - packageName := filepath.Join(p.cfg.CreateOpts.Output, p.GetPackageName()) - - // Try to remove the package if it already exists. - _ = os.Remove(packageName) - - // Create the package tarball. - if err := p.archivePackage(packageName); err != nil { - return fmt.Errorf("unable to archive package: %w", err) - } - } - - // Output the SBOM files into a directory if specified. - if p.cfg.CreateOpts.ViewSBOM || p.cfg.CreateOpts.SBOMOutputDir != "" { - outputSBOM := p.cfg.CreateOpts.SBOMOutputDir - var sbomDir string - if err := p.layout.SBOMs.Unarchive(); err != nil { - return fmt.Errorf("unable to unarchive SBOMs: %w", err) - } - sbomDir = p.layout.SBOMs.Path - - if outputSBOM != "" { - out, err := sbom.OutputSBOMFiles(sbomDir, outputSBOM, p.cfg.Pkg.Metadata.Name) - if err != nil { - return err - } - sbomDir = out - } - - if p.cfg.CreateOpts.ViewSBOM { - sbom.ViewSBOMFiles(sbomDir) - } - } - return nil -} - -func (p *Packager) getFilesToSBOM(component types.ZarfComponent) (*layout.ComponentSBOM, error) { - componentPaths, err := p.layout.Components.Create(component) - if err != nil { - return nil, err - } - // Create an struct to hold the SBOM information for this component. - componentSBOM := &layout.ComponentSBOM{ - Files: []string{}, - Component: componentPaths, - } - - appendSBOMFiles := func(path string) error { - if utils.IsDir(path) { - files, err := utils.RecursiveFileList(path, nil, false) - if err != nil { - return err - } - componentSBOM.Files = append(componentSBOM.Files, files...) - } else { - info, err := os.Lstat(path) - if err != nil { - return err - } - if info.Mode().IsRegular() { - componentSBOM.Files = append(componentSBOM.Files, path) - } - } - - return nil - } - - for filesIdx, file := range component.Files { - path := filepath.Join(componentPaths.Files, strconv.Itoa(filesIdx), filepath.Base(file.Target)) - err := appendSBOMFiles(path) - if err != nil { - return nil, err - } - } - - for dataIdx, data := range component.DataInjections { - path := filepath.Join(componentPaths.DataInjections, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) - - err := appendSBOMFiles(path) - if err != nil { - return nil, err - } - } - - return componentSBOM, nil -} - -func (p *Packager) addComponent(index int, component types.ZarfComponent) error { - message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) - - isSkeleton := p.cfg.CreateOpts.IsSkeleton - - componentPaths, err := p.layout.Components.Create(component) - if err != nil { - return err - } - - if isSkeleton && component.DeprecatedCosignKeyPath != "" { - dst := filepath.Join(componentPaths.Base, "cosign.pub") - err := utils.CreatePathAndCopy(component.DeprecatedCosignKeyPath, dst) - if err != nil { - return err - } - p.cfg.Pkg.Components[index].DeprecatedCosignKeyPath = "cosign.pub" - } - - // TODO: (@WSTARR) Shim the skeleton component's create action dirs to be empty. This prevents actions from failing by cd'ing into directories that will be flattened. - if isSkeleton { - component.Actions.OnCreate.Defaults.Dir = "" - resetActions := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { - for idx := range actions { - actions[idx].Dir = nil - } - return actions - } - component.Actions.OnCreate.Before = resetActions(component.Actions.OnCreate.Before) - component.Actions.OnCreate.After = resetActions(component.Actions.OnCreate.After) - component.Actions.OnCreate.OnSuccess = resetActions(component.Actions.OnCreate.OnSuccess) - component.Actions.OnCreate.OnFailure = resetActions(component.Actions.OnCreate.OnFailure) - } - - onCreate := component.Actions.OnCreate - if !isSkeleton { - if err := p.runActions(onCreate.Defaults, onCreate.Before, nil); err != nil { - return fmt.Errorf("unable to run component before action: %w", err) - } - } - - // If any helm charts are defined, process them. - for chartIdx, chart := range component.Charts { - - helmCfg := helm.New(chart, componentPaths.Charts, componentPaths.Values) - - if isSkeleton { - if chart.LocalPath != "" { - rel := filepath.Join(layout.ChartsDir, fmt.Sprintf("%s-%d", chart.Name, chartIdx)) - dst := filepath.Join(componentPaths.Base, rel) - - err := utils.CreatePathAndCopy(chart.LocalPath, dst) - if err != nil { - return err - } - - p.cfg.Pkg.Components[index].Charts[chartIdx].LocalPath = rel - } - - for valuesIdx, path := range chart.ValuesFiles { - if helpers.IsURL(path) { - continue - } - - rel := helm.StandardValuesName(layout.ValuesDir, chart, valuesIdx) - p.cfg.Pkg.Components[index].Charts[chartIdx].ValuesFiles[valuesIdx] = rel - - if err := utils.CreatePathAndCopy(path, filepath.Join(componentPaths.Base, rel)); err != nil { - return fmt.Errorf("unable to copy chart values file %s: %w", path, err) - } - } - } else { - err := helmCfg.PackageChart(componentPaths.Charts) - if err != nil { - return err - } - } - } - - for filesIdx, file := range component.Files { - message.Debugf("Loading %#v", file) - - rel := filepath.Join(layout.FilesDir, strconv.Itoa(filesIdx), filepath.Base(file.Target)) - dst := filepath.Join(componentPaths.Base, rel) - destinationDir := filepath.Dir(dst) - - if helpers.IsURL(file.Source) { - if isSkeleton { - continue - } - - if file.ExtractPath != "" { - - // get the compressedFileName from the source - compressedFileName, err := helpers.ExtractBasePathFromURL(file.Source) - if err != nil { - return fmt.Errorf(lang.ErrFileNameExtract, file.Source, err.Error()) - } - - compressedFile := filepath.Join(componentPaths.Temp, compressedFileName) - - // If the file is an archive, download it to the componentPath.Temp - if err := utils.DownloadToFile(file.Source, compressedFile, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) - } - - err = archiver.Extract(compressedFile, file.ExtractPath, destinationDir) - if err != nil { - return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, compressedFileName, err.Error()) - } - - } else { - if err := utils.DownloadToFile(file.Source, dst, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) - } - } - - } else { - if file.ExtractPath != "" { - if err := archiver.Extract(file.Source, file.ExtractPath, destinationDir); err != nil { - return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, file.Source, err.Error()) - } - } else { - if err := utils.CreatePathAndCopy(file.Source, dst); err != nil { - return fmt.Errorf("unable to copy file %s: %w", file.Source, err) - } - } - - } - - if file.ExtractPath != "" { - // Make sure dst reflects the actual file or directory. - updatedExtractedFileOrDir := filepath.Join(destinationDir, file.ExtractPath) - if updatedExtractedFileOrDir != dst { - if err := os.Rename(updatedExtractedFileOrDir, dst); err != nil { - return fmt.Errorf(lang.ErrWritingFile, dst, err) - } - } - } - - if isSkeleton { - // Change the source to the new relative source directory (any remote files will have been skipped above) - p.cfg.Pkg.Components[index].Files[filesIdx].Source = rel - // Remove the extractPath from a skeleton since it will already extract it - p.cfg.Pkg.Components[index].Files[filesIdx].ExtractPath = "" - } - - // Abort packaging on invalid shasum (if one is specified). - if file.Shasum != "" { - if err := utils.SHAsMatch(dst, file.Shasum); err != nil { - return err - } - } - - if file.Executable || utils.IsDir(dst) { - _ = os.Chmod(dst, helpers.ReadWriteExecuteUser) - } else { - _ = os.Chmod(dst, helpers.ReadWriteUser) - } - } - - if len(component.DataInjections) > 0 { - spinner := message.NewProgressSpinner("Loading data injections") - defer spinner.Stop() - - for dataIdx, data := range component.DataInjections { - spinner.Updatef("Copying data injection %s for %s", data.Target.Path, data.Target.Selector) - - rel := filepath.Join(layout.DataInjectionsDir, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) - dst := filepath.Join(componentPaths.Base, rel) - - if helpers.IsURL(data.Source) { - if isSkeleton { - continue - } - if err := utils.DownloadToFile(data.Source, dst, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, data.Source, err.Error()) - } - } else { - if err := utils.CreatePathAndCopy(data.Source, dst); err != nil { - return fmt.Errorf("unable to copy data injection %s: %s", data.Source, err.Error()) - } - if isSkeleton { - p.cfg.Pkg.Components[index].DataInjections[dataIdx].Source = rel - } - } - } - spinner.Success() - } - - if len(component.Manifests) > 0 { - // Get the proper count of total manifests to add. - manifestCount := 0 - - for _, manifest := range component.Manifests { - manifestCount += len(manifest.Files) - manifestCount += len(manifest.Kustomizations) - } - - spinner := message.NewProgressSpinner("Loading %d K8s manifests", manifestCount) - defer spinner.Stop() - - // Iterate over all manifests. - for manifestIdx, manifest := range component.Manifests { - for fileIdx, path := range manifest.Files { - rel := filepath.Join(layout.ManifestsDir, fmt.Sprintf("%s-%d.yaml", manifest.Name, fileIdx)) - dst := filepath.Join(componentPaths.Base, rel) - - // Copy manifests without any processing. - spinner.Updatef("Copying manifest %s", path) - if helpers.IsURL(path) { - if isSkeleton { - continue - } - if err := utils.DownloadToFile(path, dst, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, path, err.Error()) - } - } else { - if err := utils.CreatePathAndCopy(path, dst); err != nil { - return fmt.Errorf("unable to copy manifest %s: %w", path, err) - } - if isSkeleton { - p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files[fileIdx] = rel - } - } - } - - for kustomizeIdx, path := range manifest.Kustomizations { - // Generate manifests from kustomizations and place in the package. - spinner.Updatef("Building kustomization for %s", path) - - kname := fmt.Sprintf("kustomization-%s-%d.yaml", manifest.Name, kustomizeIdx) - rel := filepath.Join(layout.ManifestsDir, kname) - dst := filepath.Join(componentPaths.Base, rel) - - if err := kustomize.Build(path, dst, manifest.KustomizeAllowAnyDirectory); err != nil { - return fmt.Errorf("unable to build kustomization %s: %w", path, err) - } - if isSkeleton { - p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files = append(p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files, rel) - } - } - if isSkeleton { - // remove kustomizations - p.cfg.Pkg.Components[index].Manifests[manifestIdx].Kustomizations = nil - } - } - spinner.Success() - } - - // Load all specified git repos. - if len(component.Repos) > 0 && !isSkeleton { - spinner := message.NewProgressSpinner("Loading %d git repos", len(component.Repos)) - defer spinner.Stop() - - for _, url := range component.Repos { - // Pull all the references if there is no `@` in the string. - gitCfg := git.NewWithSpinner(types.GitServerInfo{}, spinner) - if err := gitCfg.Pull(url, componentPaths.Repos, false); err != nil { - return fmt.Errorf("unable to pull git repo %s: %w", url, err) - } - } - spinner.Success() - } - - if !isSkeleton { - if err := p.runActions(onCreate.Defaults, onCreate.After, nil); err != nil { - return fmt.Errorf("unable to run component after action: %w", err) - } - } - - return nil -} - -// generateChecksum walks through all of the files starting at the base path and generates a checksum file. -// Each file within the basePath represents a layer within the Zarf package. -// generateChecksum returns a SHA256 checksum of the checksums.txt file. -func (p *Packager) generatePackageChecksums() (string, error) { - // Loop over the "loaded" files - var checksumsData = []string{} - for rel, abs := range p.layout.Files() { - if rel == layout.ZarfYAML || rel == layout.Checksums { - continue - } - - sum, err := utils.GetSHA256OfFile(abs) - if err != nil { - return "", err - } - checksumsData = append(checksumsData, fmt.Sprintf("%s %s", sum, rel)) - } - slices.Sort(checksumsData) - - // Create the checksums file - checksumsFilePath := p.layout.Checksums - if err := os.WriteFile(checksumsFilePath, []byte(strings.Join(checksumsData, "\n")+"\n"), helpers.ReadWriteUser); err != nil { - return "", err - } - - // Calculate the checksum of the checksum file - return utils.GetSHA256OfFile(checksumsFilePath) -} - -// loadDifferentialData extracts the zarf config of a designated 'reference' package that we are building a differential over and creates a list of all images and repos that are in the reference package -func (p *Packager) loadDifferentialData() error { - // Save the fact that this is a differential build into the build data of the package - p.cfg.Pkg.Build.Differential = true - - tmpDir, _ := utils.MakeTempDir(config.CommonOptions.TempDirectory) - defer os.RemoveAll(tmpDir) - - // Load the package spec of the package we're using as a 'reference' for the differential build - if helpers.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) { - remote, err := zoci.NewRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, oci.PlatformForArch(config.GetArch())) - if err != nil { - return err - } - pkg, err := remote.FetchZarfYAML(context.TODO()) - if err != nil { - return err - } - err = utils.WriteYaml(filepath.Join(tmpDir, layout.ZarfYAML), pkg, helpers.ReadWriteUser) - if err != nil { - return err - } - } else { - if err := archiver.Extract(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, layout.ZarfYAML, tmpDir); err != nil { - return fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error()) - } - } - - var differentialZarfConfig types.ZarfPackage - if err := utils.ReadYaml(filepath.Join(tmpDir, layout.ZarfYAML), &differentialZarfConfig); err != nil { - return fmt.Errorf("unable to load the differential zarf package spec: %s", err.Error()) - } - - // Generate a map of all the images and repos that are included in the provided package - allIncludedImagesMap := map[string]bool{} - allIncludedReposMap := map[string]bool{} - for _, component := range differentialZarfConfig.Components { - for _, image := range component.Images { - allIncludedImagesMap[image] = true - } - for _, repo := range component.Repos { - allIncludedReposMap[repo] = true - } - } - - p.cfg.CreateOpts.DifferentialData.DifferentialImages = allIncludedImagesMap - p.cfg.CreateOpts.DifferentialData.DifferentialRepos = allIncludedReposMap - p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version - - return nil -} - -// removeCopiesFromDifferentialPackage will remove any images and repos that are already included in the reference package from the new package -func (p *Packager) removeCopiesFromDifferentialPackage() error { - // If a differential build was not requested, continue on as normal - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath == "" { - return nil - } - - // Loop through all of the components to determine if any of them are using already included images or repos - componentMap := make(map[int]types.ZarfComponent) - for idx, component := range p.cfg.Pkg.Components { - newImageList := []string{} - newRepoList := []string{} - // Generate a list of all unique images for this component - for _, img := range component.Images { - // If a image doesn't have a ref (or is a commonly reused ref), we will include this image in the differential package - imgRef, err := transform.ParseImageRef(img) - if err != nil { - return fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) - } - - // Only include new images or images that have a commonly overwritten tag - imgTag := imgRef.TagOrDigest - useImgAnyways := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly" - if useImgAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialImages[img] { - newImageList = append(newImageList, img) - } else { - message.Debugf("Image %s is already included in the differential package", img) - } - } - - // Generate a list of all unique repos for this component - for _, repoURL := range component.Repos { - // Split the remote url and the zarf reference - _, refPlain, err := transform.GitURLSplitRef(repoURL) - if err != nil { - return err - } - - var ref plumbing.ReferenceName - // Parse the ref from the git URL. - if refPlain != "" { - ref = git.ParseRef(refPlain) - } - - // Only include new repos or repos that were not referenced by a specific commit sha or tag - useRepoAnyways := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain)) - if useRepoAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialRepos[repoURL] { - newRepoList = append(newRepoList, repoURL) - } else { - message.Debugf("Repo %s is already included in the differential package", repoURL) - } - } - - // Update the component with the unique lists of repos and images - component.Images = newImageList - component.Repos = newRepoList - componentMap[idx] = component - } - - // Update the package with the new component list - for idx, component := range componentMap { - p.cfg.Pkg.Components[idx] = component - } - - return nil -} diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/creator/compose.go similarity index 50% rename from src/pkg/packager/compose.go rename to src/pkg/packager/creator/compose.go index 6bbbf22fff..dd2e429731 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/creator/compose.go @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: 2021-Present The Zarf Authors -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager +// Package creator contains functions for creating Zarf packages. +package creator import ( "github.com/defenseunicorns/zarf/src/pkg/message" @@ -10,17 +10,19 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -// composeComponents builds the composed components list for the current config. -func (p *Packager) composeComponents() error { +// ComposeComponents composes components and their dependencies into a single Zarf package using an import chain. +func ComposeComponents(pkg types.ZarfPackage, flavor string) (types.ZarfPackage, []string, error) { components := []types.ZarfComponent{} + warnings := []string{} - pkgVars := p.cfg.Pkg.Variables - pkgConsts := p.cfg.Pkg.Constants + pkgVars := pkg.Variables + pkgConsts := pkg.Constants - for i, component := range p.cfg.Pkg.Components { - arch := p.arch - // filter by architecture - if !composer.CompatibleComponent(component, arch, p.cfg.CreateOpts.Flavor) { + arch := pkg.Metadata.Architecture + + for i, component := range pkg.Components { + // filter by architecture and flavor + if !composer.CompatibleComponent(component, arch, flavor) { continue } @@ -29,20 +31,20 @@ func (p *Packager) composeComponents() error { component.Only.Flavor = "" // build the import chain - chain, err := composer.NewImportChain(component, i, p.cfg.Pkg.Metadata.Name, arch, p.cfg.CreateOpts.Flavor) + chain, err := composer.NewImportChain(component, i, pkg.Metadata.Name, arch, flavor) if err != nil { - return err + return types.ZarfPackage{}, nil, err } message.Debugf("%s", chain) // migrate any deprecated component configurations now - warnings := chain.Migrate(p.cfg.Pkg.Build) - p.warnings = append(p.warnings, warnings...) + warning := chain.Migrate(pkg.Build) + warnings = append(warnings, warning...) // get the composed component composed, err := chain.Compose() if err != nil { - return err + return types.ZarfPackage{}, nil, err } components = append(components, *composed) @@ -52,10 +54,10 @@ func (p *Packager) composeComponents() error { } // set the filtered + composed components - p.cfg.Pkg.Components = components + pkg.Components = components - p.cfg.Pkg.Variables = pkgVars - p.cfg.Pkg.Constants = pkgConsts + pkg.Variables = pkgVars + pkg.Constants = pkgConsts - return nil + return pkg, warnings, nil } diff --git a/src/pkg/packager/creator/creator.go b/src/pkg/packager/creator/creator.go new file mode 100644 index 0000000000..5b0f3c27b1 --- /dev/null +++ b/src/pkg/packager/creator/creator.go @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package creator contains functions for creating Zarf packages. +package creator + +import ( + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/types" +) + +// Creator is an interface for creating Zarf packages. +type Creator interface { + LoadPackageDefinition(dst *layout.PackagePaths) (pkg types.ZarfPackage, warnings []string, err error) + Assemble(dst *layout.PackagePaths, components []types.ZarfComponent, arch string) error + Output(dst *layout.PackagePaths, pkg *types.ZarfPackage) error +} diff --git a/src/pkg/packager/creator/differential.go b/src/pkg/packager/creator/differential.go new file mode 100644 index 0000000000..4533531e03 --- /dev/null +++ b/src/pkg/packager/creator/differential.go @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package creator contains functions for creating Zarf packages. +package creator + +import ( + "fmt" + "os" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/internal/packager/git" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/packager/sources" + "github.com/defenseunicorns/zarf/src/pkg/transform" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/types" + "github.com/go-git/go-git/v5/plumbing" +) + +// loadDifferentialData sets any images and repos from the existing reference package in the DifferentialData and returns it. +func loadDifferentialData(diffPkgPath string) (diffData *types.DifferentialData, err error) { + tmpdir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) + if err != nil { + return nil, err + } + + diffLayout := layout.New(tmpdir) + defer os.RemoveAll(diffLayout.Base) + + src, err := sources.New(&types.ZarfPackageOptions{ + PackageSource: diffPkgPath, + }) + if err != nil { + return nil, err + } + + if err := src.LoadPackageMetadata(diffLayout, false, false); err != nil { + return nil, err + } + + var diffPkg types.ZarfPackage + if err := utils.ReadYaml(diffLayout.ZarfYAML, &diffPkg); err != nil { + return nil, fmt.Errorf("error reading the differential Zarf package: %w", err) + } + + allIncludedImagesMap := map[string]bool{} + allIncludedReposMap := map[string]bool{} + + for _, component := range diffPkg.Components { + for _, image := range component.Images { + allIncludedImagesMap[image] = true + } + for _, repo := range component.Repos { + allIncludedReposMap[repo] = true + } + } + + return &types.DifferentialData{ + DifferentialImages: allIncludedImagesMap, + DifferentialRepos: allIncludedReposMap, + DifferentialPackageVersion: diffPkg.Metadata.Version, + }, nil +} + +// removeCopiesFromComponents removes any images and repos already present in the reference package components. +func removeCopiesFromComponents(components []types.ZarfComponent, loadedDiffData *types.DifferentialData) (diffComponents []types.ZarfComponent, err error) { + for _, component := range components { + newImageList := []string{} + newRepoList := []string{} + + for _, img := range component.Images { + imgRef, err := transform.ParseImageRef(img) + if err != nil { + return nil, fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) + } + + imgTag := imgRef.TagOrDigest + includeImage := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly" + if includeImage || !loadedDiffData.DifferentialImages[img] { + newImageList = append(newImageList, img) + } + } + + for _, repoURL := range component.Repos { + _, refPlain, err := transform.GitURLSplitRef(repoURL) + if err != nil { + return nil, err + } + + var ref plumbing.ReferenceName + if refPlain != "" { + ref = git.ParseRef(refPlain) + } + + includeRepo := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain)) + if includeRepo || !loadedDiffData.DifferentialRepos[repoURL] { + newRepoList = append(newRepoList, repoURL) + } + } + + component.Images = newImageList + component.Repos = newRepoList + diffComponents = append(diffComponents, component) + } + + return diffComponents, nil +} diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go new file mode 100644 index 0000000000..36a86940ae --- /dev/null +++ b/src/pkg/packager/creator/normal.go @@ -0,0 +1,560 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package creator contains functions for creating Zarf packages. +package creator + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/extensions/bigbang" + "github.com/defenseunicorns/zarf/src/internal/packager/git" + "github.com/defenseunicorns/zarf/src/internal/packager/helm" + "github.com/defenseunicorns/zarf/src/internal/packager/images" + "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" + "github.com/defenseunicorns/zarf/src/internal/packager/sbom" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/packager/actions" + "github.com/defenseunicorns/zarf/src/pkg/packager/sources" + "github.com/defenseunicorns/zarf/src/pkg/transform" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/pkg/zoci" + "github.com/defenseunicorns/zarf/src/types" + "github.com/mholt/archiver/v3" +) + +var ( + // verify that PackageCreator implements Creator + _ Creator = (*PackageCreator)(nil) +) + +// PackageCreator provides methods for creating normal (not skeleton) Zarf packages. +type PackageCreator struct { + createOpts types.ZarfCreateOptions + + // TODO: (@lucasrod16) remove PackagerConfig once actions do not depend on it: https://github.com/defenseunicorns/zarf/pull/2276 + cfg *types.PackagerConfig +} + +// NewPackageCreator returns a new PackageCreator. +func NewPackageCreator(createOpts types.ZarfCreateOptions, cfg *types.PackagerConfig, cwd string) *PackageCreator { + // differentials are relative to the current working directory + if createOpts.DifferentialPackagePath != "" { + createOpts.DifferentialPackagePath = filepath.Join(cwd, createOpts.DifferentialPackagePath) + } + + return &PackageCreator{createOpts, cfg} +} + +// LoadPackageDefinition loads and configures a zarf.yaml file during package create. +func (pc *PackageCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg types.ZarfPackage, warnings []string, err error) { + pkg, warnings, err = dst.ReadZarfYAML(layout.ZarfYAML) + if err != nil { + return types.ZarfPackage{}, nil, err + } + + pkg.Metadata.Architecture = config.GetArch(pkg.Metadata.Architecture) + + // Compose components into a single zarf.yaml file + pkg, composeWarnings, err := ComposeComponents(pkg, pc.createOpts.Flavor) + if err != nil { + return types.ZarfPackage{}, nil, err + } + + warnings = append(warnings, composeWarnings...) + + // After components are composed, template the active package. + pkg, templateWarnings, err := FillActiveTemplate(pkg, pc.createOpts.SetVariables) + if err != nil { + return types.ZarfPackage{}, nil, fmt.Errorf("unable to fill values in template: %w", err) + } + + warnings = append(warnings, templateWarnings...) + + // After templates are filled process any create extensions + pkg.Components, err = pc.processExtensions(pkg.Components, dst, pkg.Metadata.YOLO) + if err != nil { + return types.ZarfPackage{}, nil, err + } + + // If we are creating a differential package, remove duplicate images and repos. + if pc.createOpts.DifferentialPackagePath != "" { + pkg.Build.Differential = true + + diffData, err := loadDifferentialData(pc.createOpts.DifferentialPackagePath) + if err != nil { + return types.ZarfPackage{}, nil, err + } + + pkg.Build.DifferentialPackageVersion = diffData.DifferentialPackageVersion + + versionsMatch := diffData.DifferentialPackageVersion == pkg.Metadata.Version + if versionsMatch { + return types.ZarfPackage{}, nil, errors.New(lang.PkgCreateErrDifferentialSameVersion) + } + + noVersionSet := diffData.DifferentialPackageVersion == "" || pkg.Metadata.Version == "" + if noVersionSet { + return types.ZarfPackage{}, nil, errors.New(lang.PkgCreateErrDifferentialNoVersion) + } + + pkg.Components, err = removeCopiesFromComponents(pkg.Components, diffData) + if err != nil { + return types.ZarfPackage{}, nil, err + } + } + + return pkg, warnings, nil +} + +// Assemble assembles all of the package assets into Zarf's tmp directory layout. +func (pc *PackageCreator) Assemble(dst *layout.PackagePaths, components []types.ZarfComponent, arch string) error { + var imageList []transform.Image + + skipSBOMFlagUsed := pc.createOpts.SkipSBOM + componentSBOMs := map[string]*layout.ComponentSBOM{} + + for _, component := range components { + onCreate := component.Actions.OnCreate + + onFailure := func() { + if err := actions.Run(pc.cfg, onCreate.Defaults, onCreate.OnFailure, nil); err != nil { + message.Debugf("unable to run component failure action: %s", err.Error()) + } + } + + if err := pc.addComponent(component, dst); err != nil { + onFailure() + return fmt.Errorf("unable to add component %q: %w", component.Name, err) + } + + if err := actions.Run(pc.cfg, onCreate.Defaults, onCreate.OnSuccess, nil); err != nil { + onFailure() + return fmt.Errorf("unable to run component success action: %w", err) + } + + if !skipSBOMFlagUsed { + componentSBOM, err := pc.getFilesToSBOM(component, dst) + if err != nil { + return fmt.Errorf("unable to create component SBOM: %w", err) + } + if componentSBOM != nil && len(componentSBOM.Files) > 0 { + componentSBOMs[component.Name] = componentSBOM + } + } + + // Combine all component images into a single entry for efficient layer reuse. + for _, src := range component.Images { + refInfo, err := transform.ParseImageRef(src) + if err != nil { + return fmt.Errorf("failed to create ref for image %s: %w", src, err) + } + imageList = append(imageList, refInfo) + } + } + + imageList = helpers.Unique(imageList) + var sbomImageList []transform.Image + + // Images are handled separately from other component assets. + if len(imageList) > 0 { + message.HeaderInfof("📦 PACKAGE IMAGES") + + dst.AddImages() + + var pulled []images.ImgInfo + var err error + + doPull := func() error { + imgConfig := images.ImageConfig{ + ImagesPath: dst.Images.Base, + ImageList: imageList, + Insecure: config.CommonOptions.Insecure, + Architectures: []string{arch}, + RegistryOverrides: pc.createOpts.RegistryOverrides, + } + + pulled, err = imgConfig.PullAll() + return err + } + + if err := helpers.Retry(doPull, 3, 5*time.Second, message.Warnf); err != nil { + return fmt.Errorf("unable to pull images after 3 attempts: %w", err) + } + + for _, imgInfo := range pulled { + if err := dst.Images.AddV1Image(imgInfo.Img); err != nil { + return err + } + if imgInfo.HasImageLayers { + sbomImageList = append(sbomImageList, imgInfo.RefInfo) + } + } + } + + // Ignore SBOM creation if the flag is set. + if skipSBOMFlagUsed { + message.Debug("Skipping image SBOM processing per --skip-sbom flag") + } else { + dst.AddSBOMs() + if err := sbom.Catalog(componentSBOMs, sbomImageList, dst); err != nil { + return fmt.Errorf("unable to create an SBOM catalog for the package: %w", err) + } + } + + return nil +} + +// Output does the following: +// +// - archives components +// +// - generates checksums for all package files +// +// - writes the loaded zarf.yaml to disk +// +// - signs the package +// +// - writes the Zarf package as a tarball to a local directory, +// or an OCI registry based on the --output flag +func (pc *PackageCreator) Output(dst *layout.PackagePaths, pkg *types.ZarfPackage) (err error) { + // Process the component directories into compressed tarballs + // NOTE: This is purposefully being done after the SBOM cataloging + for _, component := range pkg.Components { + // Make the component a tar archive + if err := dst.Components.Archive(component, true); err != nil { + return fmt.Errorf("unable to archive component: %s", err.Error()) + } + } + + // Calculate all the checksums + pkg.Metadata.AggregateChecksum, err = dst.GenerateChecksums() + if err != nil { + return fmt.Errorf("unable to generate checksums for the package: %w", err) + } + + if err := recordPackageMetadata(pkg, pc.createOpts); err != nil { + return err + } + + if err := utils.WriteYaml(dst.ZarfYAML, pkg, helpers.ReadUser); err != nil { + return fmt.Errorf("unable to write zarf.yaml: %w", err) + } + + // Sign the package if a key has been provided + if pc.createOpts.SigningKeyPath != "" { + if err := dst.SignPackage(pc.createOpts.SigningKeyPath, pc.createOpts.SigningKeyPassword); err != nil { + return err + } + } + + // Create a remote ref + client for the package (if output is OCI) + // then publish the package to the remote. + if helpers.IsOCIURL(pc.createOpts.Output) { + ref, err := zoci.ReferenceFromMetadata(pc.createOpts.Output, &pkg.Metadata, &pkg.Build) + if err != nil { + return err + } + remote, err := zoci.NewRemote(ref, oci.PlatformForArch(config.GetArch())) + if err != nil { + return err + } + + ctx := context.TODO() + err = remote.PublishPackage(ctx, pkg, dst, config.CommonOptions.OCIConcurrency) + if err != nil { + return fmt.Errorf("unable to publish package: %w", err) + } + message.HorizontalRule() + flags := "" + if config.CommonOptions.Insecure { + flags = "--insecure" + } + message.Title("To inspect/deploy/pull:", "") + message.ZarfCommand("package inspect %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) + message.ZarfCommand("package deploy %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) + message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) + } else { + // Use the output path if the user specified it. + packageName := fmt.Sprintf("%s%s", sources.NameFromMetadata(pkg, pc.createOpts.IsSkeleton), sources.PkgSuffix(pkg.Metadata.Uncompressed)) + tarballPath := filepath.Join(pc.createOpts.Output, packageName) + + // Try to remove the package if it already exists. + _ = os.Remove(tarballPath) + + // Create the package tarball. + if err := dst.ArchivePackage(tarballPath, pc.createOpts.MaxPackageSizeMB); err != nil { + return fmt.Errorf("unable to archive package: %w", err) + } + } + + // Output the SBOM files into a directory if specified. + if pc.createOpts.ViewSBOM || pc.createOpts.SBOMOutputDir != "" { + outputSBOM := pc.createOpts.SBOMOutputDir + var sbomDir string + if err := dst.SBOMs.Unarchive(); err != nil { + return fmt.Errorf("unable to unarchive SBOMs: %w", err) + } + sbomDir = dst.SBOMs.Path + + if outputSBOM != "" { + out, err := dst.SBOMs.OutputSBOMFiles(outputSBOM, pkg.Metadata.Name) + if err != nil { + return err + } + sbomDir = out + } + + if pc.createOpts.ViewSBOM { + sbom.ViewSBOMFiles(sbomDir) + } + } + return nil +} + +func (pc *PackageCreator) processExtensions(components []types.ZarfComponent, layout *layout.PackagePaths, isYOLO bool) (processedComponents []types.ZarfComponent, err error) { + // Create component paths and process extensions for each component. + for _, c := range components { + componentPaths, err := layout.Components.Create(c) + if err != nil { + return nil, err + } + + // Big Bang + if c.Extensions.BigBang != nil { + if c, err = bigbang.Run(isYOLO, componentPaths, c); err != nil { + return nil, fmt.Errorf("unable to process bigbang extension: %w", err) + } + } + + processedComponents = append(processedComponents, c) + } + + return processedComponents, nil +} + +func (pc *PackageCreator) addComponent(component types.ZarfComponent, dst *layout.PackagePaths) error { + message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) + + componentPaths, err := dst.Components.Create(component) + if err != nil { + return err + } + + onCreate := component.Actions.OnCreate + if err := actions.Run(pc.cfg, onCreate.Defaults, onCreate.Before, nil); err != nil { + return fmt.Errorf("unable to run component before action: %w", err) + } + + // If any helm charts are defined, process them. + for _, chart := range component.Charts { + helmCfg := helm.New(chart, componentPaths.Charts, componentPaths.Values) + if err := helmCfg.PackageChart(componentPaths.Charts); err != nil { + return err + } + } + + for filesIdx, file := range component.Files { + message.Debugf("Loading %#v", file) + + rel := filepath.Join(layout.FilesDir, strconv.Itoa(filesIdx), filepath.Base(file.Target)) + dst := filepath.Join(componentPaths.Base, rel) + destinationDir := filepath.Dir(dst) + + if helpers.IsURL(file.Source) { + if file.ExtractPath != "" { + // get the compressedFileName from the source + compressedFileName, err := helpers.ExtractBasePathFromURL(file.Source) + if err != nil { + return fmt.Errorf(lang.ErrFileNameExtract, file.Source, err.Error()) + } + + compressedFile := filepath.Join(componentPaths.Temp, compressedFileName) + + // If the file is an archive, download it to the componentPath.Temp + if err := utils.DownloadToFile(file.Source, compressedFile, component.DeprecatedCosignKeyPath); err != nil { + return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) + } + + err = archiver.Extract(compressedFile, file.ExtractPath, destinationDir) + if err != nil { + return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, compressedFileName, err.Error()) + } + } else { + if err := utils.DownloadToFile(file.Source, dst, component.DeprecatedCosignKeyPath); err != nil { + return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) + } + } + } else { + if file.ExtractPath != "" { + if err := archiver.Extract(file.Source, file.ExtractPath, destinationDir); err != nil { + return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, file.Source, err.Error()) + } + } else { + if err := utils.CreatePathAndCopy(file.Source, dst); err != nil { + return fmt.Errorf("unable to copy file %s: %w", file.Source, err) + } + } + } + + if file.ExtractPath != "" { + // Make sure dst reflects the actual file or directory. + updatedExtractedFileOrDir := filepath.Join(destinationDir, file.ExtractPath) + if updatedExtractedFileOrDir != dst { + if err := os.Rename(updatedExtractedFileOrDir, dst); err != nil { + return fmt.Errorf(lang.ErrWritingFile, dst, err) + } + } + } + + // Abort packaging on invalid shasum (if one is specified). + if file.Shasum != "" { + if err := utils.SHAsMatch(dst, file.Shasum); err != nil { + return err + } + } + + if file.Executable || utils.IsDir(dst) { + _ = os.Chmod(dst, helpers.ReadWriteExecuteUser) + } else { + _ = os.Chmod(dst, helpers.ReadWriteUser) + } + } + + if len(component.DataInjections) > 0 { + spinner := message.NewProgressSpinner("Loading data injections") + defer spinner.Stop() + + for dataIdx, data := range component.DataInjections { + spinner.Updatef("Copying data injection %s for %s", data.Target.Path, data.Target.Selector) + + rel := filepath.Join(layout.DataInjectionsDir, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) + dst := filepath.Join(componentPaths.Base, rel) + + if helpers.IsURL(data.Source) { + if err := utils.DownloadToFile(data.Source, dst, component.DeprecatedCosignKeyPath); err != nil { + return fmt.Errorf(lang.ErrDownloading, data.Source, err.Error()) + } + } else { + if err := utils.CreatePathAndCopy(data.Source, dst); err != nil { + return fmt.Errorf("unable to copy data injection %s: %s", data.Source, err.Error()) + } + } + } + spinner.Success() + } + + if len(component.Manifests) > 0 { + // Get the proper count of total manifests to add. + manifestCount := 0 + + for _, manifest := range component.Manifests { + manifestCount += len(manifest.Files) + manifestCount += len(manifest.Kustomizations) + } + + spinner := message.NewProgressSpinner("Loading %d K8s manifests", manifestCount) + defer spinner.Stop() + + // Iterate over all manifests. + for _, manifest := range component.Manifests { + for fileIdx, path := range manifest.Files { + rel := filepath.Join(layout.ManifestsDir, fmt.Sprintf("%s-%d.yaml", manifest.Name, fileIdx)) + dst := filepath.Join(componentPaths.Base, rel) + + // Copy manifests without any processing. + spinner.Updatef("Copying manifest %s", path) + if helpers.IsURL(path) { + if err := utils.DownloadToFile(path, dst, component.DeprecatedCosignKeyPath); err != nil { + return fmt.Errorf(lang.ErrDownloading, path, err.Error()) + } + } else { + if err := utils.CreatePathAndCopy(path, dst); err != nil { + return fmt.Errorf("unable to copy manifest %s: %w", path, err) + } + } + } + + for kustomizeIdx, path := range manifest.Kustomizations { + // Generate manifests from kustomizations and place in the package. + spinner.Updatef("Building kustomization for %s", path) + + kname := fmt.Sprintf("kustomization-%s-%d.yaml", manifest.Name, kustomizeIdx) + rel := filepath.Join(layout.ManifestsDir, kname) + dst := filepath.Join(componentPaths.Base, rel) + + if err := kustomize.Build(path, dst, manifest.KustomizeAllowAnyDirectory); err != nil { + return fmt.Errorf("unable to build kustomization %s: %w", path, err) + } + } + } + spinner.Success() + } + + // Load all specified git repos. + if len(component.Repos) > 0 { + spinner := message.NewProgressSpinner("Loading %d git repos", len(component.Repos)) + defer spinner.Stop() + + for _, url := range component.Repos { + // Pull all the references if there is no `@` in the string. + gitCfg := git.NewWithSpinner(types.GitServerInfo{}, spinner) + if err := gitCfg.Pull(url, componentPaths.Repos, false); err != nil { + return fmt.Errorf("unable to pull git repo %s: %w", url, err) + } + } + spinner.Success() + } + + if err := actions.Run(pc.cfg, onCreate.Defaults, onCreate.After, nil); err != nil { + return fmt.Errorf("unable to run component after action: %w", err) + } + + return nil +} + +func (pc *PackageCreator) getFilesToSBOM(component types.ZarfComponent, dst *layout.PackagePaths) (*layout.ComponentSBOM, error) { + componentPaths, err := dst.Components.Create(component) + if err != nil { + return nil, err + } + // Create an struct to hold the SBOM information for this component. + componentSBOM := &layout.ComponentSBOM{ + Files: []string{}, + Component: componentPaths, + } + + appendSBOMFiles := func(path string) { + if utils.IsDir(path) { + files, _ := utils.RecursiveFileList(path, nil, false) + componentSBOM.Files = append(componentSBOM.Files, files...) + } else { + componentSBOM.Files = append(componentSBOM.Files, path) + } + } + + for filesIdx, file := range component.Files { + path := filepath.Join(componentPaths.Files, strconv.Itoa(filesIdx), filepath.Base(file.Target)) + appendSBOMFiles(path) + } + + for dataIdx, data := range component.DataInjections { + path := filepath.Join(componentPaths.DataInjections, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) + + appendSBOMFiles(path) + } + + return componentSBOM, nil +} diff --git a/src/pkg/packager/creator/skeleton.go b/src/pkg/packager/creator/skeleton.go new file mode 100644 index 0000000000..d23081f1a7 --- /dev/null +++ b/src/pkg/packager/creator/skeleton.go @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package creator contains functions for creating Zarf packages. +package creator + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/extensions/bigbang" + "github.com/defenseunicorns/zarf/src/internal/packager/helm" + "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/pkg/zoci" + "github.com/defenseunicorns/zarf/src/types" + "github.com/mholt/archiver/v3" +) + +var ( + // verify that SkeletonCreator implements Creator + _ Creator = (*SkeletonCreator)(nil) +) + +// SkeletonCreator provides methods for creating skeleton Zarf packages. +type SkeletonCreator struct { + createOpts types.ZarfCreateOptions + publishOpts types.ZarfPublishOptions +} + +// NewSkeletonCreator returns a new SkeletonCreator. +func NewSkeletonCreator(createOpts types.ZarfCreateOptions, publishOpts types.ZarfPublishOptions) *SkeletonCreator { + return &SkeletonCreator{createOpts, publishOpts} +} + +// LoadPackageDefinition loads and configure a zarf.yaml file when creating and publishing a skeleton package. +func (sc *SkeletonCreator) LoadPackageDefinition(dst *layout.PackagePaths) (pkg types.ZarfPackage, warnings []string, err error) { + pkg, warnings, err = dst.ReadZarfYAML(layout.ZarfYAML) + if err != nil { + return types.ZarfPackage{}, nil, err + } + + pkg.Metadata.Architecture = zoci.SkeletonArch + + // Compose components into a single zarf.yaml file + pkg, composeWarnings, err := ComposeComponents(pkg, sc.createOpts.Flavor) + if err != nil { + return types.ZarfPackage{}, nil, err + } + + warnings = append(warnings, composeWarnings...) + + pkg.Components, err = sc.processExtensions(pkg.Components, dst) + if err != nil { + return types.ZarfPackage{}, nil, err + } + + for _, warning := range warnings { + message.Warn(warning) + } + + return pkg, warnings, nil +} + +// Assemble updates all components of the loaded Zarf package with necessary modifications for package assembly. +// +// It processes each component to ensure correct structure and resource locations. +func (sc *SkeletonCreator) Assemble(dst *layout.PackagePaths, components []types.ZarfComponent, _ string) error { + for _, component := range components { + c, err := sc.addComponent(component, dst) + if err != nil { + return err + } + components = append(components, *c) + } + + return nil +} + +// Output does the following: +// +// - archives components +// +// - generates checksums for all package files +// +// - writes the loaded zarf.yaml to disk +// +// - signs the package +func (sc *SkeletonCreator) Output(dst *layout.PackagePaths, pkg *types.ZarfPackage) (err error) { + for _, component := range pkg.Components { + if err := dst.Components.Archive(component, false); err != nil { + return err + } + } + + // Calculate all the checksums + pkg.Metadata.AggregateChecksum, err = dst.GenerateChecksums() + if err != nil { + return fmt.Errorf("unable to generate checksums for the package: %w", err) + } + + if err := recordPackageMetadata(pkg, sc.createOpts); err != nil { + return err + } + + if err := utils.WriteYaml(dst.ZarfYAML, pkg, helpers.ReadUser); err != nil { + return fmt.Errorf("unable to write zarf.yaml: %w", err) + } + + // Sign the package if a key has been provided + if sc.publishOpts.SigningKeyPath != "" { + if err := dst.SignPackage(sc.publishOpts.SigningKeyPath, sc.publishOpts.SigningKeyPassword); err != nil { + return err + } + } + + return nil +} + +func (sc *SkeletonCreator) processExtensions(components []types.ZarfComponent, layout *layout.PackagePaths) (processedComponents []types.ZarfComponent, err error) { + // Create component paths and process extensions for each component. + for _, c := range components { + componentPaths, err := layout.Components.Create(c) + if err != nil { + return nil, err + } + + // Big Bang + if c.Extensions.BigBang != nil { + if c, err = bigbang.Skeletonize(componentPaths, c); err != nil { + return nil, fmt.Errorf("unable to process bigbang extension: %w", err) + } + } + + processedComponents = append(processedComponents, c) + } + + return processedComponents, nil +} + +func (sc *SkeletonCreator) addComponent(component types.ZarfComponent, dst *layout.PackagePaths) (updatedComponent *types.ZarfComponent, err error) { + message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) + + updatedComponent = &component + + componentPaths, err := dst.Components.Create(component) + if err != nil { + return nil, err + } + + if component.DeprecatedCosignKeyPath != "" { + dst := filepath.Join(componentPaths.Base, "cosign.pub") + err := utils.CreatePathAndCopy(component.DeprecatedCosignKeyPath, dst) + if err != nil { + return nil, err + } + updatedComponent.DeprecatedCosignKeyPath = "cosign.pub" + } + + // TODO: (@WSTARR) Shim the skeleton component's create action dirs to be empty. This prevents actions from failing by cd'ing into directories that will be flattened. + updatedComponent.Actions.OnCreate.Defaults.Dir = "" + + resetActions := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { + for idx := range actions { + actions[idx].Dir = nil + } + return actions + } + + updatedComponent.Actions.OnCreate.Before = resetActions(component.Actions.OnCreate.Before) + updatedComponent.Actions.OnCreate.After = resetActions(component.Actions.OnCreate.After) + updatedComponent.Actions.OnCreate.OnSuccess = resetActions(component.Actions.OnCreate.OnSuccess) + updatedComponent.Actions.OnCreate.OnFailure = resetActions(component.Actions.OnCreate.OnFailure) + + // If any helm charts are defined, process them. + for chartIdx, chart := range component.Charts { + + if chart.LocalPath != "" { + rel := filepath.Join(layout.ChartsDir, fmt.Sprintf("%s-%d", chart.Name, chartIdx)) + dst := filepath.Join(componentPaths.Base, rel) + + err := utils.CreatePathAndCopy(chart.LocalPath, dst) + if err != nil { + return nil, err + } + + updatedComponent.Charts[chartIdx].LocalPath = rel + } + + for valuesIdx, path := range chart.ValuesFiles { + if helpers.IsURL(path) { + continue + } + + rel := fmt.Sprintf("%s-%d", helm.StandardName(layout.ValuesDir, chart), valuesIdx) + updatedComponent.Charts[chartIdx].ValuesFiles[valuesIdx] = rel + + if err := utils.CreatePathAndCopy(path, filepath.Join(componentPaths.Base, rel)); err != nil { + return nil, fmt.Errorf("unable to copy chart values file %s: %w", path, err) + } + } + } + + for filesIdx, file := range component.Files { + message.Debugf("Loading %#v", file) + + if helpers.IsURL(file.Source) { + continue + } + + rel := filepath.Join(layout.FilesDir, strconv.Itoa(filesIdx), filepath.Base(file.Target)) + dst := filepath.Join(componentPaths.Base, rel) + destinationDir := filepath.Dir(dst) + + if file.ExtractPath != "" { + if err := archiver.Extract(file.Source, file.ExtractPath, destinationDir); err != nil { + return nil, fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, file.Source, err.Error()) + } + + // Make sure dst reflects the actual file or directory. + updatedExtractedFileOrDir := filepath.Join(destinationDir, file.ExtractPath) + if updatedExtractedFileOrDir != dst { + if err := os.Rename(updatedExtractedFileOrDir, dst); err != nil { + return nil, fmt.Errorf(lang.ErrWritingFile, dst, err) + } + } + } else { + if err := utils.CreatePathAndCopy(file.Source, dst); err != nil { + return nil, fmt.Errorf("unable to copy file %s: %w", file.Source, err) + } + } + + // Change the source to the new relative source directory (any remote files will have been skipped above) + updatedComponent.Files[filesIdx].Source = rel + + // Remove the extractPath from a skeleton since it will already extract it + updatedComponent.Files[filesIdx].ExtractPath = "" + + // Abort packaging on invalid shasum (if one is specified). + if file.Shasum != "" { + if err := utils.SHAsMatch(dst, file.Shasum); err != nil { + return nil, err + } + } + + if file.Executable || utils.IsDir(dst) { + _ = os.Chmod(dst, helpers.ReadWriteExecuteUser) + } else { + _ = os.Chmod(dst, helpers.ReadWriteUser) + } + } + + if len(component.DataInjections) > 0 { + spinner := message.NewProgressSpinner("Loading data injections") + defer spinner.Stop() + + for dataIdx, data := range component.DataInjections { + spinner.Updatef("Copying data injection %s for %s", data.Target.Path, data.Target.Selector) + + rel := filepath.Join(layout.DataInjectionsDir, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) + dst := filepath.Join(componentPaths.Base, rel) + + if err := utils.CreatePathAndCopy(data.Source, dst); err != nil { + return nil, fmt.Errorf("unable to copy data injection %s: %s", data.Source, err.Error()) + } + + updatedComponent.DataInjections[dataIdx].Source = rel + } + + spinner.Success() + } + + if len(component.Manifests) > 0 { + // Get the proper count of total manifests to add. + manifestCount := 0 + + for _, manifest := range component.Manifests { + manifestCount += len(manifest.Files) + manifestCount += len(manifest.Kustomizations) + } + + spinner := message.NewProgressSpinner("Loading %d K8s manifests", manifestCount) + defer spinner.Stop() + + // Iterate over all manifests. + for manifestIdx, manifest := range component.Manifests { + for fileIdx, path := range manifest.Files { + rel := filepath.Join(layout.ManifestsDir, fmt.Sprintf("%s-%d.yaml", manifest.Name, fileIdx)) + dst := filepath.Join(componentPaths.Base, rel) + + // Copy manifests without any processing. + spinner.Updatef("Copying manifest %s", path) + + if err := utils.CreatePathAndCopy(path, dst); err != nil { + return nil, fmt.Errorf("unable to copy manifest %s: %w", path, err) + } + + updatedComponent.Manifests[manifestIdx].Files[fileIdx] = rel + } + + for kustomizeIdx, path := range manifest.Kustomizations { + // Generate manifests from kustomizations and place in the package. + spinner.Updatef("Building kustomization for %s", path) + + kname := fmt.Sprintf("kustomization-%s-%d.yaml", manifest.Name, kustomizeIdx) + rel := filepath.Join(layout.ManifestsDir, kname) + dst := filepath.Join(componentPaths.Base, rel) + + if err := kustomize.Build(path, dst, manifest.KustomizeAllowAnyDirectory); err != nil { + return nil, fmt.Errorf("unable to build kustomization %s: %w", path, err) + } + } + + // remove kustomizations + updatedComponent.Manifests[manifestIdx].Kustomizations = nil + } + + spinner.Success() + } + + return updatedComponent, nil +} diff --git a/src/pkg/packager/creator/template.go b/src/pkg/packager/creator/template.go new file mode 100644 index 0000000000..53b22ba206 --- /dev/null +++ b/src/pkg/packager/creator/template.go @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package creator contains functions for creating Zarf packages. +package creator + +import ( + "fmt" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/pkg/interactive" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/types" +) + +// FillActiveTemplate merges user-specified variables into the configuration templates of a zarf.yaml. +func FillActiveTemplate(pkg types.ZarfPackage, setVariables map[string]string) (types.ZarfPackage, []string, error) { + templateMap := map[string]string{} + warnings := []string{} + + promptAndSetTemplate := func(templatePrefix string, deprecated bool) error { + yamlTemplates, err := utils.FindYamlTemplates(&pkg, templatePrefix, "###") + if err != nil { + return err + } + + for key := range yamlTemplates { + if deprecated { + warnings = append(warnings, fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key)) + } + + _, present := setVariables[key] + if !present && !config.CommonOptions.Confirm { + setVal, err := interactive.PromptVariable(types.ZarfPackageVariable{ + Name: key, + }) + if err != nil { + return err + } + setVariables[key] = setVal + } else if !present { + return fmt.Errorf("template %q must be '--set' when using the '--confirm' flag", key) + } + } + + for key, value := range setVariables { + templateMap[fmt.Sprintf("%s%s###", templatePrefix, key)] = value + } + + return nil + } + + // update the component templates on the package + if err := ReloadComponentTemplatesInPackage(&pkg); err != nil { + return types.ZarfPackage{}, nil, err + } + + if err := promptAndSetTemplate(types.ZarfPackageTemplatePrefix, false); err != nil { + return types.ZarfPackage{}, nil, err + } + // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility + if err := promptAndSetTemplate(types.ZarfPackageVariablePrefix, true); err != nil { + return types.ZarfPackage{}, nil, err + } + + // Add special variable for the current package architecture + templateMap[types.ZarfPackageArch] = pkg.Metadata.Architecture + + if err := utils.ReloadYamlTemplate(&pkg, templateMap); err != nil { + return types.ZarfPackage{}, nil, err + } + + return pkg, warnings, nil +} + +// ReloadComponentTemplate appends ###ZARF_COMPONENT_NAME### for the component, assigns value, and reloads +// Any instance of ###ZARF_COMPONENT_NAME### within a component will be replaced with that components name +func ReloadComponentTemplate(component *types.ZarfComponent) error { + mappings := map[string]string{} + mappings[types.ZarfComponentName] = component.Name + err := utils.ReloadYamlTemplate(component, mappings) + if err != nil { + return err + } + return nil +} + +// ReloadComponentTemplatesInPackage appends ###ZARF_COMPONENT_NAME### for each component, assigns value, and reloads +func ReloadComponentTemplatesInPackage(zarfPackage *types.ZarfPackage) error { + // iterate through components to and find all ###ZARF_COMPONENT_NAME, assign to component Name and value + for i := range zarfPackage.Components { + if err := ReloadComponentTemplate(&zarfPackage.Components[i]); err != nil { + return err + } + } + + return nil +} diff --git a/src/pkg/packager/creator/utils.go b/src/pkg/packager/creator/utils.go new file mode 100644 index 0000000000..43ab469565 --- /dev/null +++ b/src/pkg/packager/creator/utils.go @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package creator contains functions for creating Zarf packages. +package creator + +import ( + "os" + "runtime" + "time" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" + "github.com/defenseunicorns/zarf/src/types" +) + +// recordPackageMetadata records various package metadata during package create. +func recordPackageMetadata(pkg *types.ZarfPackage, createOpts types.ZarfCreateOptions) error { + now := time.Now() + // Just use $USER env variable to avoid CGO issue. + // https://groups.google.com/g/golang-dev/c/ZFDDX3ZiJ84. + // Record the name of the user creating the package. + if runtime.GOOS == "windows" { + pkg.Build.User = os.Getenv("USERNAME") + } else { + pkg.Build.User = os.Getenv("USER") + } + + // Record the hostname of the package creation terminal. + // The error here is ignored because the hostname is not critical to the package creation. + hostname, _ := os.Hostname() + pkg.Build.Terminal = hostname + + if pkg.IsInitConfig() { + pkg.Metadata.Version = config.CLIVersion + } + + pkg.Build.Architecture = pkg.Metadata.Architecture + + // Record the Zarf Version the CLI was built with. + pkg.Build.Version = config.CLIVersion + + // Record the time of package creation. + pkg.Build.Timestamp = now.Format(time.RFC1123Z) + + // Record the migrations that will be ran on the package. + pkg.Build.Migrations = []string{ + deprecated.ScriptsToActionsMigrated, + deprecated.PluralizeSetVariable, + } + + // Record the flavor of Zarf used to build this package (if any). + pkg.Build.Flavor = createOpts.Flavor + + pkg.Build.RegistryOverrides = createOpts.RegistryOverrides + + // Record the latest version of Zarf without breaking changes to the package structure. + pkg.Build.LastNonBreakingVersion = deprecated.LastNonBreakingVersion + + return nil +} diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 8974c71612..7a1cece304 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -23,6 +23,8 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/k8s" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/actions" + "github.com/defenseunicorns/zarf/src/pkg/packager/variables" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -44,7 +46,8 @@ func (p *Packager) Deploy() (err error) { return fmt.Errorf("unable to load the package: %w", err) } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + p.cfg.Pkg, p.warnings, err = p.layout.ReadZarfYAML(p.layout.ZarfYAML) + if err != nil { return err } @@ -52,17 +55,20 @@ func (p *Packager) Deploy() (err error) { return err } - if err := p.stageSBOMViewFiles(); err != nil { + sbomWarnings, err := p.layout.SBOMs.StageSBOMViewFiles() + if err != nil { return err } + p.warnings = append(p.warnings, sbomWarnings...) + // Confirm the overall package deployment if !p.confirmAction(config.ZarfDeployStage) { return fmt.Errorf("deployment cancelled") } // Set variables and prompt if --confirm is not set - if err := p.setVariableMapInConfig(); err != nil { + if err := variables.SetVariableMapInConfig(p.cfg); err != nil { return err } @@ -117,7 +123,7 @@ func (p *Packager) deployComponents() (deployedComponents []types.DeployedCompon // If this component requires a cluster, connect to one if requiresCluster(component) { timeout := cluster.DefaultTimeout - if p.isInitConfig() { + if p.cfg.Pkg.IsInitConfig() { timeout = 5 * time.Minute } @@ -147,7 +153,7 @@ func (p *Packager) deployComponents() (deployedComponents []types.DeployedCompon // Deploy the component var charts []types.InstalledChart var deployErr error - if p.isInitConfig() { + if p.cfg.Pkg.IsInitConfig() { charts, deployErr = p.deployInitComponent(component) } else { charts, deployErr = p.deployComponent(component, false /* keep img checksum */, false /* always push images */) @@ -156,7 +162,7 @@ func (p *Packager) deployComponents() (deployedComponents []types.DeployedCompon onDeploy := component.Actions.OnDeploy onFailure := func() { - if err := p.runActions(onDeploy.Defaults, onDeploy.OnFailure, p.valueTemplate); err != nil { + if err := actions.Run(p.cfg, onDeploy.Defaults, onDeploy.OnFailure, p.valueTemplate); err != nil { message.Debugf("unable to run component failure action: %s", err.Error()) } } @@ -184,7 +190,7 @@ func (p *Packager) deployComponents() (deployedComponents []types.DeployedCompon } } - if err := p.runActions(onDeploy.Defaults, onDeploy.OnSuccess, p.valueTemplate); err != nil { + if err := actions.Run(p.cfg, onDeploy.Defaults, onDeploy.OnSuccess, p.valueTemplate); err != nil { onFailure() return deployedComponents, fmt.Errorf("unable to run component success action: %w", err) } @@ -277,7 +283,7 @@ func (p *Packager) deployComponent(component types.ZarfComponent, noImgChecksum } } - if err = p.runActions(onDeploy.Defaults, onDeploy.Before, p.valueTemplate); err != nil { + if err = actions.Run(p.cfg, onDeploy.Defaults, onDeploy.Before, p.valueTemplate); err != nil { return charts, fmt.Errorf("unable to run component before action: %w", err) } @@ -315,7 +321,7 @@ func (p *Packager) deployComponent(component types.ZarfComponent, noImgChecksum } } - if err = p.runActions(onDeploy.Defaults, onDeploy.After, p.valueTemplate); err != nil { + if err = actions.Run(p.cfg, onDeploy.Defaults, onDeploy.After, p.valueTemplate); err != nil { return charts, fmt.Errorf("unable to run component after action: %w", err) } @@ -465,7 +471,7 @@ func (p *Packager) pushImagesToRegistry(componentImages []string, noImgChecksum NoChecksum: noImgChecksum, RegInfo: p.cfg.State.RegistryInfo, Insecure: config.CommonOptions.Insecure, - Architectures: []string{p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture}, + Architectures: []string{p.cfg.Pkg.Build.Architecture}, } return helpers.Retry(func() error { @@ -624,7 +630,7 @@ func (p *Packager) installChartAndManifests(componentPaths *layout.ComponentPath func (p *Packager) printTablesForDeployment(componentsToDeploy []types.DeployedComponent) { // If not init config, print the application connection table - if !p.isInitConfig() { + if !p.cfg.Pkg.IsInitConfig() { message.PrintConnectStringTable(p.connectStrings) } else { if p.cluster != nil { diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go index 675ee540ee..2b978ec479 100644 --- a/src/pkg/packager/dev.go +++ b/src/pkg/packager/dev.go @@ -11,6 +11,8 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/creator" + "github.com/defenseunicorns/zarf/src/pkg/packager/variables" "github.com/defenseunicorns/zarf/src/types" ) @@ -24,11 +26,14 @@ func (p *Packager) DevDeploy() error { return err } - if err := p.cdToBaseDir(p.cfg.CreateOpts.BaseDir, cwd); err != nil { - return err + if err := os.Chdir(p.cfg.CreateOpts.BaseDir); err != nil { + return fmt.Errorf("unable to access directory %q: %w", p.cfg.CreateOpts.BaseDir, err) } - if err := p.load(); err != nil { + pc := creator.NewPackageCreator(p.cfg.CreateOpts, p.cfg, cwd) + + p.cfg.Pkg, p.warnings, err = pc.LoadPackageDefinition(p.layout) + if err != nil { return err } @@ -54,14 +59,14 @@ func (p *Packager) DevDeploy() error { } } - if err := p.assemble(); err != nil { + if err := pc.Assemble(p.layout, p.cfg.Pkg.Components, p.cfg.Pkg.Metadata.Architecture); err != nil { return err } message.HeaderInfof("📦 PACKAGE DEPLOY %s", p.cfg.Pkg.Metadata.Name) // Set variables and prompt if --confirm is not set - if err := p.setVariableMapInConfig(); err != nil { + if err := variables.SetVariableMapInConfig(p.cfg); err != nil { return err } diff --git a/src/pkg/packager/extensions.go b/src/pkg/packager/extensions.go deleted file mode 100644 index b5cc0ab17d..0000000000 --- a/src/pkg/packager/extensions.go +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager - -import ( - "fmt" - - "github.com/defenseunicorns/zarf/src/extensions/bigbang" - "github.com/defenseunicorns/zarf/src/types" -) - -// Check for any extensions in use and runs the appropriate functions. -func (p *Packager) processExtensions() (err error) { - components := []types.ZarfComponent{} - - // Create component paths and process extensions for each component. - for _, c := range p.cfg.Pkg.Components { - componentPaths, err := p.layout.Components.Create(c) - if err != nil { - return err - } - - // Big Bang - if c.Extensions.BigBang != nil { - if c, err = bigbang.Run(p.cfg.Pkg.Metadata.YOLO, componentPaths, c); err != nil { - return fmt.Errorf("unable to process bigbang extension: %w", err) - } - } - - components = append(components, c) - } - - // Update the parent package config with the expanded sub components. - // This is important when the deploy package is created. - p.cfg.Pkg.Components = components - - return nil -} - -// Check for any extensions in use and skeletonize their local files. -func (p *Packager) skeletonizeExtensions() (err error) { - components := []types.ZarfComponent{} - - // Create component paths and process extensions for each component. - for _, c := range p.cfg.Pkg.Components { - componentPaths, err := p.layout.Components.Create(c) - if err != nil { - return err - } - - // Big Bang - if c.Extensions.BigBang != nil { - if c, err = bigbang.Skeletonize(componentPaths, c); err != nil { - return fmt.Errorf("unable to process bigbang extension: %w", err) - } - } - - components = append(components, c) - } - - // Update the parent package config with the expanded sub components. - // This is important when the deploy package is created. - p.cfg.Pkg.Components = components - - return nil -} diff --git a/src/pkg/packager/generate.go b/src/pkg/packager/generate.go index a82fcdeca3..286807d186 100644 --- a/src/pkg/packager/generate.go +++ b/src/pkg/packager/generate.go @@ -62,7 +62,6 @@ func (p *Packager) Generate() (err error) { generatedComponent, }, } - p.arch = config.GetArch() images, err := p.findImages() if err != nil { diff --git a/src/pkg/packager/inspect.go b/src/pkg/packager/inspect.go index 993c9ffeb0..bfc9c1aac8 100644 --- a/src/pkg/packager/inspect.go +++ b/src/pkg/packager/inspect.go @@ -17,7 +17,8 @@ func (p *Packager) Inspect() (err error) { return err } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + p.cfg.Pkg, p.warnings, err = p.layout.ReadZarfYAML(p.layout.ZarfYAML) + if err != nil { return err } @@ -26,7 +27,7 @@ func (p *Packager) Inspect() (err error) { sbomDir := p.layout.SBOMs.Path if p.cfg.InspectOpts.SBOMOutputDir != "" { - out, err := sbom.OutputSBOMFiles(sbomDir, p.cfg.InspectOpts.SBOMOutputDir, p.cfg.Pkg.Metadata.Name) + out, err := p.layout.SBOMs.OutputSBOMFiles(p.cfg.InspectOpts.SBOMOutputDir, p.cfg.Pkg.Metadata.Name) if err != nil { return err } diff --git a/src/pkg/packager/interactive.go b/src/pkg/packager/interactive.go index f338b00b90..cdd46bf069 100644 --- a/src/pkg/packager/interactive.go +++ b/src/pkg/packager/interactive.go @@ -11,7 +11,6 @@ import ( "github.com/AlecAivazis/survey/v2" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/internal/packager/sbom" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -26,7 +25,7 @@ func (p *Packager) confirmAction(stage string) (confirm bool) { // Print any potential breaking changes (if this is a Deploy confirm) between this CLI version and the deployed init package if stage == config.ZarfDeployStage { - if sbom.IsSBOMAble(p.cfg.Pkg) { + if p.cfg.Pkg.IsSBOMAble() { // Print the location that the user can view the package SBOMs from message.HorizontalRule() message.Title("Software Bill of Materials", "an inventory of all software contained in this package") diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index bc8b5df336..8ac60076d1 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -15,8 +15,8 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/packager/composer" + "github.com/defenseunicorns/zarf/src/pkg/packager/creator" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -104,8 +104,7 @@ func lintComponents(validator *Validator, createOpts *types.ZarfCreateOptions) { } func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts *types.ZarfCreateOptions) { - - err := packager.ReloadComponentTemplate(&node.ZarfComponent) + err := creator.ReloadComponentTemplate(&node.ZarfComponent) if err != nil { validator.addWarning(validatorMessage{ description: err.Error(), diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 689ddcfbaa..6dabec8a7f 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -21,14 +21,19 @@ func (p *Packager) Mirror() (err error) { if err = p.source.LoadPackage(p.layout, true); err != nil { return fmt.Errorf("unable to load the package: %w", err) } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + + p.cfg.Pkg, p.warnings, err = p.layout.ReadZarfYAML(p.layout.ZarfYAML) + if err != nil { return err } - if err := p.stageSBOMViewFiles(); err != nil { + sbomWarnings, err := p.layout.SBOMs.StageSBOMViewFiles() + if err != nil { return err } + p.warnings = append(p.warnings, sbomWarnings...) + // Confirm the overall package mirror if !p.confirmAction(config.ZarfMirrorStage) { return fmt.Errorf("mirror cancelled") diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index 76d057e568..c4cfc95d3c 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -19,8 +19,9 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/helm" "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" "github.com/defenseunicorns/zarf/src/internal/packager/template" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/creator" + "github.com/defenseunicorns/zarf/src/pkg/packager/variables" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" @@ -53,8 +54,15 @@ func (p *Packager) FindImages() (imgMap map[string][]string, err error) { } message.Note(fmt.Sprintf("Using build directory %s", p.cfg.CreateOpts.BaseDir)) - if err = p.readZarfYAML(layout.ZarfYAML); err != nil { - return nil, fmt.Errorf("unable to read the zarf.yaml file: %w", err) + c := creator.NewPackageCreator(p.cfg.CreateOpts, p.cfg, cwd) + + p.cfg.Pkg, p.warnings, err = c.LoadPackageDefinition(p.layout) + if err != nil { + return nil, err + } + + for _, warning := range p.warnings { + message.Warn(warning) } return p.findImages() @@ -70,19 +78,6 @@ func (p *Packager) findImages() (imgMap map[string][]string, err error) { erroredCosignLookups := []string{} whyResources := []string{} - if err := p.composeComponents(); err != nil { - return nil, err - } - - for _, warning := range p.warnings { - message.Warn(warning) - } - - // After components are composed, template the active package - if err := p.fillActiveTemplate(); err != nil { - return nil, fmt.Errorf("unable to fill values in template: %w", err) - } - for _, component := range p.cfg.Pkg.Components { if len(component.Repos) > 0 && repoHelmChartPath == "" { message.Note("This Zarf package contains git repositories, " + @@ -94,7 +89,7 @@ func (p *Packager) findImages() (imgMap map[string][]string, err error) { componentDefinition := "\ncomponents:\n" - if err := p.setVariableMapInConfig(); err != nil { + if err := variables.SetVariableMapInConfig(p.cfg); err != nil { return nil, err } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 05ffe3bb4e..c438b83456 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -13,6 +13,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/packager/creator" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -46,26 +47,40 @@ func (p *Packager) Publish() (err error) { } if p.cfg.CreateOpts.IsSkeleton { - cwd, err := os.Getwd() - if err != nil { - return err + if err := os.Chdir(p.cfg.CreateOpts.BaseDir); err != nil { + return fmt.Errorf("unable to access directory %q: %w", p.cfg.CreateOpts.BaseDir, err) } - if err := p.cdToBaseDir(p.cfg.CreateOpts.BaseDir, cwd); err != nil { + + sc := creator.NewSkeletonCreator(p.cfg.CreateOpts, p.cfg.PublishOpts) + + p.cfg.Pkg, p.warnings, err = sc.LoadPackageDefinition(p.layout) + if err != nil { return err } - if err := p.load(); err != nil { + + if err := sc.Assemble(p.layout, p.cfg.Pkg.Components, ""); err != nil { return err } - if err := p.assembleSkeleton(); err != nil { + + if err := sc.Output(p.layout, &p.cfg.Pkg); err != nil { return err } } else { - if err = p.source.LoadPackage(p.layout, false); err != nil { + if err := p.source.LoadPackage(p.layout, false); err != nil { return fmt.Errorf("unable to load the package: %w", err) } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + + p.cfg.Pkg, p.warnings, err = p.layout.ReadZarfYAML(p.layout.ZarfYAML) + if err != nil { return err } + + // Sign the package if a key has been provided + if p.cfg.PublishOpts.SigningKeyPath != "" { + if err := p.layout.SignPackage(p.cfg.PublishOpts.SigningKeyPath, p.cfg.PublishOpts.SigningKeyPassword); err != nil { + return err + } + } } // Get a reference to the registry for this package @@ -77,20 +92,13 @@ func (p *Packager) Publish() (err error) { if p.cfg.CreateOpts.IsSkeleton { platform = zoci.PlatformForSkeleton() } else { - platform = oci.PlatformForArch(p.arch) + platform = oci.PlatformForArch(p.cfg.Pkg.Build.Architecture) } remote, err := zoci.NewRemote(ref, platform) if err != nil { return err } - // Sign the package if a key has been provided - if p.cfg.PublishOpts.SigningKeyPath != "" { - if err := p.signPackage(p.cfg.PublishOpts.SigningKeyPath, p.cfg.PublishOpts.SigningKeyPassword); err != nil { - return err - } - } - message.HeaderInfof("📦 PACKAGE PUBLISH %s:%s", p.cfg.Pkg.Metadata.Name, ref) // Publish the package/skeleton to the registry diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index 2aaee0c3af..2f7bf05e01 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -15,6 +15,7 @@ import ( "github.com/defenseunicorns/zarf/src/internal/packager/helm" "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager/actions" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" @@ -38,9 +39,12 @@ func (p *Packager) Remove() (err error) { if err = p.source.LoadPackageMetadata(p.layout, false, false); err != nil { return err } - if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { + + p.cfg.Pkg, p.warnings, err = p.layout.ReadZarfYAML(p.layout.ZarfYAML) + if err != nil { return err } + p.filterComponents() packageName = p.cfg.Pkg.Metadata.Name @@ -124,12 +128,12 @@ func (p *Packager) removeComponent(deployedPackage *types.DeployedPackage, deplo onRemove := c.Actions.OnRemove onFailure := func() { - if err := p.runActions(onRemove.Defaults, onRemove.OnFailure, nil); err != nil { + if err := actions.Run(p.cfg, onRemove.Defaults, onRemove.OnFailure, nil); err != nil { message.Debugf("Unable to run the failure action: %s", err) } } - if err := p.runActions(onRemove.Defaults, onRemove.Before, nil); err != nil { + if err := actions.Run(p.cfg, onRemove.Defaults, onRemove.Before, nil); err != nil { onFailure() return nil, fmt.Errorf("unable to run the before action for component (%s): %w", c.Name, err) } @@ -158,12 +162,12 @@ func (p *Packager) removeComponent(deployedPackage *types.DeployedPackage, deplo p.updatePackageSecret(*deployedPackage) } - if err := p.runActions(onRemove.Defaults, onRemove.After, nil); err != nil { + if err := actions.Run(p.cfg, onRemove.Defaults, onRemove.After, nil); err != nil { onFailure() return deployedPackage, fmt.Errorf("unable to run the after action: %w", err) } - if err := p.runActions(onRemove.Defaults, onRemove.OnSuccess, nil); err != nil { + if err := actions.Run(p.cfg, onRemove.Defaults, onRemove.OnSuccess, nil); err != nil { onFailure() return deployedPackage, fmt.Errorf("unable to run the success action: %w", err) } diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index a81e5c16f1..b9a0857c9d 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -201,18 +201,11 @@ func (s *OCISource) Collect(dir string) (string, error) { spinner.Success() // TODO (@Noxsios) remove the suffix check at v1.0.0 - isSkeleton := pkg.Build.Architecture == "skeleton" || strings.HasSuffix(s.Repo().Reference.Reference, zoci.SkeletonArch) - name := NameFromMetadata(&pkg, isSkeleton) + isSkeleton := pkg.Build.Architecture == zoci.SkeletonArch || strings.HasSuffix(s.Repo().Reference.Reference, zoci.SkeletonArch) + name := fmt.Sprintf("%s%s", NameFromMetadata(&pkg, isSkeleton), PkgSuffix(pkg.Metadata.Uncompressed)) dstTarball := filepath.Join(dir, name) - // honor uncompressed flag - if pkg.Metadata.Uncompressed { - dstTarball = dstTarball + ".tar" - } else { - dstTarball = dstTarball + ".tar.zst" - } - allTheLayers, err := filepath.Glob(filepath.Join(tmp, "*")) if err != nil { return "", err diff --git a/src/pkg/packager/sources/utils.go b/src/pkg/packager/sources/utils.go index 383cc262cd..1347ee4127 100644 --- a/src/pkg/packager/sources/utils.go +++ b/src/pkg/packager/sources/utils.go @@ -14,6 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/defenseunicorns/zarf/src/types" goyaml "github.com/goccy/go-yaml" "github.com/mholt/archiver/v3" @@ -112,7 +113,7 @@ func NameFromMetadata(pkg *types.ZarfPackage, isSkeleton bool) string { arch := config.GetArch(pkg.Metadata.Architecture, pkg.Build.Architecture) if isSkeleton { - arch = "skeleton" + arch = zoci.SkeletonArch } switch pkg.Kind { @@ -125,10 +126,26 @@ func NameFromMetadata(pkg *types.ZarfPackage, isSkeleton bool) string { } if pkg.Build.Differential { - name = fmt.Sprintf("%s-differential-%s", name, pkg.Metadata.Version) + name = fmt.Sprintf("%s-%s-differential-%s", name, pkg.Build.DifferentialPackageVersion, pkg.Metadata.Version) } else if pkg.Metadata.Version != "" { name = fmt.Sprintf("%s-%s", name, pkg.Metadata.Version) } return name } + +// GetInitPackageName returns the formatted name of the init package. +func GetInitPackageName() string { + // No package has been loaded yet so lookup GetArch() with no package info + arch := config.GetArch() + return fmt.Sprintf("zarf-init-%s-%s.tar.zst", arch, config.CLIVersion) +} + +// PkgSuffix returns a package suffix based on whether it is uncompressed or not. +func PkgSuffix(uncompressed bool) (suffix string) { + suffix = ".tar.zst" + if uncompressed { + suffix = ".tar" + } + return suffix +} diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go deleted file mode 100644 index 53f4e05df3..0000000000 --- a/src/pkg/packager/variables.go +++ /dev/null @@ -1,160 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager - -import ( - "fmt" - "regexp" - - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/interactive" - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/types" -) - -// ReloadComponentTemplate appends ###ZARF_COMPONENT_NAME### for the component, assigns value, and reloads -// Any instance of ###ZARF_COMPONENT_NAME### within a component will be replaced with that components name -func ReloadComponentTemplate(component *types.ZarfComponent) error { - mappings := map[string]string{} - mappings[types.ZarfComponentName] = component.Name - err := utils.ReloadYamlTemplate(component, mappings) - if err != nil { - return err - } - return nil -} - -// ReloadComponentTemplatesInPackage appends ###ZARF_COMPONENT_NAME### for each component, assigns value, and reloads -func ReloadComponentTemplatesInPackage(zarfPackage *types.ZarfPackage) error { - // iterate through components to and find all ###ZARF_COMPONENT_NAME, assign to component Name and value - for i := range zarfPackage.Components { - if err := ReloadComponentTemplate(&zarfPackage.Components[i]); err != nil { - return err - } - } - - return nil -} - -// fillActiveTemplate handles setting the active variables and reloading the base template. -func (p *Packager) fillActiveTemplate() error { - templateMap := map[string]string{} - - promptAndSetTemplate := func(templatePrefix string, deprecated bool) error { - yamlTemplates, err := utils.FindYamlTemplates(&p.cfg.Pkg, templatePrefix, "###") - if err != nil { - return err - } - - for key := range yamlTemplates { - if deprecated { - p.warnings = append(p.warnings, fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key)) - } - - _, present := p.cfg.CreateOpts.SetVariables[key] - if !present && !config.CommonOptions.Confirm { - setVal, err := interactive.PromptVariable(types.ZarfPackageVariable{ - Name: key, - }) - - if err == nil { - p.cfg.CreateOpts.SetVariables[key] = setVal - } else { - return err - } - } else if !present { - return fmt.Errorf("template '%s' must be '--set' when using the '--confirm' flag", key) - } - } - - for key, value := range p.cfg.CreateOpts.SetVariables { - templateMap[fmt.Sprintf("%s%s###", templatePrefix, key)] = value - } - - return nil - } - - // update the component templates on the package - err := ReloadComponentTemplatesInPackage(&p.cfg.Pkg) - if err != nil { - return err - } - - if err := promptAndSetTemplate(types.ZarfPackageTemplatePrefix, false); err != nil { - return err - } - // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility - if err := promptAndSetTemplate(types.ZarfPackageVariablePrefix, true); err != nil { - return err - } - - // Add special variable for the current package architecture - templateMap[types.ZarfPackageArch] = p.arch - - return utils.ReloadYamlTemplate(&p.cfg.Pkg, templateMap) -} - -// setVariableMapInConfig handles setting the active variables used to template component files. -func (p *Packager) setVariableMapInConfig() error { - for name, value := range p.cfg.PkgOpts.SetVariables { - p.setVariableInConfig(name, value, false, false, "") - } - - for _, variable := range p.cfg.Pkg.Variables { - _, present := p.cfg.SetVariableMap[variable.Name] - - // Variable is present, no need to continue checking - if present { - p.cfg.SetVariableMap[variable.Name].Sensitive = variable.Sensitive - p.cfg.SetVariableMap[variable.Name].AutoIndent = variable.AutoIndent - p.cfg.SetVariableMap[variable.Name].Type = variable.Type - if err := p.checkVariablePattern(variable.Name, variable.Pattern); err != nil { - return err - } - continue - } - - // First set default (may be overridden by prompt) - p.setVariableInConfig(variable.Name, variable.Default, variable.Sensitive, variable.AutoIndent, variable.Type) - - // Variable is set to prompt the user - if variable.Prompt && !config.CommonOptions.Confirm { - // Prompt the user for the variable - val, err := interactive.PromptVariable(variable) - - if err != nil { - return fmt.Errorf("unable to get value from prompt: %w", err) - } - - p.setVariableInConfig(variable.Name, val, variable.Sensitive, variable.AutoIndent, variable.Type) - } - - if err := p.checkVariablePattern(variable.Name, variable.Pattern); err != nil { - return err - } - } - - return nil -} - -func (p *Packager) setVariableInConfig(name, value string, sensitive bool, autoIndent bool, varType types.VariableType) { - p.cfg.SetVariableMap[name] = &types.ZarfSetVariable{ - Name: name, - Value: value, - Sensitive: sensitive, - AutoIndent: autoIndent, - Type: varType, - } -} - -// checkVariablePattern checks to see if a current variable is set to a value that matches its pattern -func (p *Packager) checkVariablePattern(name, pattern string) error { - if regexp.MustCompile(pattern).MatchString(p.cfg.SetVariableMap[name].Value) { - return nil - } - - return fmt.Errorf("provided value for variable %q does not match pattern \"%s\"", name, pattern) -} diff --git a/src/pkg/packager/variables/variables.go b/src/pkg/packager/variables/variables.go new file mode 100644 index 0000000000..a2ab07a2e3 --- /dev/null +++ b/src/pkg/packager/variables/variables.go @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package variables contains functions for working with variables within Zarf packages. +package variables + +import ( + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/interactive" + "github.com/defenseunicorns/zarf/src/types" +) + +// SetVariableMapInConfig handles setting the active variables used to template component files. +func SetVariableMapInConfig(cfg *types.PackagerConfig) error { + for name, value := range cfg.PkgOpts.SetVariables { + cfg.SetVariable(name, value, false, false, "") + } + + for _, variable := range cfg.Pkg.Variables { + _, present := cfg.SetVariableMap[variable.Name] + + // Variable is present, no need to continue checking + if present { + cfg.SetVariableMap[variable.Name].Sensitive = variable.Sensitive + cfg.SetVariableMap[variable.Name].AutoIndent = variable.AutoIndent + cfg.SetVariableMap[variable.Name].Type = variable.Type + if err := cfg.CheckVariablePattern(variable.Name, variable.Pattern); err != nil { + return err + } + continue + } + + // First set default (may be overridden by prompt) + cfg.SetVariable(variable.Name, variable.Default, variable.Sensitive, variable.AutoIndent, variable.Type) + + // Variable is set to prompt the user + if variable.Prompt && !config.CommonOptions.Confirm { + // Prompt the user for the variable + val, err := interactive.PromptVariable(variable) + + if err != nil { + return err + } + + cfg.SetVariable(variable.Name, val, variable.Sensitive, variable.AutoIndent, variable.Type) + } + + if err := cfg.CheckVariablePattern(variable.Name, variable.Pattern); err != nil { + return err + } + } + + return nil +} diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go deleted file mode 100644 index ba38a31bf1..0000000000 --- a/src/pkg/packager/yaml.go +++ /dev/null @@ -1,123 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package packager contains functions for interacting with, managing and deploying Zarf packages. -package packager - -import ( - "os" - "runtime" - "time" - - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/types" -) - -// readZarfYAML reads a Zarf YAML file. -func (p *Packager) readZarfYAML(path string) error { - var warnings []string - - if err := utils.ReadYaml(path, &p.cfg.Pkg); err != nil { - return err - } - - if p.layout.IsLegacyLayout() { - warning := "Detected deprecated package layout, migrating to new layout - support for this package will be dropped in v1.0.0" - p.warnings = append(p.warnings, warning) - } - - if len(p.cfg.Pkg.Build.Migrations) > 0 { - for idx, component := range p.cfg.Pkg.Components { - // Handle component configuration deprecations - p.cfg.Pkg.Components[idx], warnings = deprecated.MigrateComponent(p.cfg.Pkg.Build, component) - p.warnings = append(p.warnings, warnings...) - } - } - - p.arch = config.GetArch(p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture) - - return nil -} - -// filterComponents removes components not matching the current OS if filterByOS is set. -func (p *Packager) filterComponents() { - // Filter each component to only compatible platforms. - filteredComponents := []types.ZarfComponent{} - for _, component := range p.cfg.Pkg.Components { - // Ignore only filters that are empty - var validArch, validOS bool - - // Test for valid architecture - if component.Only.Cluster.Architecture == "" || component.Only.Cluster.Architecture == p.arch { - validArch = true - } else { - message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.Cluster.Architecture, p.arch) - } - - // Test for a valid OS - if component.Only.LocalOS == "" || component.Only.LocalOS == runtime.GOOS { - validOS = true - } else { - message.Debugf("Skipping component %s, %s is not compatible with %s", component.Name, component.Only.LocalOS, runtime.GOOS) - } - - // If both the OS and architecture are valid, add the component to the filtered list - if validArch && validOS { - filteredComponents = append(filteredComponents, component) - } - } - // Update the active package with the filtered components. - p.cfg.Pkg.Components = filteredComponents -} - -// writeYaml adds build information and writes the config to the temp directory. -func (p *Packager) writeYaml() error { - now := time.Now() - // Just use $USER env variable to avoid CGO issue. - // https://groups.google.com/g/golang-dev/c/ZFDDX3ZiJ84. - // Record the name of the user creating the package. - if runtime.GOOS == "windows" { - p.cfg.Pkg.Build.User = os.Getenv("USERNAME") - } else { - p.cfg.Pkg.Build.User = os.Getenv("USER") - } - hostname, hostErr := os.Hostname() - - // Normalize these for the package confirmation. - p.cfg.Pkg.Metadata.Architecture = p.arch - p.cfg.Pkg.Build.Architecture = p.arch - - if p.cfg.CreateOpts.IsSkeleton { - p.cfg.Pkg.Build.Architecture = "skeleton" - } - - // Record the time of package creation. - p.cfg.Pkg.Build.Timestamp = now.Format(time.RFC1123Z) - - // Record the Zarf Version the CLI was built with. - p.cfg.Pkg.Build.Version = config.CLIVersion - - if hostErr == nil { - // Record the hostname of the package creation terminal. - p.cfg.Pkg.Build.Terminal = hostname - } - - // Record the migrations that will be run on the package. - p.cfg.Pkg.Build.Migrations = []string{ - deprecated.ScriptsToActionsMigrated, - deprecated.PluralizeSetVariable, - } - - // Record the flavor of Zarf used to build this package (if any). - p.cfg.Pkg.Build.Flavor = p.cfg.CreateOpts.Flavor - - p.cfg.Pkg.Build.RegistryOverrides = p.cfg.CreateOpts.RegistryOverrides - - // Record the latest version of Zarf without breaking changes to the package structure. - p.cfg.Pkg.Build.LastNonBreakingVersion = deprecated.LastNonBreakingVersion - - return utils.WriteYaml(p.layout.ZarfYAML, p.cfg.Pkg, 0400) -} diff --git a/src/pkg/utils/helpers/io.go b/src/pkg/utils/helpers/io.go index e0b1879f86..996d643e52 100644 --- a/src/pkg/utils/helpers/io.go +++ b/src/pkg/utils/helpers/io.go @@ -5,11 +5,12 @@ package helpers const ( + // ReadUser is used for any internal file to be read only + ReadUser = 0400 // ReadWriteUser is used for any internal file not normally used by the end user or containing sensitive data ReadWriteUser = 0600 // ReadAllWriteUser is used for any non sensitive file intended to be consumed by the end user ReadAllWriteUser = 0644 - // ReadWriteExecuteUser is used for any directory or executable not normally used by the end user or containing sensitive data ReadWriteExecuteUser = 0700 // ReadExecuteAllWriteUser is used for any non sensitive directory or executable intended to be consumed by the end user diff --git a/src/pkg/utils/io.go b/src/pkg/utils/io.go index 8dad0da6ba..0e158cf1e0 100755 --- a/src/pkg/utils/io.go +++ b/src/pkg/utils/io.go @@ -47,9 +47,15 @@ func MakeTempDir(basePath string) (string, error) { return "", err } } + tmp, err := os.MkdirTemp(basePath, tmpPathPrefix) + if err != nil { + return "", err + } + message.Debug("Using temporary directory:", tmp) - return tmp, err + + return tmp, nil } // VerifyBinary returns true if binary is available. diff --git a/src/pkg/utils/yaml.go b/src/pkg/utils/yaml.go index fcf81c497c..f2c73d5f77 100644 --- a/src/pkg/utils/yaml.go +++ b/src/pkg/utils/yaml.go @@ -125,8 +125,8 @@ func AddRootHint(hints map[string]string, rootKey string, hintText string) map[s // ReadYaml reads a yaml file and unmarshals it into a given config. func ReadYaml(path string, destConfig any) error { message.Debugf("Reading YAML at %s", path) - file, err := os.ReadFile(path) + file, err := os.ReadFile(path) if err != nil { return err } @@ -148,7 +148,6 @@ func WriteYaml(path string, srcConfig any, perm fs.FileMode) error { // ReloadYamlTemplate marshals a given config, replaces strings and unmarshals it back. func ReloadYamlTemplate(config any, mappings map[string]string) error { text, err := goyaml.Marshal(config) - if err != nil { return err } @@ -172,9 +171,8 @@ func FindYamlTemplates(config any, prefix string, suffix string) (map[string]str mappings := map[string]string{} text, err := goyaml.Marshal(config) - if err != nil { - return mappings, err + return nil, err } // Find all strings that are between the given prefix and suffix diff --git a/src/test/e2e/03_deprecations_test.go b/src/test/e2e/03_deprecations_test.go index 808a5b25af..9d1c96b0c9 100644 --- a/src/test/e2e/03_deprecations_test.go +++ b/src/test/e2e/03_deprecations_test.go @@ -81,7 +81,7 @@ func TestDeprecatedSetAndPackageVariables(t *testing.T) { // Check that the command still errors out stdOut, stdErr, err := e2e.Zarf("package", "create", testPackageDirPath, outputFlag, "--confirm") require.Error(t, err, stdOut, stdErr) - require.Contains(t, stdErr, "template 'ECHO' must be '--set'") + require.Contains(t, stdErr, "template \"ECHO\" must be '--set'") // Check that the command displays a warning on create stdOut, stdErr, err = e2e.Zarf("package", "create", testPackageDirPath, outputFlag, "--confirm", "--set", "ECHO=Zarf-The-Axolotl") diff --git a/src/test/e2e/08_create_differential_test.go b/src/test/e2e/08_create_differential_test.go index 78f4d0ed81..042ed2e185 100644 --- a/src/test/e2e/08_create_differential_test.go +++ b/src/test/e2e/08_create_differential_test.go @@ -9,6 +9,7 @@ import ( "path/filepath" "testing" + "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" @@ -16,14 +17,14 @@ import ( "github.com/stretchr/testify/require" ) -// TestCreateDifferential creates several differential packages and ensures the already built images and repos and not included in the new package +// TestCreateDifferential creates several differential packages and ensures the reference package images and repos are not included in the new package. func TestCreateDifferential(t *testing.T) { t.Log("E2E: Test Differential Package Behavior") tmpdir := t.TempDir() packagePath := "src/test/packages/08-differential-package" - packageName := "zarf-package-differential-package-amd64-v0.25.0.tar.zst" - differentialPackageName := "zarf-package-differential-package-amd64-v0.25.0-differential-v0.26.0.tar.zst" + packageName := fmt.Sprintf("zarf-package-differential-package-%s-v0.25.0.tar.zst", e2e.Arch) + differentialPackageName := fmt.Sprintf("zarf-package-differential-package-%s-v0.25.0-differential-v0.26.0.tar.zst", e2e.Arch) differentialFlag := fmt.Sprintf("--differential=%s", packageName) // Build the package a first time @@ -34,7 +35,7 @@ func TestCreateDifferential(t *testing.T) { // Build the differential package without changing the version _, stdErr, err = e2e.Zarf("package", "create", packagePath, "--set=PACKAGE_VERSION=v0.25.0", differentialFlag, "--confirm") require.Error(t, err, "zarf package create should have errored when a differential package was being created without updating the package version number") - require.Contains(t, stdErr, "unable to create a differential package with the same version") + require.Contains(t, e2e.StripMessageFormatting(stdErr), lang.PkgCreateErrDifferentialSameVersion) // Build the differential package stdOut, stdErr, err = e2e.Zarf("package", "create", packagePath, "--set=PACKAGE_VERSION=v0.26.0", differentialFlag, "--confirm") diff --git a/src/test/e2e/20_zarf_init_test.go b/src/test/e2e/20_zarf_init_test.go index fb1934561b..d8ae669721 100644 --- a/src/test/e2e/20_zarf_init_test.go +++ b/src/test/e2e/20_zarf_init_test.go @@ -70,7 +70,7 @@ func TestZarfInit(t *testing.T) { require.NoError(t, err) require.Contains(t, initStdErr, "an inventory of all software contained in this package") - logText := e2e.GetLogFileContents(t, initStdErr) + logText := e2e.GetLogFileContents(t, e2e.StripMessageFormatting(initStdErr)) // Verify that any state secrets were not included in the log state := types.ZarfState{} diff --git a/src/test/e2e/24_variables_test.go b/src/test/e2e/24_variables_test.go index eacd883f41..c0f546ad83 100644 --- a/src/test/e2e/24_variables_test.go +++ b/src/test/e2e/24_variables_test.go @@ -64,7 +64,7 @@ func TestVariables(t *testing.T) { // Verify that the sensitive variable 'unicorn-land' was not printed to the screen require.NotContains(t, stdErr, "unicorn-land") - logText := e2e.GetLogFileContents(t, stdErr) + logText := e2e.GetLogFileContents(t, e2e.StripMessageFormatting(stdErr)) // Verify that the sensitive variable 'unicorn-land' was not included in the log require.NotContains(t, logText, "unicorn-land") diff --git a/src/test/e2e/51_oci_compose_test.go b/src/test/e2e/51_oci_compose_test.go index 4ad8769d8e..4412e13226 100644 --- a/src/test/e2e/51_oci_compose_test.go +++ b/src/test/e2e/51_oci_compose_test.go @@ -19,6 +19,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + corev1 "k8s.io/api/core/v1" "oras.land/oras-go/v2/registry" ) @@ -71,6 +72,11 @@ func (suite *SkeletonSuite) Test_0_Publish_Skeletons() { suite.NoError(err) suite.Contains(stdErr, "Published "+ref) + composable := filepath.Join("src", "test", "packages", "09-composable-packages") + _, stdErr, err = e2e.Zarf("package", "publish", composable, "oci://"+ref, "--insecure") + suite.NoError(err) + suite.Contains(stdErr, "Published "+ref) + _, stdErr, err = e2e.Zarf("package", "publish", importEverything, "oci://"+ref, "--insecure") suite.NoError(err) suite.Contains(stdErr, "Published "+ref) @@ -86,6 +92,9 @@ func (suite *SkeletonSuite) Test_0_Publish_Skeletons() { _, _, err = e2e.Zarf("package", "pull", "oci://"+ref+"/big-bang-min:2.10.0", "-o", "build", "--insecure", "-a", "skeleton") suite.NoError(err) + + _, _, err = e2e.Zarf("package", "pull", "oci://"+ref+"/test-compose-package:0.0.1", "-o", "build", "--insecure", "-a", "skeleton") + suite.NoError(err) } func (suite *SkeletonSuite) Test_1_Compose_Everything_Inception() { @@ -122,6 +131,7 @@ func (suite *SkeletonSuite) Test_2_FilePaths() { filepath.Join("build", fmt.Sprintf("zarf-package-importception-%s-0.0.1.tar.zst", e2e.Arch)), filepath.Join("build", "zarf-package-helm-charts-skeleton-0.0.1.tar.zst"), filepath.Join("build", "zarf-package-big-bang-min-skeleton-2.10.0.tar.zst"), + filepath.Join("build", "zarf-package-test-compose-package-skeleton-0.0.1.tar.zst"), } for _, pkgTar := range pkgTars { @@ -134,6 +144,24 @@ func (suite *SkeletonSuite) Test_2_FilePaths() { suite.NoError(err) suite.DirExists(unpacked) + // Verify skeleton contains kustomize-generated manifests. + if strings.HasSuffix(pkgTar, "zarf-package-test-compose-package-skeleton-0.0.1.tar.zst") { + kustomizeGeneratedManifests := []string{ + "kustomization-connect-service-0.yaml", + "kustomization-connect-service-1.yaml", + "kustomization-connect-service-two-0.yaml", + } + manifestDir := filepath.Join(unpacked, "components", "test-compose-package", "manifests") + for _, manifest := range kustomizeGeneratedManifests { + manifestPath := filepath.Join(manifestDir, manifest) + suite.FileExists(manifestPath, "expected to find kustomize-generated manifest: %q", manifestPath) + var configMap corev1.ConfigMap + err := utils.ReadYaml(manifestPath, &configMap) + suite.NoError(err) + suite.Equal("ConfigMap", configMap.Kind, "expected manifest %q to be of kind ConfigMap", manifestPath) + } + } + err = utils.ReadYaml(filepath.Join(unpacked, layout.ZarfYAML), &pkg) suite.NoError(err) suite.NotNil(pkg) diff --git a/src/test/packages/09-composable-packages/zarf.yaml b/src/test/packages/09-composable-packages/zarf.yaml index 21e40e5df2..cf121120e6 100644 --- a/src/test/packages/09-composable-packages/zarf.yaml +++ b/src/test/packages/09-composable-packages/zarf.yaml @@ -2,6 +2,7 @@ kind: ZarfPackageConfig metadata: name: test-compose-package description: A contrived example for podinfo using many Zarf primitives for compose testing + version: 0.0.1 components: - name: test-compose-package diff --git a/src/types/package.go b/src/types/package.go index 0ec8f58a23..7a4e91a503 100644 --- a/src/types/package.go +++ b/src/types/package.go @@ -24,6 +24,21 @@ type ZarfPackage struct { Variables []ZarfPackageVariable `json:"variables,omitempty" jsonschema:"description=Variable template values applied on deploy for K8s resources"` } +// IsInitConfig returns whether a Zarf package is an init config. +func (pkg ZarfPackage) IsInitConfig() bool { + return pkg.Kind == ZarfInitConfig +} + +// IsSBOMAble checks if a package has contents that an SBOM can be created on (i.e. images, files, or data injections). +func (pkg ZarfPackage) IsSBOMAble() bool { + for _, c := range pkg.Components { + if len(c.Images) > 0 || len(c.Files) > 0 || len(c.DataInjections) > 0 { + return true + } + } + return false +} + // ZarfMetadata lists information about the current ZarfPackage. type ZarfMetadata struct { Name string `json:"name" jsonschema:"description=Name to identify this Zarf package,pattern=^[a-z0-9\\-]*[a-z0-9]$"` @@ -43,17 +58,18 @@ type ZarfMetadata struct { // ZarfBuildData is written during the packager.Create() operation to track details of the created package. type ZarfBuildData struct { - Terminal string `json:"terminal" jsonschema:"description=The machine name that created this package"` - User string `json:"user" jsonschema:"description=The username who created this package"` - Architecture string `json:"architecture" jsonschema:"description=The architecture this package was created on"` - Timestamp string `json:"timestamp" jsonschema:"description=The timestamp when this package was created"` - Version string `json:"version" jsonschema:"description=The version of Zarf used to build this package"` - Migrations []string `json:"migrations,omitempty" jsonschema:"description=Any migrations that have been run on this package"` - Differential bool `json:"differential,omitempty" jsonschema:"description=Whether this package was created with differential components"` - RegistryOverrides map[string]string `json:"registryOverrides,omitempty" jsonschema:"description=Any registry domains that were overridden on package create when pulling images"` - DifferentialMissing []string `json:"differentialMissing,omitempty" jsonschema:"description=List of components that were not included in this package due to differential packaging"` - LastNonBreakingVersion string `json:"lastNonBreakingVersion,omitempty" jsonschema:"description=The minimum version of Zarf that does not have breaking package structure changes"` - Flavor string `json:"flavor,omitempty" jsonschema:"description=The flavor of Zarf used to build this package"` + Terminal string `json:"terminal" jsonschema:"description=The machine name that created this package"` + User string `json:"user" jsonschema:"description=The username who created this package"` + Architecture string `json:"architecture" jsonschema:"description=The architecture this package was created on"` + Timestamp string `json:"timestamp" jsonschema:"description=The timestamp when this package was created"` + Version string `json:"version" jsonschema:"description=The version of Zarf used to build this package"` + Migrations []string `json:"migrations,omitempty" jsonschema:"description=Any migrations that have been run on this package"` + RegistryOverrides map[string]string `json:"registryOverrides,omitempty" jsonschema:"description=Any registry domains that were overridden on package create when pulling images"` + Differential bool `json:"differential,omitempty" jsonschema:"description=Whether this package was created with differential components"` + DifferentialPackageVersion string `json:"differentialPackageVersion,omitempty" jsonschema:"description=Version of a previously built package used as the basis for creating this differential package"` + DifferentialMissing []string `json:"differentialMissing,omitempty" jsonschema:"description=List of components that were not included in this package due to differential packaging"` + LastNonBreakingVersion string `json:"lastNonBreakingVersion,omitempty" jsonschema:"description=The minimum version of Zarf that does not have breaking package structure changes"` + Flavor string `json:"flavor,omitempty" jsonschema:"description=The flavor of Zarf used to build this package"` } // ZarfPackageVariable are variables that can be used to dynamically template K8s resources. diff --git a/src/types/packager.go b/src/types/packager.go index 9e945e1472..857683a529 100644 --- a/src/types/packager.go +++ b/src/types/packager.go @@ -4,6 +4,11 @@ // Package types contains all the types used by Zarf. package types +import ( + "fmt" + "regexp" +) + // PackagerConfig is the main struct that the packager uses to hold high-level options. type PackagerConfig struct { // CreateOpts tracks the user-defined options used to create the package @@ -45,3 +50,22 @@ type PackagerConfig struct { // Variables set by the user SetVariableMap map[string]*ZarfSetVariable } + +// SetVariable sets a value for a variable in PackagerConfig.SetVariableMap. +func (cfg *PackagerConfig) SetVariable(name, value string, sensitive bool, autoIndent bool, varType VariableType) { + cfg.SetVariableMap[name] = &ZarfSetVariable{ + Name: name, + Value: value, + Sensitive: sensitive, + AutoIndent: autoIndent, + Type: varType, + } +} + +// CheckVariablePattern checks to see if a variable is set to a value that matches its pattern. +func (cfg *PackagerConfig) CheckVariablePattern(name, pattern string) error { + if regexp.MustCompile(pattern).MatchString(cfg.SetVariableMap[name].Value) { + return nil + } + return fmt.Errorf("provided value for variable %q does not match pattern \"%s\"", name, pattern) +} diff --git a/src/types/runtime.go b/src/types/runtime.go index 4b69491a08..1ee40f91a0 100644 --- a/src/types/runtime.go +++ b/src/types/runtime.go @@ -111,20 +111,20 @@ type ZarfInitOptions struct { // ZarfCreateOptions tracks the user-defined options used to create the package. type ZarfCreateOptions struct { - SkipSBOM bool `json:"skipSBOM" jsonschema:"description=Disable the generation of SBOM materials during package creation"` - BaseDir string `json:"baseDir" jsonschema:"description=Location where the Zarf package will be created from"` - Output string `json:"output" jsonschema:"description=Location where the finalized Zarf package will be placed"` - ViewSBOM bool `json:"sbom" jsonschema:"description=Whether to pause to allow for viewing the SBOM post-creation"` - SBOMOutputDir string `json:"sbomOutput" jsonschema:"description=Location to output an SBOM into after package creation"` - SetVariables map[string]string `json:"setVariables" jsonschema:"description=Key-Value map of variable names and their corresponding values that will be used to template against the Zarf package being used"` - MaxPackageSizeMB int `json:"maxPackageSizeMB" jsonschema:"description=Size of chunks to use when splitting a zarf package into multiple files in megabytes"` - SigningKeyPath string `json:"signingKeyPath" jsonschema:"description=Location where the private key component of a cosign key-pair can be found"` - SigningKeyPassword string `json:"signingKeyPassword" jsonschema:"description=Password to the private key signature file that will be used to sigh the created package"` - DifferentialData DifferentialData `json:"differential" jsonschema:"description=A package's differential images and git repositories from a referenced previously built package"` - RegistryOverrides map[string]string `json:"registryOverrides" jsonschema:"description=A map of domains to override on package create when pulling images"` - Flavor string `json:"flavor" jsonschema:"description=An optional variant that controls which components will be included in a package"` - IsSkeleton bool `json:"isSkeleton" jsonschema:"description=Whether to create a skeleton package"` - NoYOLO bool `json:"noYOLO" jsonschema:"description=Whether to create a YOLO package"` + SkipSBOM bool `json:"skipSBOM" jsonschema:"description=Disable the generation of SBOM materials during package creation"` + BaseDir string `json:"baseDir" jsonschema:"description=Location where the Zarf package will be created from"` + Output string `json:"output" jsonschema:"description=Location where the finalized Zarf package will be placed"` + ViewSBOM bool `json:"sbom" jsonschema:"description=Whether to pause to allow for viewing the SBOM post-creation"` + SBOMOutputDir string `json:"sbomOutput" jsonschema:"description=Location to output an SBOM into after package creation"` + SetVariables map[string]string `json:"setVariables" jsonschema:"description=Key-Value map of variable names and their corresponding values that will be used to template against the Zarf package being used"` + MaxPackageSizeMB int `json:"maxPackageSizeMB" jsonschema:"description=Size of chunks to use when splitting a zarf package into multiple files in megabytes"` + SigningKeyPath string `json:"signingKeyPath" jsonschema:"description=Location where the private key component of a cosign key-pair can be found"` + SigningKeyPassword string `json:"signingKeyPassword" jsonschema:"description=Password to the private key signature file that will be used to sigh the created package"` + DifferentialPackagePath string `json:"differentialPackagePath" jsonschema:"description=Path to a previously built package used as the basis for creating a differential package"` + RegistryOverrides map[string]string `json:"registryOverrides" jsonschema:"description=A map of domains to override on package create when pulling images"` + Flavor string `json:"flavor" jsonschema:"description=An optional variant that controls which components will be included in a package"` + IsSkeleton bool `json:"isSkeleton" jsonschema:"description=Whether to create a skeleton package"` + NoYOLO bool `json:"noYOLO" jsonschema:"description=Whether to create a YOLO package"` } // ZarfSplitPackageData contains info about a split package. @@ -154,8 +154,7 @@ type ConnectStrings map[string]ConnectString // DifferentialData contains image and repository information about the package a Differential Package is Based on. type DifferentialData struct { - DifferentialPackagePath string - DifferentialPackageVersion string DifferentialImages map[string]bool DifferentialRepos map[string]bool + DifferentialPackageVersion string } diff --git a/zarf.schema.json b/zarf.schema.json index f8e53e98e2..539081048a 100644 --- a/zarf.schema.json +++ b/zarf.schema.json @@ -118,10 +118,6 @@ "type": "array", "description": "Any migrations that have been run on this package" }, - "differential": { - "type": "boolean", - "description": "Whether this package was created with differential components" - }, "registryOverrides": { "patternProperties": { ".*": { @@ -131,6 +127,14 @@ "type": "object", "description": "Any registry domains that were overridden on package create when pulling images" }, + "differential": { + "type": "boolean", + "description": "Whether this package was created with differential components" + }, + "differentialPackageVersion": { + "type": "string", + "description": "Version of a previously built package used as the basis for creating this differential package" + }, "differentialMissing": { "items": { "type": "string"