From c9135aff92e429eb6d218532cab899033a589871 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Wed, 31 May 2023 16:26:49 -0500 Subject: [PATCH 01/20] Initial implementation of bb oci compose --- src/extensions/bigbang/bigbang.go | 24 ++++++++++++ src/extensions/bigbang/manifests.go | 2 +- src/pkg/packager/create.go | 22 ++++------- src/pkg/packager/extensions.go | 61 +++++++++++++++++++++++++---- src/pkg/packager/publish.go | 5 +++ 5 files changed, 91 insertions(+), 23 deletions(-) diff --git a/src/extensions/bigbang/bigbang.go b/src/extensions/bigbang/bigbang.go index 9e2d1fb93a..3eea11b591 100644 --- a/src/extensions/bigbang/bigbang.go +++ b/src/extensions/bigbang/bigbang.go @@ -244,6 +244,30 @@ func Run(YOLO bool, tmpPaths types.ComponentPaths, c types.ZarfComponent) (types return c, nil } +// Skeletonize Mutates a component so that the valuesFiles can be contained inside a skeleton package +func Skeletonize(tmpPaths types.ComponentPaths, c types.ZarfComponent) (types.ZarfComponent, error) { + for valuesIdx, valuesFile := range c.Extensions.BigBang.ValuesFiles { + // Define the name as the file name without the extension. + baseName := strings.TrimSuffix(valuesFile, filepath.Ext(valuesFile)) + + // Replace non-alphanumeric characters with a dash. + baseName = nonAlphnumeric.ReplaceAllString(baseName, "-") + + // Add the skeleton name prefix. + skelName := fmt.Sprintf("bb-ext-skeleton-values-%s", baseName) + + dst := filepath.Join(tmpPaths.Temp, skelName) + + if err := utils.CreatePathAndCopy(valuesFile, dst); err != nil { + return c, err + } + + c.Extensions.BigBang.ValuesFiles[valuesIdx] = dst + } + + return c, nil +} + // isValidVersion check if the version is 1.54.0 or greater. func isValidVersion(version string) (bool, error) { specifiedVersion, err := semver.NewVersion(version) diff --git a/src/extensions/bigbang/manifests.go b/src/extensions/bigbang/manifests.go index 0bda10c991..aee7295234 100644 --- a/src/extensions/bigbang/manifests.go +++ b/src/extensions/bigbang/manifests.go @@ -116,7 +116,7 @@ func manifestValuesFile(path string) (secret corev1.Secret, err error) { } // Define the name as the file name without the extension. - baseName := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) + baseName := strings.TrimSuffix(path, filepath.Ext(path)) // Replace non-alphanumeric characters with a dash. baseName = nonAlphnumeric.ReplaceAllString(baseName, "-") diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index a418e97da9..ee3d9132cc 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -64,6 +64,7 @@ func (p *Packager) Create(baseDir string) error { return err } + // Compose components into a single zarf.yaml file if err := p.composeComponents(); err != nil { return err } @@ -73,7 +74,12 @@ func (p *Packager) Create(baseDir string) error { return fmt.Errorf("unable to fill values in template: %s", err.Error()) } - // Remove unnecessary repos and images if we are building a differential package + // After templates are filled process any create extensions + if err := p.processExtensions(); err != nil { + return err + } + + // After we have a full zarf.yaml remove unnecessary repos and images if we are building a differential package if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { // Verify the package version of the package we're using as a 'reference' for the differential build is different than the package we're building // If the package versions are the same return an error @@ -90,20 +96,6 @@ func (p *Packager) Create(baseDir string) error { } } - // Create component paths and process extensions for each component. - for i, c := range p.cfg.Pkg.Components { - componentPath, err := p.createOrGetComponentPaths(c) - if err != nil { - return err - } - - // Process any extensions. - p.cfg.Pkg.Components[i], err = p.processExtensions(p.cfg.Pkg.Metadata.YOLO, componentPath, c) - if err != nil { - return fmt.Errorf("unable to process extensions: %w", err) - } - } - // Perform early package validation. if err := validate.Run(p.cfg.Pkg); err != nil { return fmt.Errorf("unable to validate package: %w", err) diff --git a/src/pkg/packager/extensions.go b/src/pkg/packager/extensions.go index db0b6fd694..37bf06c665 100644 --- a/src/pkg/packager/extensions.go +++ b/src/pkg/packager/extensions.go @@ -8,19 +8,66 @@ import ( "fmt" "github.com/defenseunicorns/zarf/src/extensions/bigbang" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/types" ) // Check for any extensions in use and runs the appropriate functions. -func (p *Packager) processExtensions(YOLO bool, cPaths types.ComponentPaths, c types.ZarfComponent) (types.ZarfComponent, error) { - var err error +func (p *Packager) processExtensions() error { + message.Debugf("packager.processExtensions()") - // Big Bang - if c.Extensions.BigBang != nil { - if c, err = bigbang.Run(YOLO, cPaths, c); err != nil { - return c, fmt.Errorf("unable to process bigbang extension: %w", err) + components := []types.ZarfComponent{} + + // Create component paths and process extensions for each component. + for _, c := range p.cfg.Pkg.Components { + componentPaths, err := p.createOrGetComponentPaths(c) + if err != nil { + return err + } + + // Big Bang + if c.Extensions.BigBang != nil { + if c, err = bigbang.Run(p.cfg.Pkg.Metadata.YOLO, componentPaths, c); err != nil { + return fmt.Errorf("unable to process bigbang extension: %w", err) + } + } + + components = append(components, c) + } + + // Update the parent package config with the expanded sub components. + // This is important when the deploy package is created. + p.cfg.Pkg.Components = components + + return nil +} + +// Check for any extensions in use and skeletonize their local files. +func (p *Packager) skeletonizeExtentions() error { + message.Debugf("packager.processExtensions()") + + components := []types.ZarfComponent{} + + // Create component paths and process extensions for each component. + for _, c := range p.cfg.Pkg.Components { + componentPaths, err := p.createOrGetComponentPaths(c) + if err != nil { + return err } + + // Big Bang + if c.Extensions.BigBang != nil { + if c, err = bigbang.Skeletonize(componentPaths, c); err != nil { + return fmt.Errorf("unable to process bigbang extension: %w", err) + } + } + + components = append(components, c) } - return c, nil + // Update the parent package config with the expanded sub components. + // This is important when the deploy package is created. + p.cfg.Pkg.Components = components + + return nil } diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 6091b72b83..c80ad6cb7b 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -293,6 +293,11 @@ func (p *Packager) loadSkeleton() error { return err } + err = p.skeletonizeExtentions() + if err != nil { + return err + } + for idx, component := range p.cfg.Pkg.Components { isSkeleton := true err := p.addComponent(idx, component, isSkeleton) From 0abbe610866919bee616d67b5331a5862a5a0fc9 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Wed, 31 May 2023 16:33:35 -0500 Subject: [PATCH 02/20] lowercase mutates --- src/extensions/bigbang/bigbang.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/extensions/bigbang/bigbang.go b/src/extensions/bigbang/bigbang.go index 3eea11b591..905371f53d 100644 --- a/src/extensions/bigbang/bigbang.go +++ b/src/extensions/bigbang/bigbang.go @@ -37,7 +37,7 @@ var tenMins = metav1.Duration{ Duration: 10 * time.Minute, } -// Run Mutates a component that should deploy Big Bang to a set of manifests +// Run mutates a component that should deploy Big Bang to a set of manifests // that contain the flux deployment of Big Bang func Run(YOLO bool, tmpPaths types.ComponentPaths, c types.ZarfComponent) (types.ZarfComponent, error) { var err error @@ -244,7 +244,7 @@ func Run(YOLO bool, tmpPaths types.ComponentPaths, c types.ZarfComponent) (types return c, nil } -// Skeletonize Mutates a component so that the valuesFiles can be contained inside a skeleton package +// Skeletonize mutates a component so that the valuesFiles can be contained inside a skeleton package func Skeletonize(tmpPaths types.ComponentPaths, c types.ZarfComponent) (types.ZarfComponent, error) { for valuesIdx, valuesFile := range c.Extensions.BigBang.ValuesFiles { // Define the name as the file name without the extension. From c02c38ba16666ed946611f07e92364b0b9bddb14 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Thu, 1 Jun 2023 09:52:09 -0500 Subject: [PATCH 03/20] Add an error message if the import URL is not a skeleton --- src/pkg/packager/compose.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index 94182efca8..7b1f0b9363 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -88,6 +88,10 @@ func (p *Packager) getChildComponent(parent types.ZarfComponent, pathAncestry st var cachePath string if parent.Import.URL != "" { + if !strings.HasSuffix(parent.Import.URL, skeletonSuffix) { + return child, fmt.Errorf("import URL must be a 'skeleton' package: %s", parent.Import.URL) + } + // Save all the OCI imported components into our build data p.cfg.Pkg.Build.OCIImportedComponents[parent.Import.URL] = childComponentName From 1e96f6f06e3379ef038118c0d1e5289fcd763763 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Thu, 1 Jun 2023 11:12:42 -0500 Subject: [PATCH 04/20] Get skeletonization and Composability working --- src/extensions/bigbang/bigbang.go | 17 ++++++++++++++--- src/pkg/packager/compose.go | 2 ++ src/pkg/packager/extensions.go | 14 +++++++++++++- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/extensions/bigbang/bigbang.go b/src/extensions/bigbang/bigbang.go index 905371f53d..faad798596 100644 --- a/src/extensions/bigbang/bigbang.go +++ b/src/extensions/bigbang/bigbang.go @@ -254,20 +254,31 @@ func Skeletonize(tmpPaths types.ComponentPaths, c types.ZarfComponent) (types.Za baseName = nonAlphnumeric.ReplaceAllString(baseName, "-") // Add the skeleton name prefix. - skelName := fmt.Sprintf("bb-ext-skeleton-values-%s", baseName) + skelName := fmt.Sprintf("bb-ext-skeleton-values-%s.yaml", baseName) - dst := filepath.Join(tmpPaths.Temp, skelName) + rel := filepath.Join(types.TempFolder, skelName) + dst := filepath.Join(tmpPaths.Base, rel) if err := utils.CreatePathAndCopy(valuesFile, dst); err != nil { return c, err } - c.Extensions.BigBang.ValuesFiles[valuesIdx] = dst + c.Extensions.BigBang.ValuesFiles[valuesIdx] = rel } return c, nil } +// Compose mutates a component so that the valuesFiles are relative to the parent importing component +func Compose(pathAncestry string, c types.ZarfComponent) types.ZarfComponent { + for valuesIdx, valuesFile := range c.Extensions.BigBang.ValuesFiles { + parentRel := filepath.Join(pathAncestry, valuesFile) + c.Extensions.BigBang.ValuesFiles[valuesIdx] = parentRel + } + + return c +} + // isValidVersion check if the version is 1.54.0 or greater. func isValidVersion(version string) (bool, error) { specifiedVersion, err := semver.NewVersion(version) diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index 7b1f0b9363..e935a97e7a 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -260,6 +260,8 @@ func (p *Packager) fixComposedFilepaths(pathAncestry string, child types.ZarfCom child.CosignKeyPath = composed } + child = p.composeExtentions(pathAncestry, child) + return child, nil } diff --git a/src/pkg/packager/extensions.go b/src/pkg/packager/extensions.go index 37bf06c665..9190581bca 100644 --- a/src/pkg/packager/extensions.go +++ b/src/pkg/packager/extensions.go @@ -42,9 +42,21 @@ func (p *Packager) processExtensions() error { return nil } +// Mutate any local files to be relative to the parent +func (p *Packager) composeExtentions(pathAncestry string, component types.ZarfComponent) types.ZarfComponent { + message.Debugf("packager.composeExtentions()") + + // Big Bang + if component.Extensions.BigBang != nil { + component = bigbang.Compose(pathAncestry, component) + } + + return component +} + // Check for any extensions in use and skeletonize their local files. func (p *Packager) skeletonizeExtentions() error { - message.Debugf("packager.processExtensions()") + message.Debugf("packager.skeletonizeExtentions()") components := []types.ZarfComponent{} From 292ee782169b47cd43062e1a976122e5e2442c35 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Thu, 1 Jun 2023 17:11:20 -0500 Subject: [PATCH 05/20] Improve docs for skeleton packages and other fixes --- .../2-zarf-components.md | 2 +- .../2-deploying-zarf-packages.md | 2 +- docs/5-zarf-tutorials/7-publish-and-deploy.md | 10 ++++- examples/composable-packages/README.md | 38 +++++++++++++++---- .../composable-packages/quake-service.yaml | 18 +++++++++ examples/composable-packages/zarf.yaml | 34 ++++++++++++++--- src/cmd/initialize.go | 2 + 7 files changed, 89 insertions(+), 17 deletions(-) create mode 100644 examples/composable-packages/quake-service.yaml diff --git a/docs/3-create-a-zarf-package/2-zarf-components.md b/docs/3-create-a-zarf-package/2-zarf-components.md index 979ed1af1f..33fc1075f4 100644 --- a/docs/3-create-a-zarf-package/2-zarf-components.md +++ b/docs/3-create-a-zarf-package/2-zarf-components.md @@ -168,7 +168,7 @@ The [`podinfo-flux`](/examples/podinfo-flux/) example showcases a simple GitOps - + diff --git a/docs/5-zarf-tutorials/2-deploying-zarf-packages.md b/docs/5-zarf-tutorials/2-deploying-zarf-packages.md index bc6b0e6c28..d328bcfb79 100644 --- a/docs/5-zarf-tutorials/2-deploying-zarf-packages.md +++ b/docs/5-zarf-tutorials/2-deploying-zarf-packages.md @@ -28,7 +28,7 @@ If you do not provide the path to the package as an argument to the `zarf packag -By hitting 'tab', you can use the arrow keys to select which package you want to deploy. Since we are deploying the Helm OCI chart package in this tutorial, we will select that package and hit 'enter'. +By hitting 'tab', you can use the arrow keys to select which package you want to deploy. Since we are deploying the WordPress package in this tutorial, we will select that package and hit 'enter'. diff --git a/docs/5-zarf-tutorials/7-publish-and-deploy.md b/docs/5-zarf-tutorials/7-publish-and-deploy.md index c9589418b4..bc8b3e33f7 100644 --- a/docs/5-zarf-tutorials/7-publish-and-deploy.md +++ b/docs/5-zarf-tutorials/7-publish-and-deploy.md @@ -1,4 +1,4 @@ -# Store & Deploy Packages with OCI +# Publish & Deploy Packages w/OCI ## Introduction @@ -71,6 +71,14 @@ To modify, edit `zarf.yaml` and re-run `zarf package create .` ::: +:::info + +The above publish command publishes a fully-built package that is ready to deploy to a cluster, but this isn't the only kind of package you can publish to an OCI registry. + +If you specify a path to a directory containing a `zarf.yaml` (as you would on `zarf package create`) you can create a `skeleton` package that is importable into other packages using [component composability](../../examples/composable-packages/README.md). These can be inspected like normal Zarf packages but cannot be deployed as they haven't been fully created yet. + +::: + ### Inspect Package [CLI Reference](../2-the-zarf-cli/100-cli-commands/zarf_package_inspect.md) diff --git a/examples/composable-packages/README.md b/examples/composable-packages/README.md index fb1b398bf5..f6a4792a65 100644 --- a/examples/composable-packages/README.md +++ b/examples/composable-packages/README.md @@ -2,20 +2,25 @@ import ExampleYAML from "@site/src/components/ExampleYAML"; # Composable Packages -This example demonstrates using Zarf to compose existing zarf packages into another package. It uses the existing [zarf game](../dos-games/) example by simply adding an `import` and `path` in the new [zarf.yaml](zarf.yaml). +This example demonstrates using Zarf to import components from existing Zarf package definitions while merging overrides to add or change functionality. It uses the existing [DOS games](../dos-games/README.md) and [WordPress](../wordpress/README.md) examples by simply adding `import` keys in the new [zarf.yaml](zarf.yaml) file. -## Example Prerequisites +The `import` key in Zarf supports two modes to pull in a component: -Creating this example requires a locally hosted container registry that has the `helm-charts` skeleton package published and available. You can do this by running the following commands: +1. The `path` key allows you to specify a path to a directory that contains the `zarf.yaml` that you wish to import on your local filesystem. This allows you to have a common component that you can reuse across multiple packages *within* a project. -```bash -docker run -d -p 5000:5000 --restart=always --name registry registry:2 -zarf package publish examples/helm-charts oci://127.0.0.1:5000 --insecure -``` +2. The `url` key allows you to specify an `oci://` URL to a skeleton package that was published to an OCI registry. Skeleton packages are special package bundles that contain the `zarf.yaml` package definition and any local files referenced by that definition at publish time. This allows you to version a set of components and import them into multiple packages *across* projects. + +:::info + +As you can see in the example the `import` key can be combined with other keys to merge components together. This can be done many components deep if you wish and in the end will generate one main `zarf.yaml` with all of the defined resources included. + +This is useful if you want to slightly tweak a given component while maintaining a common core. + +::: :::note -Import paths must be statically defined at create time. You cannot use [variables](../variables/) in them. +The import `path` or `url` must be statically defined at create time. You cannot use [package templates](../variables/README.md#create-time-package-configuration-templates) in them. ::: @@ -27,4 +32,21 @@ To view the example in its entirety, select the `Edit this page` link below the ::: +:::note + +Creating this example requires a locally hosted container registry that has the `wordpress` skeleton package published and available. You can do this by running the following commands: + +```bash +docker run -d -p 5000:5000 --restart=always --name registry registry:2 +zarf package publish examples/wordpress oci://127.0.0.1:5000 --insecure +``` + +You will also need to pass the `--insecure` flag to `zarf package create` to pull from the `http` registry: + +```bash +zarf package create examples/composable-packages/ --insecure +``` + +::: + diff --git a/examples/composable-packages/quake-service.yaml b/examples/composable-packages/quake-service.yaml new file mode 100644 index 0000000000..8b68aac4e8 --- /dev/null +++ b/examples/composable-packages/quake-service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: quake + annotations: + zarf.dev/connect-description: "Play quake!!!" + zarf.dev/connect-url: "?quake" + labels: + # Enables "zarf connect quake" + zarf.dev/connect-name: quake +spec: + selector: + app: game + ports: + - name: http + port: 8000 + protocol: TCP + targetPort: 8000 diff --git a/examples/composable-packages/zarf.yaml b/examples/composable-packages/zarf.yaml index 14df579b51..664b8ba2ad 100644 --- a/examples/composable-packages/zarf.yaml +++ b/examples/composable-packages/zarf.yaml @@ -4,16 +4,38 @@ metadata: description: Demo Zarf package composability components: + # LOCAL COMPOSE IMPORT - name: games + # The component logic keys ('required', 'group', and 'default') always override those of the imported package required: true - description: Example of a composed package with a unique description for this component + # group: "" # the initial value overrides the child component + # default: false # the initial value overrides the child component + description: "Example of a local composed package with a unique description for this component" import: + # The local relative path to the folder containing this component's package definition path: ../dos-games - # Example optional custom name to point to in the imported package + # Example optional custom name to point to in the imported package (default is to use this component's name) name: baseline + # Array keys are appended to the end of anything the imported component defined (in this case an extra service) + manifests: + - name: additional-games + namespace: dos-games + files: + - quake-service.yaml - - name: chart-via-oci - required: false + # OCI COMPOSE IMPORT + - name: wordpress + # The component logic keys ('required', 'group', and 'default') always override those of the imported package + # required: false # the initial value overrides the child component + # group: "" # the initial value overrides the child component + # default: false # the initial value overrides the child component import: - url: oci://localhost:5000/helm-charts:0.0.1-skeleton - name: demo-helm-local-chart + # The URL to the skeleton package containing this component's package definition + url: oci://localhost:5000/wordpress:16.0.4-skeleton + # Array keys are appended to the end of anything the imported component defined (in this case an override of the blog name) + actions: + onDeploy: + before: + - cmd: ./zarf tools kubectl get -n dos-games deployment -o jsonpath={.items[0].metadata.creationTimestamp} + setVariables: + - name: WORDPRESS_BLOG_NAME diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index c85b9ba75f..f20dac3079 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -47,6 +47,8 @@ var initCmd = &cobra.Command{ message.Fatal(err, err.Error()) } + pkgConfig.PkgSourcePath = pkgConfig.DeployOpts.PackagePath + // Ensure uppercase keys from viper viperConfig := utils.TransformMapKeys(v.GetStringMapString(V_PKG_DEPLOY_SET), strings.ToUpper) pkgConfig.DeployOpts.SetVariables = utils.MergeMap(viperConfig, pkgConfig.DeployOpts.SetVariables) From c5c197125f8e2b0fc7f1794b2e6c874e0a1bf528 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Thu, 1 Jun 2023 17:48:41 -0500 Subject: [PATCH 06/20] Add tests for BB OCI --- src/extensions/bigbang/test/bigbang_test.go | 4 +-- src/test/e2e/51_oci_compose_test.go | 27 ++++++++++++++++--- .../packages/51-import-everything/zarf.yaml | 1 + 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/extensions/bigbang/test/bigbang_test.go b/src/extensions/bigbang/test/bigbang_test.go index 87cfbd95ee..bb4bbbc6cc 100644 --- a/src/extensions/bigbang/test/bigbang_test.go +++ b/src/extensions/bigbang/test/bigbang_test.go @@ -71,13 +71,13 @@ func TestReleases(t *testing.T) { // Build the latest version bbVersion = fmt.Sprintf("--set=BB_VERSION=%s", latest) bbMajor = fmt.Sprintf("--set=BB_MAJOR=%s", latest[0:1]) - zarfExec(t, "package", "create", "../src/extensions/bigbang/test/package", bbVersion, bbMajor, "--confirm") + zarfExec(t, "package", "create", "../src/extensions/bigbang/test/package", bbVersion, bbMajor, "--differential", pkgPath, "--confirm") // Clean up zarf cache now that all packages are built to reduce disk pressure zarfExec(t, "tools", "clear-cache") // Deploy the latest version - pkgPath = fmt.Sprintf("zarf-package-big-bang-test-amd64-%s.tar.zst", latest) + pkgPath = fmt.Sprintf("zarf-package-big-bang-test-amd64-%s-differential-%s.tar.zst", previous, latest) zarfExec(t, "package", "deploy", pkgPath, "--confirm") // Cluster info diff --git a/src/test/e2e/51_oci_compose_test.go b/src/test/e2e/51_oci_compose_test.go index 5c6ea8cf17..c1b1231879 100644 --- a/src/test/e2e/51_oci_compose_test.go +++ b/src/test/e2e/51_oci_compose_test.go @@ -71,10 +71,15 @@ func (suite *SkeletonSuite) TearDownSuite() { func (suite *SkeletonSuite) Test_0_Publish_Skeletons() { suite.T().Log("E2E: Skeleton Package Publish oci://") - - helmLocal := filepath.Join("examples", "helm-charts") ref := suite.Reference.String() - _, stdErr, err := e2e.Zarf("package", "publish", helmLocal, "oci://"+ref, "--insecure") + + helmCharts := filepath.Join("examples", "helm-charts") + _, stdErr, err := e2e.Zarf("package", "publish", helmCharts, "oci://"+ref, "--insecure") + suite.NoError(err) + suite.Contains(stdErr, "Published "+ref) + + bigBang := filepath.Join("examples", "big-bang") + _, stdErr, err = e2e.Zarf("package", "publish", bigBang, "oci://"+ref, "--insecure") suite.NoError(err) suite.Contains(stdErr, "Published "+ref) @@ -85,8 +90,14 @@ func (suite *SkeletonSuite) Test_0_Publish_Skeletons() { _, _, err = e2e.Zarf("package", "inspect", "oci://"+ref+"/import-everything:0.0.1-skeleton", "--insecure") suite.NoError(err) + _, _, err = e2e.Zarf("package", "pull", "oci://"+ref+"/import-everything:0.0.1-skeleton", "-o", "build", "--insecure") + suite.NoError(err) + _, _, err = e2e.Zarf("package", "pull", "oci://"+ref+"/helm-charts:0.0.1-skeleton", "-o", "build", "--insecure") suite.NoError(err) + + _, _, err = e2e.Zarf("package", "pull", "oci://"+ref+"/big-bang-example:2.0.0-skeleton", "-o", "build", "--insecure") + suite.NoError(err) } func (suite *SkeletonSuite) Test_1_Compose() { @@ -104,8 +115,10 @@ func (suite *SkeletonSuite) Test_3_FilePaths() { pkgTars := []string{ filepath.Join("build", fmt.Sprintf("zarf-package-import-everything-%s-0.0.1.tar.zst", e2e.Arch)), + filepath.Join("build", "zarf-package-import-everything-skeleton-0.0.1.tar.zst"), filepath.Join("build", fmt.Sprintf("zarf-package-importception-%s-0.0.1.tar.zst", e2e.Arch)), filepath.Join("build", "zarf-package-helm-charts-skeleton-0.0.1.tar.zst"), + filepath.Join("build", "zarf-package-big-bang-example-skeleton-2.0.0.tar.zst"), } for _, pkgTar := range pkgTars { @@ -126,7 +139,7 @@ func (suite *SkeletonSuite) Test_3_FilePaths() { suite.NotNil(components) isSkeleton := false - if pkgTar == filepath.Join("build", "zarf-package-helm-charts-skeleton-0.0.1.tar.zst") { + if strings.Contains(pkgTar, "-skeleton-") { isSkeleton = true } suite.verifyComponentPaths(unpacked, components, isSkeleton) @@ -167,6 +180,12 @@ func (suite *SkeletonSuite) verifyComponentPaths(unpackedPath string, components suite.FileExists(filepath.Join(base, component.CosignKeyPath)) } + if isSkeleton && component.Extensions.BigBang != nil { + for _, valuesFile := range component.Extensions.BigBang.ValuesFiles { + suite.FileExists(filepath.Join(base, valuesFile)) + } + } + for chartIdx, chart := range component.Charts { if isSkeleton && chart.URL != "" { continue diff --git a/src/test/packages/51-import-everything/zarf.yaml b/src/test/packages/51-import-everything/zarf.yaml index ea3901a32d..8c536be486 100644 --- a/src/test/packages/51-import-everything/zarf.yaml +++ b/src/test/packages/51-import-everything/zarf.yaml @@ -32,6 +32,7 @@ components: - name: file-imports required: false + cosignKeyPath: ../../../../cosign.pub files: # Import of a file within the current directory - source: files/coffee-ipsum.txt From 88708a25d308bb736dccfedb777015c09a64ab4b Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Thu, 1 Jun 2023 17:57:53 -0500 Subject: [PATCH 07/20] Make the README have more information --- examples/composable-packages/README.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/examples/composable-packages/README.md b/examples/composable-packages/README.md index f6a4792a65..2fec4cd10c 100644 --- a/examples/composable-packages/README.md +++ b/examples/composable-packages/README.md @@ -10,9 +10,19 @@ The `import` key in Zarf supports two modes to pull in a component: 2. The `url` key allows you to specify an `oci://` URL to a skeleton package that was published to an OCI registry. Skeleton packages are special package bundles that contain the `zarf.yaml` package definition and any local files referenced by that definition at publish time. This allows you to version a set of components and import them into multiple packages *across* projects. +:::tip + +You can create a skeleton package from a `zarf.yaml` by pointing `zarf package publish` at the directory that contains it: + +```bash +zarf package publish path/containing/package/definition oci://your-registry.com +``` + +::: + :::info -As you can see in the example the `import` key can be combined with other keys to merge components together. This can be done many components deep if you wish and in the end will generate one main `zarf.yaml` with all of the defined resources included. +As you can see in the example, the `import` key can be combined with other keys to merge components together. This can be done as many components deep as you wish and in the end will generate one main `zarf.yaml` file with all of the defined resources included. This is useful if you want to slightly tweak a given component while maintaining a common core. @@ -20,7 +30,7 @@ This is useful if you want to slightly tweak a given component while maintaining :::note -The import `path` or `url` must be statically defined at create time. You cannot use [package templates](../variables/README.md#create-time-package-configuration-templates) in them. +The import `path` or `url` must be statically defined at create time. You cannot use [package templates](../variables/README.md#create-time-package-configuration-templates) within them. ::: From bd091590b5bb07febadb341a469c0568ea63b92f Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Thu, 1 Jun 2023 18:00:05 -0500 Subject: [PATCH 08/20] put the info and note in a better spot --- examples/composable-packages/README.md | 28 +++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/composable-packages/README.md b/examples/composable-packages/README.md index 2fec4cd10c..da6ca4b1bc 100644 --- a/examples/composable-packages/README.md +++ b/examples/composable-packages/README.md @@ -20,20 +20,6 @@ zarf package publish path/containing/package/definition oci://your-registry.com ::: -:::info - -As you can see in the example, the `import` key can be combined with other keys to merge components together. This can be done as many components deep as you wish and in the end will generate one main `zarf.yaml` file with all of the defined resources included. - -This is useful if you want to slightly tweak a given component while maintaining a common core. - -::: - -:::note - -The import `path` or `url` must be statically defined at create time. You cannot use [package templates](../variables/README.md#create-time-package-configuration-templates) within them. - -::: - ## `zarf.yaml` {#zarf.yaml} :::info @@ -60,3 +46,17 @@ zarf package create examples/composable-packages/ --insecure ::: + +:::info + +As you can see in the example, the `import` key can be combined with other keys to merge components together. This can be done as many components deep as you wish and in the end will generate one main `zarf.yaml` file with all of the defined resources included. + +This is useful if you want to slightly tweak a given component while maintaining a common core. + +::: + +:::note + +The import `path` or `url` must be statically defined at create time. You cannot use [package templates](../variables/README.md#create-time-package-configuration-templates) within them. + +::: From 0fe6b594a458f7b5902ef2a6c179a19ea1b9a5c7 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Fri, 2 Jun 2023 11:52:44 -0500 Subject: [PATCH 09/20] Add correct files checks and dataInjection and kustomize checks --- src/extensions/bigbang/test/bigbang_test.go | 6 +++--- src/test/e2e/51_oci_compose_test.go | 10 ++++++---- .../51-import-everything/overrides/kustomization.yaml | 7 +++++++ src/test/packages/51-import-everything/zarf.yaml | 11 +++++++++++ 4 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 src/test/packages/51-import-everything/overrides/kustomization.yaml diff --git a/src/extensions/bigbang/test/bigbang_test.go b/src/extensions/bigbang/test/bigbang_test.go index bb4bbbc6cc..3cbf98c985 100644 --- a/src/extensions/bigbang/test/bigbang_test.go +++ b/src/extensions/bigbang/test/bigbang_test.go @@ -56,9 +56,6 @@ func TestReleases(t *testing.T) { pkgPath := fmt.Sprintf("zarf-package-big-bang-test-amd64-%s.tar.zst", previous) zarfExec(t, "package", "deploy", pkgPath, "--confirm") - // Remove the previous version package - _ = os.RemoveAll(pkgPath) - // HACK: scale down the flux deployments due to very-low CPU in the test runner fluxControllers := []string{"helm-controller", "source-controller", "kustomize-controller", "notification-controller"} for _, deployment := range fluxControllers { @@ -73,6 +70,9 @@ func TestReleases(t *testing.T) { bbMajor = fmt.Sprintf("--set=BB_MAJOR=%s", latest[0:1]) zarfExec(t, "package", "create", "../src/extensions/bigbang/test/package", bbVersion, bbMajor, "--differential", pkgPath, "--confirm") + // Remove the previous version package + _ = os.RemoveAll(pkgPath) + // Clean up zarf cache now that all packages are built to reduce disk pressure zarfExec(t, "tools", "clear-cache") diff --git a/src/test/e2e/51_oci_compose_test.go b/src/test/e2e/51_oci_compose_test.go index c1b1231879..fdfd1d6545 100644 --- a/src/test/e2e/51_oci_compose_test.go +++ b/src/test/e2e/51_oci_compose_test.go @@ -202,20 +202,21 @@ func (suite *SkeletonSuite) verifyComponentPaths(unpackedPath string, components if isSkeleton && utils.IsURL(file.Source) { continue } else if isSkeleton { - suite.FileExists(filepath.Join(componentPaths.Files, file.Source)) + suite.FileExists(filepath.Join(base, file.Source)) continue } - suite.DirOrFileExists(filepath.Join(componentPaths.Files, strconv.Itoa(filesIdx))) + path := filepath.Join(componentPaths.Files, strconv.Itoa(filesIdx), filepath.Base(file.Target)) + suite.DirOrFileExists(path) } for dataIdx, data := range component.DataInjections { if isSkeleton && utils.IsURL(data.Source) { continue } else if isSkeleton { - suite.DirOrFileExists(filepath.Join(componentPaths.DataInjections, data.Source)) + suite.DirOrFileExists(filepath.Join(base, data.Source)) continue } - path := filepath.Join(componentPaths.DataInjections, fmt.Sprintf("injection-%d", dataIdx)) + path := filepath.Join(componentPaths.DataInjections, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) suite.DirOrFileExists(path) } @@ -232,6 +233,7 @@ func (suite *SkeletonSuite) verifyComponentPaths(unpackedPath string, components } suite.FileExists(filepath.Join(componentPaths.Manifests, fmt.Sprintf("%s-%d.yaml", manifest.Name, filesIdx))) } + // TODO: WRONG - UNTESTED DEAD CODE for kustomizeIdx := range manifest.Kustomizations { path := filepath.Join(componentPaths.Manifests, fmt.Sprintf("kustomization-%s-%d.yaml", manifest.Name, kustomizeIdx)) suite.FileExists(path) diff --git a/src/test/packages/51-import-everything/overrides/kustomization.yaml b/src/test/packages/51-import-everything/overrides/kustomization.yaml new file mode 100644 index 0000000000..6b7f66eb9f --- /dev/null +++ b/src/test/packages/51-import-everything/overrides/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +metadata: + name: override + +resources: +- ../../../../examples/composable-package/quake-service.yaml diff --git a/src/test/packages/51-import-everything/zarf.yaml b/src/test/packages/51-import-everything/zarf.yaml index 8c536be486..726b69e8d3 100644 --- a/src/test/packages/51-import-everything/zarf.yaml +++ b/src/test/packages/51-import-everything/zarf.yaml @@ -16,6 +16,10 @@ components: import: path: ../../../../examples/dos-games name: baseline + manifests: + - name: override + kustomizations: + - overrides - name: import-component-wordpress required: false @@ -106,6 +110,13 @@ components: url: oci://ghcr.io/stefanprodan/charts/podinfo images: - ghcr.io/stefanprodan/podinfo:6.3.5 + dataInjections: + - source: files/coffee-ipsum.txt + target: + namespace: podinfo + selector: app.kubernetes.io/name=oci-demo-podinfo + container: podinfo + path: /files/coffee-ipsum.txt actions: onDeploy: after: From f2e064d302021bde59ced7ca2cb0e3f317808867 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Fri, 2 Jun 2023 12:00:03 -0500 Subject: [PATCH 10/20] Make component names clearer --- docs/3-create-a-zarf-package/2-zarf-components.md | 4 ++-- examples/composable-packages/zarf.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/3-create-a-zarf-package/2-zarf-components.md b/docs/3-create-a-zarf-package/2-zarf-components.md index 33fc1075f4..d177f88d22 100644 --- a/docs/3-create-a-zarf-package/2-zarf-components.md +++ b/docs/3-create-a-zarf-package/2-zarf-components.md @@ -164,10 +164,10 @@ The [`podinfo-flux`](/examples/podinfo-flux/) example showcases a simple GitOps - + - + diff --git a/examples/composable-packages/zarf.yaml b/examples/composable-packages/zarf.yaml index 664b8ba2ad..73a219bf12 100644 --- a/examples/composable-packages/zarf.yaml +++ b/examples/composable-packages/zarf.yaml @@ -4,8 +4,7 @@ metadata: description: Demo Zarf package composability components: - # LOCAL COMPOSE IMPORT - - name: games + - name: local-games-path # The component logic keys ('required', 'group', and 'default') always override those of the imported package required: true # group: "" # the initial value overrides the child component @@ -23,8 +22,7 @@ components: files: - quake-service.yaml - # OCI COMPOSE IMPORT - - name: wordpress + - name: oci-wordpress-url # The component logic keys ('required', 'group', and 'default') always override those of the imported package # required: false # the initial value overrides the child component # group: "" # the initial value overrides the child component @@ -32,6 +30,8 @@ components: import: # The URL to the skeleton package containing this component's package definition url: oci://localhost:5000/wordpress:16.0.4-skeleton + # Example optional custom name to point to in the imported package (default is to use this component's name) + name: wordpress # Array keys are appended to the end of anything the imported component defined (in this case an override of the blog name) actions: onDeploy: From b7d1078ae7991215918b7e956ed33464cb499f43 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Fri, 2 Jun 2023 12:31:21 -0500 Subject: [PATCH 11/20] Fix the dot dot --- .../packages/51-import-everything/overrides/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/packages/51-import-everything/overrides/kustomization.yaml b/src/test/packages/51-import-everything/overrides/kustomization.yaml index 6b7f66eb9f..42c019c1f4 100644 --- a/src/test/packages/51-import-everything/overrides/kustomization.yaml +++ b/src/test/packages/51-import-everything/overrides/kustomization.yaml @@ -4,4 +4,4 @@ metadata: name: override resources: -- ../../../../examples/composable-package/quake-service.yaml +- ../../../../../examples/composable-package/quake-service.yaml From 152aae72063bec750c024e347ef2827d1ba12e4b Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Fri, 2 Jun 2023 13:16:22 -0500 Subject: [PATCH 12/20] Add the S --- .../packages/51-import-everything/overrides/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/packages/51-import-everything/overrides/kustomization.yaml b/src/test/packages/51-import-everything/overrides/kustomization.yaml index 42c019c1f4..9224e3e4cd 100644 --- a/src/test/packages/51-import-everything/overrides/kustomization.yaml +++ b/src/test/packages/51-import-everything/overrides/kustomization.yaml @@ -4,4 +4,4 @@ metadata: name: override resources: -- ../../../../../examples/composable-package/quake-service.yaml +- ../../../../../examples/composable-packages/quake-service.yaml From 693fb89ef4378979f4bd09b5e3cd6939f5f626be Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Fri, 2 Jun 2023 13:58:55 -0500 Subject: [PATCH 13/20] Fix kustomize --- .../{overrides => files}/kustomization.yaml | 6 ++++-- src/test/packages/51-import-everything/zarf.yaml | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) rename src/test/packages/51-import-everything/{overrides => files}/kustomization.yaml (54%) diff --git a/src/test/packages/51-import-everything/overrides/kustomization.yaml b/src/test/packages/51-import-everything/files/kustomization.yaml similarity index 54% rename from src/test/packages/51-import-everything/overrides/kustomization.yaml rename to src/test/packages/51-import-everything/files/kustomization.yaml index 9224e3e4cd..5105346887 100644 --- a/src/test/packages/51-import-everything/overrides/kustomization.yaml +++ b/src/test/packages/51-import-everything/files/kustomization.yaml @@ -3,5 +3,7 @@ kind: Kustomization metadata: name: override -resources: -- ../../../../../examples/composable-packages/quake-service.yaml +configMapGenerator: +- name: coffee-ipsum + files: + - coffee-ipsum.txt diff --git a/src/test/packages/51-import-everything/zarf.yaml b/src/test/packages/51-import-everything/zarf.yaml index 726b69e8d3..87b15e9966 100644 --- a/src/test/packages/51-import-everything/zarf.yaml +++ b/src/test/packages/51-import-everything/zarf.yaml @@ -19,7 +19,7 @@ components: manifests: - name: override kustomizations: - - overrides + - files - name: import-component-wordpress required: false From 629b2efc1bbb59149645177d77ddc7c97b6ff39d Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Fri, 2 Jun 2023 14:30:56 -0500 Subject: [PATCH 14/20] Look in the correct place for manifest skeletons --- src/test/e2e/51_oci_compose_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/e2e/51_oci_compose_test.go b/src/test/e2e/51_oci_compose_test.go index fdfd1d6545..53913c7cbb 100644 --- a/src/test/e2e/51_oci_compose_test.go +++ b/src/test/e2e/51_oci_compose_test.go @@ -228,7 +228,7 @@ func (suite *SkeletonSuite) verifyComponentPaths(unpackedPath string, components if isSkeleton && utils.IsURL(path) { continue } else if isSkeleton { - suite.FileExists(filepath.Join(componentPaths.Manifests, path)) + suite.FileExists(filepath.Join(base, path)) continue } suite.FileExists(filepath.Join(componentPaths.Manifests, fmt.Sprintf("%s-%d.yaml", manifest.Name, filesIdx))) From dc4a7fb55c64ef1964d0c69399fbf2bb11f20dc4 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Fri, 2 Jun 2023 15:18:57 -0500 Subject: [PATCH 15/20] Remove TODO --- src/test/e2e/51_oci_compose_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/e2e/51_oci_compose_test.go b/src/test/e2e/51_oci_compose_test.go index 53913c7cbb..5e7a077695 100644 --- a/src/test/e2e/51_oci_compose_test.go +++ b/src/test/e2e/51_oci_compose_test.go @@ -233,7 +233,6 @@ func (suite *SkeletonSuite) verifyComponentPaths(unpackedPath string, components } suite.FileExists(filepath.Join(componentPaths.Manifests, fmt.Sprintf("%s-%d.yaml", manifest.Name, filesIdx))) } - // TODO: WRONG - UNTESTED DEAD CODE for kustomizeIdx := range manifest.Kustomizations { path := filepath.Join(componentPaths.Manifests, fmt.Sprintf("kustomization-%s-%d.yaml", manifest.Name, kustomizeIdx)) suite.FileExists(path) From d0bb367208b65157cd526976d1bce42993175e19 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Fri, 2 Jun 2023 15:30:10 -0500 Subject: [PATCH 16/20] Update src/pkg/packager/publish.go Co-authored-by: razzle --- src/pkg/packager/publish.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index c13765d83d..63143a9676 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -293,7 +293,7 @@ func (p *Packager) loadSkeleton() error { return err } - err = p.skeletonizeExtentions() + err = p.skeletonizeExtensions() if err != nil { return err } From 9ecfe978daa6ea5681dd5ba7fdeb983859b13283 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Fri, 2 Jun 2023 15:30:15 -0500 Subject: [PATCH 17/20] Update src/pkg/packager/compose.go Co-authored-by: razzle --- src/pkg/packager/compose.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index e935a97e7a..fd6c9e5263 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -260,7 +260,7 @@ func (p *Packager) fixComposedFilepaths(pathAncestry string, child types.ZarfCom child.CosignKeyPath = composed } - child = p.composeExtentions(pathAncestry, child) + child = p.composeExtensions(pathAncestry, child) return child, nil } From bbebf784844f3c713fdb0ca96be19774996dc9eb Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Fri, 2 Jun 2023 15:32:23 -0500 Subject: [PATCH 18/20] Update src/pkg/packager/extensions.go Co-authored-by: razzle --- src/pkg/packager/extensions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/extensions.go b/src/pkg/packager/extensions.go index 9190581bca..c4ebe09a45 100644 --- a/src/pkg/packager/extensions.go +++ b/src/pkg/packager/extensions.go @@ -55,7 +55,7 @@ func (p *Packager) composeExtentions(pathAncestry string, component types.ZarfCo } // Check for any extensions in use and skeletonize their local files. -func (p *Packager) skeletonizeExtentions() error { +func (p *Packager) skeletonizeExtensions() error { message.Debugf("packager.skeletonizeExtentions()") components := []types.ZarfComponent{} From 415832d436ad7d175b9292a8a8bda9691e69d32f Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Fri, 2 Jun 2023 15:32:32 -0500 Subject: [PATCH 19/20] Update src/pkg/packager/extensions.go Co-authored-by: razzle --- src/pkg/packager/extensions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/extensions.go b/src/pkg/packager/extensions.go index c4ebe09a45..7228123469 100644 --- a/src/pkg/packager/extensions.go +++ b/src/pkg/packager/extensions.go @@ -43,7 +43,7 @@ func (p *Packager) processExtensions() error { } // Mutate any local files to be relative to the parent -func (p *Packager) composeExtentions(pathAncestry string, component types.ZarfComponent) types.ZarfComponent { +func (p *Packager) composeExtensions(pathAncestry string, component types.ZarfComponent) types.ZarfComponent { message.Debugf("packager.composeExtentions()") // Big Bang From 0f42339e26e4dffad600bf3117534e104a0d448e Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Fri, 2 Jun 2023 18:31:18 -0500 Subject: [PATCH 20/20] Fix spacing --- src/pkg/packager/publish.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 63143a9676..20bffa76c1 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -296,7 +296,7 @@ func (p *Packager) loadSkeleton() error { err = p.skeletonizeExtensions() if err != nil { return err - } + } for _, warning := range p.warnings { message.Warn(warning)