From f27d0f2b92d226c5a4fcadd578b64de86b3e1371 Mon Sep 17 00:00:00 2001 From: Michael Winberry Date: Wed, 9 Mar 2022 10:45:41 -0800 Subject: [PATCH] #208, #351. UPDATE Makefile: add package-example-compose for e2e tests. Update .github/workflows/test*: add package-compose-example to make packages stage. UPDATE packager/compose: rename GetComposedAssets to GetComposedComponents UPDATE reference in packager/create. refactored and rename hasValidSubPackage to validateOrBail. Rename shouldAddImportedPackage to shouldComposePackage. Extracted component required logic to componentConfirmedForInclusion. UPDATE e2e/e2e_composability_test: to work with new testing strategy. --- .github/workflows/test-k3d.yml | 2 +- .github/workflows/test-k3s.yml | 2 +- .github/workflows/test-kind.yml | 2 +- Makefile | 8 ++- cli/internal/packager/compose.go | 101 ++++++++++++++++------------- cli/internal/packager/create.go | 2 +- test/e2e/e2e_composability_test.go | 64 +++++++----------- 7 files changed, 91 insertions(+), 90 deletions(-) diff --git a/.github/workflows/test-k3d.yml b/.github/workflows/test-k3d.yml index ec739e4a46..26833fc65c 100644 --- a/.github/workflows/test-k3d.yml +++ b/.github/workflows/test-k3d.yml @@ -15,6 +15,6 @@ jobs: - name: Build CLI run: make build-cli-linux - name: Make Packages - run: make init-package package-example-game package-example-data-injection package-example-gitops-data + run: make init-package package-example-game package-example-data-injection package-example-gitops-data package-example-compose - name: Run Tests run: TESTDISTRO=k3d make test-e2e diff --git a/.github/workflows/test-k3s.yml b/.github/workflows/test-k3s.yml index b2a53f57c8..d2f129744a 100644 --- a/.github/workflows/test-k3s.yml +++ b/.github/workflows/test-k3s.yml @@ -15,7 +15,7 @@ jobs: - name: Build CLI run: make build-cli-linux - name: Make Packages - run: make init-package package-example-game package-example-data-injection package-example-gitops-data + run: make init-package package-example-game package-example-data-injection package-example-gitops-data package-example-compose - name: Run Tests # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of go installed in a previous step run: sudo env "PATH=$PATH" TESTDISTRO=k3s make test-e2e diff --git a/.github/workflows/test-kind.yml b/.github/workflows/test-kind.yml index c0ba17cfdf..6ccc5b0b2e 100644 --- a/.github/workflows/test-kind.yml +++ b/.github/workflows/test-kind.yml @@ -15,6 +15,6 @@ jobs: - name: Build CLI run: make build-cli-linux - name: Make Packages - run: make init-package package-example-game package-example-data-injection package-example-gitops-data + run: make init-package package-example-game package-example-data-injection package-example-gitops-data package-example-compose - name: Run Tests run: TESTDISTRO=kind make test-e2e diff --git a/Makefile b/Makefile index edf779ba0e..cb25b6e4d7 100644 --- a/Makefile +++ b/Makefile @@ -78,6 +78,10 @@ package-example-gitops-data: package-example-tiny-kafka: cd examples/tiny-kafka && ../../$(ZARF_BIN) package create --confirm && mv zarf-package-* ../../build/ +.PHONY: package-example-compose +package-example-compose: + cd examples/composable-packages && ../../$(ZARF_BIN) package create --confirm && mv zarf-package-* ../../build/ + # TODO: This can be cleaned up a little more when `zarf init` is able to provide the path to the `zarf-init.tar.zst` .PHONY: test-new-e2e test-e2e: ## Run e2e tests on a KiND cluster. All dependencies are assumed to be built and in the ./build directory @@ -97,5 +101,7 @@ test-e2e: ## Run e2e tests on a KiND cluster. All dependencies are assumed to be @if [ ! -f ./build/zarf-package-gitops-service-data.tar.zst ]; then\ $(MAKE) package-example-gitops-data;\ fi - + @if [ ! -f ./build/zarf-package-compose-example.tar.zst ]; then\ + $(MAKE) package-example-compose;\ + fi cd test/e2e && cp ../../build/zarf-init.tar.zst . && go test ./... -v -timeout 2400s && rm zarf-init.tar.zst diff --git a/cli/internal/packager/compose.go b/cli/internal/packager/compose.go index 1aa43c6452..86c5858d6e 100644 --- a/cli/internal/packager/compose.go +++ b/cli/internal/packager/compose.go @@ -8,15 +8,16 @@ import ( "github.com/defenseunicorns/zarf/cli/types" ) -func GetComposedAssets() (components []types.ZarfComponent) { +func GetComposedComponents() (components []types.ZarfComponent) { for _, component := range config.GetComponents() { - // Build components list by expanding imported components. - if shouldAddImportedPackage(&component) { + // Check for standard component. + if !hasComposedPackage(&component) { + // Append standard component to list. + components = append(components, component) + } else if shouldComposePackage(&component) { // Validate and confirm inclusion of imported package. + // Expand and add components from imported package. importedComponents := getSubPackageAssets(component) components = append(components, importedComponents...) - - } else if !hasSubPackage(&component) { - components = append(components, component) } } // Update the parent package config with the expanded sub components. @@ -25,39 +26,46 @@ func GetComposedAssets() (components []types.ZarfComponent) { return components } -// Get the sub package components to add to parent assets, recurses on sub imports. -func getSubPackageAssets(importComponent types.ZarfComponent) (components []types.ZarfComponent) { - importedPackage := getSubPackage(&importComponent) - for _, componentToCompose := range importedPackage.Components { - if shouldAddImportedPackage(&componentToCompose) { - components = append(components, getSubPackageAssets(componentToCompose)...) - } else if !hasSubPackage(&componentToCompose) { - prepComponentToCompose(&componentToCompose, importedPackage.Metadata.Name, importComponent.Import.Path) - components = append(components, componentToCompose) - } - } - return components +// Returns true if import field is populated. +func hasComposedPackage(component *types.ZarfComponent) bool { + return component.Import != types.ZarfImport{} } -// Confirms inclusion of SubPackage. Need team input. -func shouldAddImportedPackage(component *types.ZarfComponent) bool { - return hasValidSubPackage(component) && (config.DeployOptions.Confirm || component.Required || ConfirmOptionalComponent(*component)) +// Validates and confirms inclusion of imported package. +func shouldComposePackage(component *types.ZarfComponent) bool { + validateOrBail(component) + return componentConfirmedForInclusion(component) } -// Validates the sub component, errors out if validation fails. -func hasValidSubPackage(component *types.ZarfComponent) bool { - if !hasSubPackage(component) { - return false - } +// Returns true if confirm flag is true, the component is required, or the user confirms inclusion. +func componentConfirmedForInclusion(component *types.ZarfComponent) bool { + return config.DeployOptions.Confirm || component.Required || ConfirmOptionalComponent(*component) +} + +// Validates the sub component, exits program if validation fails. +func validateOrBail(component *types.ZarfComponent) { if err := validate.ValidateImportPackage(component); err != nil { message.Fatalf(err, "Invalid import definition in the %s component: %s", component.Name, err) } - return true } -// returns true if import field is populated -func hasSubPackage(component *types.ZarfComponent) bool { - return component.Import != types.ZarfImport{} +// Get expanded components from imported component. +func getSubPackageAssets(importComponent types.ZarfComponent) (components []types.ZarfComponent) { + // Read the imported package. + importedPackage := getSubPackage(&importComponent) + // Iterate imported components. + for _, componentToCompose := range importedPackage.Components { + // Check for standard component. + if !hasComposedPackage(&componentToCompose) { + // Doctor standard component name and included files. + prepComponentToCompose(&componentToCompose, importedPackage.Metadata.Name, importComponent.Import.Path) + components = append(components, componentToCompose) + } else if shouldComposePackage(&componentToCompose) { + // Recurse on imported components. + components = append(components, getSubPackageAssets(componentToCompose)...) + } + } + return components } // Reads the locally imported zarf.yaml @@ -68,35 +76,38 @@ func getSubPackage(component *types.ZarfComponent) (importedPackage types.ZarfPa // Updates the name and sets all local asset paths relative to the importing package. func prepComponentToCompose(component *types.ZarfComponent, parentPackageName string, importPath string) { + // Prefix component name with parent package name to distinguish similarly named components. component.Name = parentPackageName + "-" + component.Name - // Add import path to local component files. + // Prefix composed component file paths. for fileIdx, file := range component.Files { - if !utils.IsUrl(file.Source) { - component.Files[fileIdx].Source = importPath + file.Source - } + component.Files[fileIdx].Source = getComposedFilePath(file.Source, importPath) } - // Add import path to local chart values files. + // Prefix non-url composed component chart values files. for chartIdx, chart := range component.Charts { for valuesIdx, valuesFile := range chart.ValuesFiles { - if !utils.IsUrl(valuesFile) { - component.Charts[chartIdx].ValuesFiles[valuesIdx] = importPath + valuesFile - } + component.Charts[chartIdx].ValuesFiles[valuesIdx] = getComposedFilePath(valuesFile, importPath) } } - // Add import path to local manifest files and kustomizations + // Prefix non-url composed manifest files and kustomizations. for manifestIdx, manifest := range component.Manifests { for fileIdx, file := range manifest.Files { - if !utils.IsUrl(file) { - component.Manifests[manifestIdx].Files[fileIdx] = importPath + file - } + component.Manifests[manifestIdx].Files[fileIdx] = getComposedFilePath(file, importPath) } for kustomIdx, kustomization := range manifest.Kustomizations { - if !utils.IsUrl(kustomization) { - component.Manifests[manifestIdx].Kustomizations[kustomIdx] = importPath + kustomization - } + component.Manifests[manifestIdx].Kustomizations[kustomIdx] = getComposedFilePath(kustomization, importPath) } } } + +// Prefix file path with importPath if original file path is not a url. +func getComposedFilePath(originalPath string, pathPrefix string) string { + // Return original if it is a remote file. + if utils.IsUrl(originalPath) { + return originalPath + } + // Add prefix for local files. + return pathPrefix + originalPath +} diff --git a/cli/internal/packager/create.go b/cli/internal/packager/create.go index c092e0ebe5..12e0b84095 100644 --- a/cli/internal/packager/create.go +++ b/cli/internal/packager/create.go @@ -30,7 +30,7 @@ func Create() { tempPath := createPaths() defer tempPath.clean() - components := GetComposedAssets() + components := GetComposedComponents() seedImages := config.GetSeedImages() packageName := config.GetPackageName() dataInjections := config.GetDataInjections() diff --git a/test/e2e/e2e_composability_test.go b/test/e2e/e2e_composability_test.go index ddaa444917..74cf1513c1 100644 --- a/test/e2e/e2e_composability_test.go +++ b/test/e2e/e2e_composability_test.go @@ -1,57 +1,41 @@ package test import ( - "fmt" + "io" + "net/http" "testing" - "time" - teststructure "github.com/gruntwork-io/terratest/modules/test-structure" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestE2eExampleComposability(t *testing.T) { + //run `zarf init` + output, err := e2e.execZarfCommand("init", "--confirm") + require.NoError(t, err, output) - e2e := NewE2ETest(t) + // Deploy the composable game package + output, err = e2e.execZarfCommand("package", "deploy", "../../build/zarf-package-compose-example.tar.zst", "--confirm") + require.NoError(t, err, output) - // At the end of the test, run `terraform destroy` to clean up any resources that were created - defer teststructure.RunTestStage(e2e.testing, "TEARDOWN", e2e.teardown) + // Validate that the composed sub packages exist + require.Contains(t, output, "appliance-demo-multi-games-baseline") - // Upload the Zarf artifacts - teststructure.RunTestStage(e2e.testing, "UPLOAD", func() { - e2e.syncFileToRemoteServer("../../build/zarf", fmt.Sprintf("/home/%s/build/zarf", e2e.username), "0700") - e2e.syncFileToRemoteServer("../../build/zarf-init.tar.zst", fmt.Sprintf("/home/%s/build/zarf-init.tar.zst", e2e.username), "0600") - e2e.syncFileToRemoteServer("../../build/zarf-package-compose-example.tar.zst", fmt.Sprintf("/home/%s/build/zarf-package-compose-example.tar.zst", e2e.username), "0600") - }) + // Establish the port-forward into the game service + err = e2e.execZarfBackgroundCommand("connect", "doom", "--local-port=22333") + require.NoError(t, err, "unable to connect to the doom port-forward") - teststructure.RunTestStage(e2e.testing, "TEST", func() { - // Make sure `zarf --help` doesn't error - output, err := e2e.runSSHCommand("sudo /home/%s/build/zarf --help", e2e.username) - require.NoError(e2e.testing, err, output) + // Right now we're just checking that `curl` returns 0. It can be enhanced by scraping the HTML that gets returned or something. + resp, err := http.Get("http://127.0.0.1:22333?doom") + assert.NoError(t, err, resp) - // run `zarf init` - output, err = e2e.runSSHCommand("sudo bash -c 'cd /home/%s/build && ./zarf init --confirm --components k3s'", e2e.username) - require.NoError(e2e.testing, err, output) + // Read the body into string + body, err := io.ReadAll(resp.Body) + assert.NoError(t, err, body) - // Deploy the composable package - output, err = e2e.runSSHCommand("sudo bash -c 'cd /home/%s/build && ./zarf package deploy zarf-package-composable-example.tar.zst --confirm'", e2e.username) - require.NoError(e2e.testing, err, output) - - // Validate that the composed sub packages exist - require.Contains(e2e.testing, "appliance-demo-multi-games-baseline", output) - - // Establish the port-forward into the game service; give the service a few seconds to come up since this is not a command we can retry - time.Sleep(5 * time.Second) - output, err = e2e.runSSHCommand("sudo bash -c '(/home/%s/build/zarf connect doom --local-port 22333 &> /dev/nul &)'", e2e.username) - require.NoError(e2e.testing, err, output) - - // Right now we're just checking that `curl` returns 0. It can be enhanced by scraping the HTML that gets returned or something. - output, err = e2e.runSSHCommand("bash -c '[[ $(curl -sfSL --retry 15 --retry-connrefused --retry-delay 5 http://127.0.0.1:22333?doom') ]]'") - require.NoError(e2e.testing, err, output) - require.Contains(e2e.testing, "Zarf needs games too", output) - - // Run `zarf destroy` to make sure that works correctly - output, err = e2e.runSSHCommand("sudo bash -c 'cd /home/%s/build && ./zarf destroy --confirm'", e2e.username) - require.NoError(e2e.testing, err, output) - }) + // Validate the doom title in body. + assert.Contains(t, string(body), "Zarf needs games too") + assert.Equal(t, 200, resp.StatusCode) + e2e.cleanupAfterTest(t) }