#208, #351. UPDATE Makefile: add package-example-compose for e2e test…

…s. Update .github/workflows/test*: add package-compose-example to make packages stage. UPDATE packager/compose: rename GetComposedAssets to GetComposedComponents UPDATE reference in packager/create. refactored and rename hasValidSubPackage to validateOrBail. Rename shouldAddImportedPackage to shouldComposePackage. Extracted component required logic to componentConfirmedForInclusion. UPDATE e2e/e2e_composability_test: to work with new testing strategy.

.github/workflows/test-k3d.yml

@@ -15,6 +15,6 @@ jobs:
       - name: Build CLI
         run: make build-cli-linux
       - name: Make Packages
-        run: make init-package package-example-game package-example-data-injection package-example-gitops-data
+        run: make init-package package-example-game package-example-data-injection package-example-gitops-data package-example-compose
       - name: Run Tests
         run: TESTDISTRO=k3d make test-e2e

.github/workflows/test-k3s.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Build CLI
         run: make build-cli-linux
       - name: Make Packages
-        run: make init-package package-example-game package-example-data-injection package-example-gitops-data
+        run: make init-package package-example-game package-example-data-injection package-example-gitops-data package-example-compose
       - name: Run Tests
         # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of go installed in a previous step
         run: sudo env "PATH=$PATH" TESTDISTRO=k3s make test-e2e

.github/workflows/test-kind.yml

@@ -15,6 +15,6 @@ jobs:
       - name: Build CLI
         run: make build-cli-linux
       - name: Make Packages
-        run: make init-package package-example-game package-example-data-injection package-example-gitops-data
+        run: make init-package package-example-game package-example-data-injection package-example-gitops-data package-example-compose
       - name: Run Tests
         run: TESTDISTRO=kind make test-e2e

Makefile

@@ -78,6 +78,10 @@ package-example-gitops-data:
 package-example-tiny-kafka:
 	cd examples/tiny-kafka && ../../$(ZARF_BIN) package create --confirm && mv zarf-package-* ../../build/
 
+.PHONY: package-example-compose
+package-example-compose:
+	cd examples/composable-packages && ../../$(ZARF_BIN) package create --confirm && mv zarf-package-* ../../build/
+
 # TODO: This can be cleaned up a little more when `zarf init` is able to provide the path to the `zarf-init.tar.zst`
 .PHONY: test-new-e2e
 test-e2e: ## Run e2e tests on a KiND cluster. All dependencies are assumed to be built and in the ./build directory
@@ -97,5 +101,7 @@ test-e2e: ## Run e2e tests on a KiND cluster. All dependencies are assumed to be
 	@if [ ! -f ./build/zarf-package-gitops-service-data.tar.zst ]; then\
 		$(MAKE) package-example-gitops-data;\
 	fi
-
+	@if [ ! -f ./build/zarf-package-compose-example.tar.zst ]; then\
+		$(MAKE) package-example-compose;\
+	fi
 	cd test/e2e && cp ../../build/zarf-init.tar.zst . && go test ./... -v -timeout 2400s && rm zarf-init.tar.zst

cli/internal/packager/compose.go

@@ -8,15 +8,16 @@ import (
 	"github.com/defenseunicorns/zarf/cli/types"
 )
 
-func GetComposedAssets() (components []types.ZarfComponent) {
+func GetComposedComponents() (components []types.ZarfComponent) {
 	for _, component := range config.GetComponents() {
-		// Build components list by expanding imported components.
-		if shouldAddImportedPackage(&component) {
+		// Check for standard component.
+		if !hasComposedPackage(&component) {
+			// Append standard component to list.
+			components = append(components, component)
+		} else if shouldComposePackage(&component) { // Validate and confirm inclusion of imported package.
+			// Expand and add components from imported package.
 			importedComponents := getSubPackageAssets(component)
 			components = append(components, importedComponents...)
-
-		} else if !hasSubPackage(&component) {
-			components = append(components, component)
 		}
 	}
 	// Update the parent package config with the expanded sub components.
@@ -25,39 +26,46 @@ func GetComposedAssets() (components []types.ZarfComponent) {
 	return components
 }
 
-// Get the sub package components to add to parent assets, recurses on sub imports.
-func getSubPackageAssets(importComponent types.ZarfComponent) (components []types.ZarfComponent) {
-	importedPackage := getSubPackage(&importComponent)
-	for _, componentToCompose := range importedPackage.Components {
-		if shouldAddImportedPackage(&componentToCompose) {
-			components = append(components, getSubPackageAssets(componentToCompose)...)
-		} else if !hasSubPackage(&componentToCompose) {
-			prepComponentToCompose(&componentToCompose, importedPackage.Metadata.Name, importComponent.Import.Path)
-			components = append(components, componentToCompose)
-		}
-	}
-	return components
+// Returns true if import field is populated.
+func hasComposedPackage(component *types.ZarfComponent) bool {
+	return component.Import != types.ZarfImport{}
 }
 
-// Confirms inclusion of SubPackage. Need team input.
-func shouldAddImportedPackage(component *types.ZarfComponent) bool {
-	return hasValidSubPackage(component) && (config.DeployOptions.Confirm || component.Required || ConfirmOptionalComponent(*component))
+// Validates and confirms inclusion of imported package.
+func shouldComposePackage(component *types.ZarfComponent) bool {
+	validateOrBail(component)
+	return componentConfirmedForInclusion(component)
 }
 
-// Validates the sub component, errors out if validation fails.
-func hasValidSubPackage(component *types.ZarfComponent) bool {
-	if !hasSubPackage(component) {
-		return false
-	}
+// Returns true if confirm flag is true, the component is required, or the user confirms inclusion.
+func componentConfirmedForInclusion(component *types.ZarfComponent) bool {
+	return config.DeployOptions.Confirm || component.Required || ConfirmOptionalComponent(*component)
+}
+
+// Validates the sub component, exits program if validation fails.
+func validateOrBail(component *types.ZarfComponent) {
 	if err := validate.ValidateImportPackage(component); err != nil {
 		message.Fatalf(err, "Invalid import definition in the %s component: %s", component.Name, err)
 	}
-	return true
 }
 
-// returns true if import field is populated
-func hasSubPackage(component *types.ZarfComponent) bool {
-	return component.Import != types.ZarfImport{}
+// Get expanded components from imported component.
+func getSubPackageAssets(importComponent types.ZarfComponent) (components []types.ZarfComponent) {
+	// Read the imported package.
+	importedPackage := getSubPackage(&importComponent)
+	// Iterate imported components.
+	for _, componentToCompose := range importedPackage.Components {
+		// Check for standard component.
+		if !hasComposedPackage(&componentToCompose) {
+			// Doctor standard component name and included files.
+			prepComponentToCompose(&componentToCompose, importedPackage.Metadata.Name, importComponent.Import.Path)
+			components = append(components, componentToCompose)
+		} else if shouldComposePackage(&componentToCompose) {
+			// Recurse on imported components.
+			components = append(components, getSubPackageAssets(componentToCompose)...)
+		}
+	}
+	return components
 }
 
 // Reads the locally imported zarf.yaml
@@ -68,35 +76,38 @@ func getSubPackage(component *types.ZarfComponent) (importedPackage types.ZarfPa
 
 // Updates the name and sets all local asset paths relative to the importing package.
 func prepComponentToCompose(component *types.ZarfComponent, parentPackageName string, importPath string) {
+	// Prefix component name with parent package name to distinguish similarly named components.
 	component.Name = parentPackageName + "-" + component.Name
 
-	// Add import path to local component files.
+	// Prefix composed component file paths.
 	for fileIdx, file := range component.Files {
-		if !utils.IsUrl(file.Source) {
-			component.Files[fileIdx].Source = importPath + file.Source
-		}
+		component.Files[fileIdx].Source = getComposedFilePath(file.Source, importPath)
 	}
 
-	// Add import path to local chart values files.
+	// Prefix non-url composed component chart values files.
 	for chartIdx, chart := range component.Charts {
 		for valuesIdx, valuesFile := range chart.ValuesFiles {
-			if !utils.IsUrl(valuesFile) {
-				component.Charts[chartIdx].ValuesFiles[valuesIdx] = importPath + valuesFile
-			}
+			component.Charts[chartIdx].ValuesFiles[valuesIdx] = getComposedFilePath(valuesFile, importPath)
 		}
 	}
 
-	// Add import path to local manifest files and kustomizations
+	// Prefix non-url composed manifest files and kustomizations.
 	for manifestIdx, manifest := range component.Manifests {
 		for fileIdx, file := range manifest.Files {
-			if !utils.IsUrl(file) {
-				component.Manifests[manifestIdx].Files[fileIdx] = importPath + file
-			}
+			component.Manifests[manifestIdx].Files[fileIdx] = getComposedFilePath(file, importPath)
 		}
 		for kustomIdx, kustomization := range manifest.Kustomizations {
-			if !utils.IsUrl(kustomization) {
-				component.Manifests[manifestIdx].Kustomizations[kustomIdx] = importPath + kustomization
-			}
+			component.Manifests[manifestIdx].Kustomizations[kustomIdx] = getComposedFilePath(kustomization, importPath)
 		}
 	}
 }
+
+// Prefix file path with importPath if original file path is not a url.
+func getComposedFilePath(originalPath string, pathPrefix string) string {
+	// Return original if it is a remote file.
+	if utils.IsUrl(originalPath) {
+		return originalPath
+	}
+	// Add prefix for local files.
+	return pathPrefix + originalPath
+}

cli/internal/packager/create.go

@@ -30,7 +30,7 @@ func Create() {
 	tempPath := createPaths()
 	defer tempPath.clean()
 
-	components := GetComposedAssets()
+	components := GetComposedComponents()
 	seedImages := config.GetSeedImages()
 	packageName := config.GetPackageName()
 	dataInjections := config.GetDataInjections()

test/e2e/e2e_composability_test.go

@@ -1,57 +1,41 @@
 package test
 
 import (
-	"fmt"
+	"io"
+	"net/http"
 	"testing"
-	"time"
 
-	teststructure "github.com/gruntwork-io/terratest/modules/test-structure"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestE2eExampleComposability(t *testing.T) {
+	//run `zarf init`
+	output, err := e2e.execZarfCommand("init", "--confirm")
+	require.NoError(t, err, output)
 
-	e2e := NewE2ETest(t)
+	// Deploy the composable game package
+	output, err = e2e.execZarfCommand("package", "deploy", "../../build/zarf-package-compose-example.tar.zst", "--confirm")
+	require.NoError(t, err, output)
 
-	// At the end of the test, run `terraform destroy` to clean up any resources that were created
-	defer teststructure.RunTestStage(e2e.testing, "TEARDOWN", e2e.teardown)
+	// Validate that the composed sub packages exist
+	require.Contains(t, output, "appliance-demo-multi-games-baseline")
 
-	// Upload the Zarf artifacts
-	teststructure.RunTestStage(e2e.testing, "UPLOAD", func() {
-		e2e.syncFileToRemoteServer("../../build/zarf", fmt.Sprintf("/home/%s/build/zarf", e2e.username), "0700")
-		e2e.syncFileToRemoteServer("../../build/zarf-init.tar.zst", fmt.Sprintf("/home/%s/build/zarf-init.tar.zst", e2e.username), "0600")
-		e2e.syncFileToRemoteServer("../../build/zarf-package-compose-example.tar.zst", fmt.Sprintf("/home/%s/build/zarf-package-compose-example.tar.zst", e2e.username), "0600")
-	})
+	// Establish the port-forward into the game service
+	err = e2e.execZarfBackgroundCommand("connect", "doom", "--local-port=22333")
+	require.NoError(t, err, "unable to connect to the doom port-forward")
 
-	teststructure.RunTestStage(e2e.testing, "TEST", func() {
-		// Make sure `zarf --help` doesn't error
-		output, err := e2e.runSSHCommand("sudo /home/%s/build/zarf --help", e2e.username)
-		require.NoError(e2e.testing, err, output)
+	// Right now we're just checking that `curl` returns 0. It can be enhanced by scraping the HTML that gets returned or something.
+	resp, err := http.Get("http://127.0.0.1:22333?doom")
+	assert.NoError(t, err, resp)
 
-		// run `zarf init`
-		output, err = e2e.runSSHCommand("sudo bash -c 'cd /home/%s/build && ./zarf init --confirm --components k3s'", e2e.username)
-		require.NoError(e2e.testing, err, output)
+	// Read the body into string
+	body, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err, body)
 
-		// Deploy the composable package
-		output, err = e2e.runSSHCommand("sudo bash -c 'cd /home/%s/build && ./zarf package deploy zarf-package-composable-example.tar.zst --confirm'", e2e.username)
-		require.NoError(e2e.testing, err, output)
-
-		// Validate that the composed sub packages exist
-		require.Contains(e2e.testing, "appliance-demo-multi-games-baseline", output)
-
-		// Establish the port-forward into the game service; give the service a few seconds to come up since this is not a command we can retry
-		time.Sleep(5 * time.Second)
-		output, err = e2e.runSSHCommand("sudo bash -c '(/home/%s/build/zarf connect doom --local-port 22333 &> /dev/nul &)'", e2e.username)
-		require.NoError(e2e.testing, err, output)
-
-		// Right now we're just checking that `curl` returns 0. It can be enhanced by scraping the HTML that gets returned or something.
-		output, err = e2e.runSSHCommand("bash -c '[[ $(curl -sfSL --retry 15 --retry-connrefused --retry-delay 5 http://127.0.0.1:22333?doom') ]]'")
-		require.NoError(e2e.testing, err, output)
-		require.Contains(e2e.testing, "Zarf needs games too", output)
-
-		// Run `zarf destroy` to make sure that works correctly
-		output, err = e2e.runSSHCommand("sudo bash -c 'cd /home/%s/build && ./zarf destroy --confirm'", e2e.username)
-		require.NoError(e2e.testing, err, output)
-	})
+	// Validate the doom title in body.
+	assert.Contains(t, string(body), "Zarf needs games too")
+	assert.Equal(t, 200, resp.StatusCode)
 
+	e2e.cleanupAfterTest(t)
 }