Simplify cluster instantiation in tests (#407)

.github/workflows/test-k3d.yml

@@ -6,15 +6,16 @@ jobs:
   build-and-test:
     runs-on: ubuntu-latest
     steps:
-      - name: Install GoLang
+      - name: "Install GoLang"
         uses: actions/setup-go@v2
         with:
           go-version: 1.16.x
-      - name: Checkout Repo
+      - name: "Checkout Repo"
         uses: actions/checkout@v2
-      - name: Build CLI
+      - name: "Build CLI"
         run: make build-cli-linux
-      - name: Make Packages
+      - name: "Make Packages"
         run: make init-package package-example-game package-example-data-injection package-example-gitops-data package-example-compose
-      - name: Run Tests
+      - name: "Run Tests"
+        # NOTE: This test run will create its own K3d cluster. A single cluster will be used throughout the test run.
         run: TESTDISTRO=k3d make test-e2e

.github/workflows/test-k3s.yml

@@ -6,16 +6,18 @@ jobs:
   build-and-test:
     runs-on: ubuntu-latest
     steps:
-      - name: Install GoLang
+      - name: "Install GoLang"
         uses: actions/setup-go@v2
         with:
           go-version: 1.16.x
-      - name: Checkout Repo
+      - name: "Checkout Repo"
         uses: actions/checkout@v2
-      - name: Build CLI
+      - name: "Build CLI"
         run: make build-cli-linux
-      - name: Make Packages
+      - name: "Make Packages"
         run: make init-package package-example-game package-example-data-injection package-example-gitops-data package-example-compose
-      - name: Run Tests
-        # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of go installed in a previous step
+      - name: "Run Tests"
+        # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of go installed
+        #       in a previous step. This test run will use Zarf to create a K3s cluster, and a brand new cluster will be
+        #       used for each test
         run: sudo env "PATH=$PATH" TESTDISTRO=k3s make test-e2e

.github/workflows/test-kind.yml

@@ -6,15 +6,20 @@ jobs:
   build-and-test:
     runs-on: ubuntu-latest
     steps:
-      - name: Install GoLang
+      - name: "Install GoLang"
         uses: actions/setup-go@v2
         with:
           go-version: 1.16.x
-      - name: Checkout Repo
+      - name: "Checkout Repo"
         uses: actions/checkout@v2
-      - name: Build CLI
+      - name: "Create k8s Kind Cluster"
+        uses: helm/kind-action@v1.2.0
+      - name: "Build CLI"
         run: make build-cli-linux
-      - name: Make Packages
+      - name: "Make Packages"
         run: make init-package package-example-game package-example-data-injection package-example-gitops-data package-example-compose
-      - name: Run Tests
-        run: TESTDISTRO=kind make test-e2e
+      - name: "Run Tests"
+        # NOTE: We want to test providing a cluster to the test framework so this one creates its own KinD cluster
+        #       rather than having the test suite do it. The K3d tests do a self-provisioned cluster and the K3s tests
+        #       use Zarf to create the cluster. In this test a single cluster will be used throughout the test run.
+        run: TESTDISTRO=provided make test-e2e

.golangci.yml

@@ -1,13 +1,17 @@
 run:
   timeout: 5m
   skip-files:
-    - cli/internal/message/logo.go  
+    - cli/internal/message/logo.go
 linters:
   enable-all: true
   disable:
     - exhaustivestruct
+    - goerr113
+    - gofumpt
     - lll
     - stylecheck
+    - testpackage
+    - varnamelen
     - wrapcheck
     - wsl
 issues:

Makefile

@@ -86,9 +86,9 @@ package-example-compose:
 	cd examples/composable-packages && ../../$(ZARF_BIN) package create --confirm && mv zarf-package-* ../../build/
 
 # TODO: This can be cleaned up a little more when `zarf init` is able to provide the path to the `zarf-init-<arch>.tar.zst`
-.PHONY: test-new-e2e
-test-e2e: ## Run e2e tests on a KiND cluster. All dependencies are assumed to be built and in the ./build directory
-	@ #Check to make sure all the packages we need exist
+.PHONY: test-e2e
+test-e2e: ## Run e2e tests. Will automatically build any required dependencies that aren't present. Requires env var TESTDISTRO=[provided|kind|k3d|k3s]
+	@#Check to make sure all the packages we need exist
 	@if [ ! -f $(ZARF_BIN) ]; then\
 		$(MAKE) build-cli;\
 	fi
@@ -107,4 +107,4 @@ test-e2e: ## Run e2e tests on a KiND cluster. All dependencies are assumed to be
 	@if [ ! -f ./build/zarf-package-compose-example-$(ARCH).tar.zst ]; then\
 		$(MAKE) package-example-compose;\
 	fi
-	cd test/e2e && cp ../../build/zarf-init-$(ARCH).tar.zst . && go test ./... -v -timeout 2400s && rm zarf-init-$(ARCH).tar.zst
+	cd test/e2e && cp ../../build/zarf-init-$(ARCH).tar.zst . && go test ./... -v -count=1 -timeout 2400s && rm zarf-init-$(ARCH).tar.zst

test/e2e/README.md

@@ -1,40 +1,50 @@
 # Zarf End-To-End Tests
 
-This directory holds all of our e2e tests that we use to verify Zarf functionality in an environment that replicates a live setting. The tests in this directory are automatically run against all the default K8s distros whenever a PR is opened against the repo. 
+This directory holds all of our e2e tests that we use to verify Zarf functionality in an environment that replicates a live setting. The tests in this directory are automatically run against several K8s distros whenever a PR is opened or updated.
 
-
-# Running Tests Locally
+## Running Tests Locally
 The tests in this directory are also able to be run locally!
 
-## Dependencies
+### Dependencies
 Running the tests locally have the same prerequisites as running and building Zarf:
  1. GoLang >= `1.16.x`
  2. Make
- 3. Docker 
-
+ 3. (for the `kind` and `k3d` options only) Docker
+ 4. (for the `k3s` cluster only) Linux with root privileges
 
 ### Existing K8s Cluster
-If you have a cluster already running, great! As long as your kubeconfig is accessible at `~/.kube/.config` or is exposed in your `$KUBECONFIG` environment variable, the e2e tests will be able to use your cluster to run your tests. This means that you can even test remote distros like EKS, AKS, and GKE. When the tests are completed, your cluster will still be accessible but note that since the tests are actively using your cluster it might be in a very different state.
+If you have a cluster already running, and use the env var `TESTDISTRO=provided`, the test suite will use the `KUBECONFIG` env var and the cluster that is currently configured as the active context. To make sure you are running in the right cluster, run something like `kubectl get nodes` with no optional flags and if the nodes that appear are the ones you expect, then the tests will use that cluster as well.
 
+This means that you are able to run the tests against any remote distro you want, like EKS, AKS, GKE, RKE, etc.
 
 ### No Existing K8s Cluster
-If you do not have a local cluster running, no worries! The e2e tests use the `sigs.k8s.io/kind` and `github.com/rancher/k3d/v5` libraries to stand up local clusters to test against.
+If you do not have a local cluster running, no worries! The e2e tests use the `sigs.k8s.io/kind` and `github.com/rancher/k3d/v5` libraries to stand up local clusters to test against. All you have to do is make sure Docker is running and set the `TESTDISTRO` env var to either `"kind"` or `"k3d"` and the test suite will automatically create the appropriate cluster before the test run, run the tests on it, then automatically destroy it to clean up.
 
-If you want to specify which distros to run the tests against you can set the `TESTDISTRO` environment variable with a comma separated list of K8s distros to use for the testing, each distro is run iteratively. The list of potential distros lists in the `distroTests` struct in `main_test.go`. If nothing is specified, it all 'default' distros are run.
+You can also use K3s by setting `TESTDISTRO=k3s` but note that there are extra requirements of being on Linux with root privileges.
 
-> NOTE: If running against the k3s distro you have to be 'root' to successfully create the cluster.
+### Actually Running The Test
+Here are a few different ways to run the tests, based on your specific situation:
 
-If you did not have a local cluster running before the e2e test but you want to keep it up afterwards to do some debugging, you can set the `SKIP_TEARDOWN` environment variable and the e2e tests will leave the create cluster up after all testing is completed.
+```shell
+# The default way, from the root directory of the repo. Will run all of the tests against your chosen k8s distro. Will automatically build any binary dependencies that don't already exist.
+TESTDISTRO="[provided|kind|k3d|k3s]" make test-e2e
 
-### Actually Running The Test
-We recommend running the tests by going to the main directory of the Zarf repo and running `make test-e2e` this will guarantee all the necessary packages are built and in the right place for the test to find. If you already built everything you can run the tests by staying in this directory and using the command `go test ./... -v`
+# If you already have everything build, you can run this inside this folder. This lets you customize the test run.
+TESTDISTRO=YourChoiceHere go test ./... -v
+
+# Let's say you only want to run one test. You would run:
+TESTDISTRO=YourChoiceHere go test ./... -v -run TestFooBarBaz
+```
 
 > NOTE: The zarf binary and built packages need to live in the ./build directory but if you're trying to run the tests locally with 'go test ./...' then the zarf-init package will need to be in this directory.
 
 ## Adding More Tests
-> NOTE: Since all of the tests use the same K8s cluster, do not write new tests to be executed in parallel and remember to cleanup the cluster after each test by executing `e2e.cleanupAfterTest(t)`. This runs a `zarf destroy --confirm --remove-components` so that the cluster is in a good enough state to run the next test against.
-
-
-Coming Soon: In the future, our goals is to be able to run all of the tests while using an exhaustive combination of different k8s distros and base operating systems. 
+There are a few requirements for all of our tests, that will need to be followed when new tests are added.
 
+1. Tests may not run in parallel, since they use the same kubernetes cluster to run them.
+2. Each test must begin with `defer e2e.cleanupAfterTest(t)` so that the cluster can be reset back to empty when finished.
 
+## Coming Soon
+1. More Linux distros tested
+2. More K8s distros tested, including cloud distros like EKS
+3. Make the tests that run in the CI pipeline more efficient by using more parallelization