test: implement new development environment (#628)

Replace the multiple bash scripts used to setup the `csi-driver` test environment with Terraform and Make. --------- Co-authored-by: Julian Tölle <julian.toelle@hetzner-cloud.de>
hetznercloud · Jun 24, 2024 · a583929 · a583929
1 parent 2541af0
commit a583929
Show file tree

Hide file tree

Showing 24 changed files with 583 additions and 389 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -23,8 +23,8 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
-  test-kubernetes:
-    name: e2e k3s ${{ matrix.k3s }}
+  kubernetes:
+    name: kubernetes ${{ matrix.k3s }}
     runs-on: ubuntu-latest
 
     permissions:
@@ -33,21 +33,22 @@ jobs:
     concurrency: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.k3s }}
 
     strategy:
-      # The e2e tests are flaky and often one of the jobs fails. The default setting
-      # causes all other currently running jobs to abort and all need to be restarted.
-      fail-fast: false
+      fail-fast: false # Continue tests matrix if a flaky run occur.
       matrix:
         include:
-          # All k3s after January 2024 break our e2e tests, we hardcode
-          # the versions for now until we can fix the source of this.
-          - k3s: v1.26.12+k3s1
-            k8s-test: v1.26.12
-          - k3s: v1.27.9+k3s1
-            k8s-test: v1.27.9
-          - k3s: v1.28.5+k3s1
-            k8s-test: v1.28.5
-          - k3s: v1.29.0+k3s1
-            k8s-test: v1.29.0
+          - k3s: v1.26
+            k8s-test: v1.26.15
+          - k3s: v1.27
+            k8s-test: v1.27.15
+          - k3s: v1.28
+            k8s-test: v1.28.11
+          - k3s: v1.29
+            k8s-test: v1.29.6
+
+    env:
+      K3S_CHANNEL: ${{ matrix.k3s }}
+      K8S_TEST_VERSION: ${{ matrix.k8s-test }}
+      ENV: gha-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.k3s }}
 
     steps:
       - uses: actions/checkout@v4
@@ -56,36 +57,48 @@ jobs:
         with:
           go-version-file: go.mod
 
-      - uses: hetznercloud/tps-action@main
+      - uses: opentofu/setup-opentofu@v1
+        with:
+          tofu_version: v1.7.2 # renovate: datasource=github-releases depName=opentofu/opentofu
+          tofu_wrapper: false
 
-      - uses: hetznercloud/setup-hcloud@v1
+      - uses: docker/setup-buildx-action@v3
 
       - uses: yokawasa/action-setup-kube-tools@v0.11.1
         with:
           setup-tools: |
             helm
             kubectl
             skaffold
-          helm: v3.11.2
-          kubectl: v1.29.0
-          skaffold: v2.3.0
+          helm: v3.15.2 # renovate: datasource=github-releases depName=helm/helm
+          kubectl: v1.29.6 # renovate: datasource=github-releases depName=kubernetes/kubernetes
+          skaffold: v2.12.0 # renovate: datasource=github-releases depName=GoogleContainerTools/skaffold
 
-      - name: Run tests
-        env:
-          K3S_VERSION: ${{ matrix.k3s }}
-          K8S_TEST_VERSION: ${{ matrix.k8s-test }}
-          SCOPE: gha-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.k3s }}
-        run: |
-          curl -sLS https://get.k3sup.dev | sh
+      - name: Setup k3sup
+        run: curl -sLS https://get.k3sup.dev | sh
+
+      - name: Setup k8s test binaries
+        run: make -C test/e2e/kubernetes bin
 
-          trap "hack/dev-down.sh" EXIT
-          source <(hack/dev-up.sh)
+      - uses: hetznercloud/tps-action@main
 
-          skaffold build --tag="e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_NUMBER}"
-          tag=$(skaffold build --tag="e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_NUMBER}" --quiet --output="{{ (index .Builds 0).Tag }}")
-          skaffold deploy --images=docker.io/hetznercloud/hcloud-csi-driver=$tag
+      - name: Setup environment
+        run: make -C dev up
 
-          test/e2e/kubernetes/run-e2e-tests.sh
+      - name: Run skaffold
+        run: |
+          source dev/files/env.sh
+          skaffold run
+
+      - name: Run parallel tests
+        run: |
+          source dev/files/env.sh
+          make -C test/e2e/kubernetes parallel
+
+      - name: Run serial tests
+        run: |
+          source dev/files/env.sh
+          make -C test/e2e/kubernetes serial
 
   deploy-manifests:
     runs-on: ubuntu-latest
@@ -95,7 +108,7 @@ jobs:
         with:
           setup-tools: |
             helm
-          helm: v3.11.2
+          helm: v3.11.2 # renovate: datasource=github-releases depName=helm/helm
 
       - uses: actions/checkout@v4
 
@@ -117,7 +130,7 @@ jobs:
         with:
           setup-tools: |
             helm
-          helm: v3.11.2
+          helm: v3.11.2 # renovate: datasource=github-releases depName=helm/helm
 
       - uses: actions/checkout@v4
 

diff --git a/.gitignore b/.gitignore
@@ -1,9 +1,6 @@
 # rootfs for docker plugin
 deploy/docker-swarm/pkg/plugin
 
-# Tmp files for dev env
-hack/.*
-
 # Build output
 dist/
 hcloud-csi-*.tgz

diff --git a/README.md b/README.md
@@ -17,117 +17,77 @@ relevant to that Container Orchestrator behind the link:
 - [Docker Swarm](./docs/docker-swarm/README.md)️ _⚠️ Not officially supported_
 - [HashiCorp Nomad](./docs/nomad/README.md)️ _⚠️ Not officially supported_
 
-## Tests
+## Development
 
-### Integration Tests
+### Setup a development environment
 
-**Requirements: Docker**
+To setup a development environment, make sure you installed the following tools:
 
-The core operations like publishing and resizing can be tested locally with Docker.
+- [tofu](https://opentofu.org/)
+- [k3sup](https://github.com/alexellis/k3sup)
+- [docker](https://www.docker.com/)
+- [skaffold](https://skaffold.dev/)
 
-```bash
-go test $(go list ./... | grep integration) -v
-```
+1. Configure a `HCLOUD_TOKEN` in your shell session.
 
-### E2E Tests
+> [!WARNING]
+> The development environment runs on Hetzner Cloud servers which will induce costs.
 
-> ⚠️ Kubernetes E2E Tests were recently refactored and the docs are now outdated.
-> See the [GitHub Actions workflow](.github/workflows/test_e2e.yml) for an
-> up-to-date script to run the e2e tests.
+2. Deploy the development cluster:
 
-The Hetzner Cloud CSI Driver was tested against the official k8s e2e
-tests for a specific version. You can run the tests with the following
-commands. Keep in mind, that these tests run on real cloud servers and
-will create volumes that will be billed.
+```sh
+make -C dev up
+```
 
-**Test Server Setup**:
+3. Load the generated configuration to access the development cluster:
 
-1x CPX21 (Ubuntu 18.04)
+```sh
+source dev/files/env.sh
+```
 
-**Requirements: Docker and Go 1.17**
+4. Check that the development cluster is healthy:
 
-1. Configure your environment correctly
-   ```bash
-   export HCLOUD_TOKEN=<specify a project token>
-   export K8S_VERSION=1.21.0 # The specific (latest) version is needed here
-   export USE_SSH_KEYS=key1,key2 # Name or IDs of your SSH Keys within the Hetzner Cloud, the servers will be accessible with that keys
-   ```
-2. Run the tests
-   ```bash
-   go test $(go list ./... | grep e2e) -v -timeout 60m
-   ```
+```sh
+kubectl get nodes -o wide
+```
 
-The tests will now run, this will take a while (~30 min).
+5. Start developing the CSI driver in the development cluster:
 
-**If the tests fail, make sure to clean up the project with the Hetzner Cloud Console or the hcloud cli.**
+```sh
+skaffold dev
+```
 
-### Local test setup  
+On code change, skaffold will rebuild the image, redeploy it and print all logs from csi components.
 
-> ⚠️ Local Kubernetes Dev Setup was recently refactored and the docs are now
-> outdated. Check out the scripts [dev-up.sh](hack/dev-up.sh) &
-> [dev-down.sh](hack/dev-down.sh) for an automatic dev setup.
+⚠️ Do not forget to clean up the development cluster once are finished:
 
-This repository provides [skaffold](https://skaffold.dev/) to easily deploy / debug this driver on demand
+```sh
+make -C dev down
+```
 
-#### Requirements
-1. Install [hcloud-cli](https://github.com/hetznercloud/cli)
-2. Install [k3sup](https://github.com/alexellis/k3sup)
-3. Install [cilium](https://github.com/cilium/cilium-cli)
-4. Install [docker](https://www.docker.com/)
+### Run the docker e2e tests
 
-You will also need to set a `HCLOUD_TOKEN` in your shell session
+To run the integrations tests, make sure you installed the following tools:
 
-#### Manual Installation guide
+- [docker](https://www.docker.com/)
 
-1. Create an SSH key
+1. Run the following command to run the integrations tests:
 
-Assuming you already have created an ssh key via `ssh-keygen`
-```
-hcloud ssh-key create --name ssh-key-csi-test --public-key-from-file ~/.ssh/id_rsa.pub 
+```sh
+go test -v ./test/integration
 ```
 
-2. Create a server
-```
-hcloud server create --name csi-test-server --image ubuntu-20.04 --ssh-key ssh-key-csi-test --type cx22 
-```
+### Run the kubernetes e2e tests
 
-3. Setup k3s on this server
-```
-k3sup install --ip $(hcloud server ip csi-test-server) --local-path=/tmp/kubeconfig --cluster --k3s-channel=v1.23 --k3s-extra-args='--no-flannel --no-deploy=servicelb --no-deploy=traefik --disable-cloud-controller --disable-network-policy --kubelet-arg=cloud-provider=external'
-```
-- The kubeconfig will be created under `/tmp/kubeconfig`
-- Kubernetes version can be configured via `--k3s-channel`
+The Hetzner Cloud CSI driver is tested against the official kubernetes e2e tests.
 
-4. Switch your kubeconfig to the test cluster
-```
-export KUBECONFIG=/tmp/kubeconfig
-```
-
-5. Install cilium + test your cluster
-```
-cilium install
-```
+Before running the integrations tests, make sure you followed the [Setup a development environment](#setup-a-development-environment) steps.
 
-6. Add your secret to the cluster
-```
-kubectl -n kube-system create secret generic hcloud --from-literal="token=$HCLOUD_TOKEN"
-```
+1. Run the kubernetes e2e tests using the following command:
 
-7. Install hcloud-cloud-controller-manager + test your cluster
+```sh
+make -C test/e2e/kubernetes test
 ```
-kubectl apply -f  https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm.yaml
-kubectl config set-context default
-kubectl get node -o wide
-```
-
-8. Deploy your CSI driver
-```
-SKAFFOLD_DEFAULT_REPO=naokiii skaffold dev
-```
-- `docker login` required
-- Skaffold is using your own dockerhub repo to push the CSI image.
-
-On code change, skaffold will repack the image & deploy it to your test cluster again. Also, it is printing all logs from csi components.
 
 ## License
 

diff --git a/dev/.gitignore b/dev/.gitignore
@@ -0,0 +1,9 @@
+.terraform*
+terraform.tfstate
+terraform.tfstate.backup
+*.auto.tfvars
+
+files/*
+!files/.gitkeep
+
+.env
diff --git a/hack/Dockerfile → dev/Dockerfile b/hack/Dockerfile → dev/Dockerfile
diff --git a/dev/Makefile b/dev/Makefile
@@ -0,0 +1,27 @@
+SHELL = bash
+.ONESHELL:
+
+ENV ?= dev
+K3S_CHANNEL ?= stable
+
+env.auto.tfvars:
+	@echo 'name = "$(ENV)"' > "$@"
+	@echo 'hcloud_token = "$(HCLOUD_TOKEN)"' >> "$@"
+	@echo 'k3s_channel = "$(K3S_CHANNEL)"' >> "$@"
+
+.terraform:
+	tofu init
+
+up: .terraform env.auto.tfvars
+	tofu apply -auto-approve
+	$(MAKE) port-forward
+
+down: .terraform env.auto.tfvars
+	tofu destroy -auto-approve
+
+port-forward:
+	source files/env.sh
+	bash files/registry-port-forward.sh
+
+clean:
+	rm -Rf files/* .terraform* terraform.tfstate* env.auto.tfvars
diff --git a/dev/files/.gitkeep b/dev/files/.gitkeep