feat(cni): run e2e multizone on calico without cni

Signed-off-by: slonka <slonka@users.noreply.github.com>
kumahq · Aug 11, 2022 · 1be377e · 1be377e
1 parent 60b1816
commit 1be377e
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 36 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -255,6 +255,10 @@ jobs:
         description: The golang arch
         type: string
         default: amd64
+      cniNetworkPlugin:
+        description: The CNI networking plugin to use [flannel | calico]
+        type: string
+        default: flannel
     executor:
       name: vm-<< parameters.arch >>
     parallelism: << parameters.parallelism >>
@@ -270,6 +274,7 @@ jobs:
               target:<< parameters.target >> \
               parallelism:<< parameters.parallelism >> \
               arch:<< parameters.arch >> \
+              cniNetworkPlugin:<< parameters.cniNetworkPlugin >> \
             "
             function skip() {
               echo "Non valid job combination halting job reason: $1"
@@ -284,8 +289,8 @@ jobs:
             fi
             if [[ ! $(./tools/ci/has_label.sh ci/run-full-matrix) == "true" ]]; then
               if [[ "<< pipeline.git.branch >>" != "master" && "<< pipeline.git.branch >>" != "release-"* ]]; then
-                if [[ "<< parameters.k8sVersion >>" == "kindIpv6" || "<< parameters.k8sVersion >>" == "v1.19.16-k3s1" || "<< parameters.arch >>" == "arm64" ]]; then
-                  skip "Not running tests on PRs with kindIpv6, oldK8s and arm64"
+                if [[ "<< parameters.cniNetworkPlugin >>" == "calico" || "<< parameters.k8sVersion >>" == "kindIpv6" || "<< parameters.k8sVersion >>" == "v1.19.16-k3s1" || "<< parameters.arch >>" == "arm64" ]]; then
+                  skip "Not running tests on PRs with kindIpv6, oldK8s, calico and arm64"
                 fi
               fi
             fi
@@ -334,6 +339,7 @@ jobs:
             if [[ "<< parameters.k8sVersion >>" != "kind"* ]]; then
               export CI_K3S_VERSION=<< parameters.k8sVersion >>
               export K3D=true
+              export K3D_NETWORK_CNI=<< parameters.cniNetworkPlugin >>
             fi
             if [[ "<< parameters.arch >>" == "arm64" ]]; then
               export MAKE_PARAMETERS="-j1"
@@ -479,6 +485,17 @@ workflows:
               arch: [ amd64, arm64 ]
           parallelism: 1
           requires: [ build-<< matrix.arch >>, check ]
+      - e2e:
+          name: << matrix.target >>:<< matrix.arch >>-<< matrix.k8sVersion >>-calico
+          matrix:
+            alias: test/e2e
+            parameters:
+              k8sVersion: [ v1.21.7-k3s1 ]
+              target: [ test/e2e-multizone ]
+              arch: [ amd64 ]
+              cniNetworkPlugin: [ calico ]
+          parallelism: 1
+          requires: [ build-<< matrix.arch >>, check ]
       - release:
           # publish artifacts speculatively for normal commits
           filters: # only on master and release-* branches (never on PRs and tags)

diff --git a/mk/k3d.mk b/mk/k3d.mk
@@ -40,6 +40,19 @@ KUMA_NAMESPACE ?= kuma-system
 #  [...etc]
 PORT_PREFIX := $$(($(patsubst 300-%,300+%-1,$(KIND_CLUSTER_NAME:kuma%=300%))))
 
+K3D_NETWORK_CNI ?= flannel
+K3D_CLUSTER_CREATE_OPTS ?= -i rancher/k3s:$(CI_K3S_VERSION) \
+	--k3s-arg '--no-deploy=traefik@server:0' \
+	--k3s-arg '--disable=metrics-server@server:0' \
+	--network kind \
+	--port "$(PORT_PREFIX)80-$(PORT_PREFIX)89:30080-30089@server:0" \
+	--timeout 120s
+
+ifeq ($(K3D_NETWORK_CNI),calico)
+	K3D_CLUSTER_CREATE_OPTS += --volume "$(TOP)/test/k3d/calico.yaml.kubelint-excluded:/var/lib/rancher/k3s/server/manifests/calico.yaml" \
+		--k3s-arg '--flannel-backend=none@server:*'
+endif
+
 .PHONY: k3d/network/create
 k3d/network/create:
 	@touch $(BUILD_DIR)/k3d_network.lock && \
@@ -51,35 +64,7 @@ k3d/network/create:
 k3d/start: ${KIND_KUBECONFIG_DIR} k3d/network/create
 	@echo "PORT_PREFIX=$(PORT_PREFIX)"
 	@KUBECONFIG=$(KIND_KUBECONFIG) \
-		k3d cluster create "$(KIND_CLUSTER_NAME)" \
-			-i rancher/k3s:$(CI_K3S_VERSION) \
-			--k3s-arg '--no-deploy=traefik@server:0' \
-			--k3s-arg '--flannel-backend=none@server:*' \
-			--volume "/tmp/calico.yaml:/var/lib/rancher/k3s/server/manifests/calico.yaml" \
-			--k3s-arg '--disable=metrics-server@server:0' \
-			--network kind \
-			--port "$(PORT_PREFIX)80-$(PORT_PREFIX)89:30080-30089@server:0" \
-			--timeout 120s && \
-	$(MAKE) k3d/wait
-	@echo
-	@echo '>>> You need to manually run the following command in your shell: >>>'
-	@echo
-	@echo export KUBECONFIG="$(KIND_KUBECONFIG)"
-	@echo
-	@echo '<<< ------------------------------------------------------------- <<<'
-	@echo
-
-.PHONY: k3d-flannel/start
-k3d-flannel/start: ${KIND_KUBECONFIG_DIR} k3d/network/create
-	@echo "PORT_PREFIX=$(PORT_PREFIX)"
-	@KUBECONFIG=$(KIND_KUBECONFIG) \
-		k3d cluster create "$(KIND_CLUSTER_NAME)" \
-			-i rancher/k3s:$(CI_K3S_VERSION) \
-			--k3s-arg '--no-deploy=traefik@server:0' \
-			--k3s-arg '--disable=metrics-server@server:0' \
-			--network kind \
-			--port "$(PORT_PREFIX)80-$(PORT_PREFIX)89:30080-30089@server:0" \
-			--timeout 120s && \
+		k3d cluster create "$(KIND_CLUSTER_NAME)" $(K3D_CLUSTER_CREATE_OPTS)
 	$(MAKE) k3d/wait
 	@echo
 	@echo '>>> You need to manually run the following command in your shell: >>>'

diff --git a/test/framework/config.go b/test/framework/config.go
@@ -125,11 +125,11 @@ func (c E2eConfig) AutoConfigure() error {
 type K8sType string
 
 const (
-	KindK8sType  K8sType = "kind"
-	K3dK8sType   K8sType = "k3d"
-	K3dCalicoK8sType   K8sType = "k3d-calico"
-	AzureK8sType K8sType = "azure"
-	AwsK8sType   K8sType = "aws"
+	KindK8sType      K8sType = "kind"
+	K3dK8sType       K8sType = "k3d"
+	K3dCalicoK8sType K8sType = "k3d-calico"
+	AzureK8sType     K8sType = "azure"
+	AwsK8sType       K8sType = "aws"
 )
 
 type CniConf struct {

diff --git a/test/k3d/calico.yaml → test/k3d/calico.yaml.kubelint-excluded b/test/k3d/calico.yaml → test/k3d/calico.yaml.kubelint-excluded
@@ -1,3 +1,5 @@
+# file taken from k3d docs https://github.com/k3d-io/k3d/blob/90c19610ba877141bc642894d5b43db66f0d3ecf/docs/usage/advanced/calico.md#use-calico-instead-of-flannel
+# it's extension is .kubelint-excluded so that kube-lint does not complain that it has privileged containers
 ---
 # Source: calico/templates/calico-config.yaml
 # This ConfigMap is used to configure a self-hosted Calico installation.