Fix NetworkPolicy failures in Kind clusters

When running NetworkPolicy tests in Kind, we "force" the tunnel type to VXLAN as we observe some failures with Geneve (see #897). We also change the Github CI jobs so that CNP tests are run in their own job. This is to address the issue above, and is also a partial implementation of #893.
antrea-io · Jul 3, 2020 · f9c9db2 · f9c9db2
1 parent 8258cc8
commit f9c9db2
Show file tree

Hide file tree

Showing 3 changed files with 116 additions and 40 deletions.
diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -52,7 +52,7 @@ jobs:
         sudo mv kind /usr/local/bin
     - name: Run e2e tests
       run: |
-        ./ci/kind/test-e2e-kind.sh encap
+        ./ci/kind/test-e2e-kind.sh --encap-mode encap
 
   test-e2e-encap-proxy:
     name: E2e tests on a Kind cluster on Linux with proxy enabled
@@ -83,7 +83,7 @@ jobs:
         sudo mv kind /usr/local/bin
     - name: Run e2e tests
       run: |
-        ./ci/kind/test-e2e-kind.sh encap --proxy
+        ./ci/kind/test-e2e-kind.sh --encap-mode encap --proxy
 
   test-e2e-noencap:
     name: E2e tests on a Kind cluster on Linux (noEncap)
@@ -114,7 +114,7 @@ jobs:
         sudo mv kind /usr/local/bin
     - name: Run e2e tests
       run: |
-        ./ci/kind/test-e2e-kind.sh noEncap
+        ./ci/kind/test-e2e-kind.sh --encap-mode noEncap
 
   test-e2e-noencap-proxy:
     name: E2e tests on a Kind cluster on Linux (noEncap) with proxy enabled
@@ -145,7 +145,7 @@ jobs:
         sudo mv kind /usr/local/bin
     - name: Run e2e tests
       run: |
-        ./ci/kind/test-e2e-kind.sh noEncap --proxy
+        ./ci/kind/test-e2e-kind.sh --encap-mode noEncap --proxy
 
   test-e2e-hybrid:
     name: E2e tests on a Kind cluster on Linux (hybrid)
@@ -176,7 +176,7 @@ jobs:
         sudo mv kind /usr/local/bin
     - name: Run e2e tests
       run: |
-        ./ci/kind/test-e2e-kind.sh hybrid
+        ./ci/kind/test-e2e-kind.sh --encap-mode hybrid
 
   test-e2e-hybrid-proxy:
     name: E2e tests on a Kind cluster on Linux (hybrid) with proxy enabled
@@ -207,7 +207,38 @@ jobs:
         sudo mv kind /usr/local/bin
     - name: Run e2e tests
       run: |
-        ./ci/kind/test-e2e-kind.sh hybrid --proxy
+        ./ci/kind/test-e2e-kind.sh --encap-mode hybrid --proxy
+
+  test-e2e-encap-np:
+    name: E2e tests on a Kind cluster on Linux with Antrea NetworkPolicies enabled
+    needs: build-antrea-image
+    runs-on: [ubuntu-18.04]
+    steps:
+    - name: Free disk space
+      # https://github.com/actions/virtual-environments/issues/709
+      run: |
+        sudo apt-get clean
+        df -h
+    - uses: actions/checkout@v2
+    - uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+    - name: Download Antrea image from previous job
+      uses: actions/download-artifact@v1
+      with:
+        name: antrea-ubuntu
+    - name: Load Antrea image
+      run:  docker load -i antrea-ubuntu/antrea-ubuntu.tar
+    - name: Install Kind
+      env:
+        KIND_VERSION: v0.7.0
+      run: |
+        curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
+        chmod +x ./kind
+        sudo mv kind /usr/local/bin
+    - name: Run e2e tests
+      run: |
+        ./ci/kind/test-e2e-kind.sh --encap-mode encap --np
 
   test-netpol-tmp:
     name: Run experimental network policy tests (netpol) on Kind cluster

diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh
@@ -18,31 +18,80 @@
 
 set -eo pipefail
 
+function echoerr {
+    >&2 echo "$@"
+}
+
+_usage="Usage: $0 [--encap-mode <mode>] [--proxy] [--np] [--help|-h]
+        --encap-mode                  Traffic encapsulation mode. (default is 'encap')
+        --proxy                       Enables Antrea proxy.
+        --np                          Enables Namespaced Antrea NetworkPolicy CRDs and ClusterNetworkPolicy related CRDs.
+        --help, -h                    Print this message and exit
+"
+
+function print_usage {
+    echoerr "$_usage"
+}
+
+
 TESTBED_CMD=$(dirname $0)"/kind-setup.sh"
 YML_CMD=$(dirname $0)"/../../hack/generate-manifest.sh"
 COMMON_IMAGES="busybox nginx antrea/antrea-ubuntu:latest"
 
 function quit {
   if [[ $? != 0 ]]; then
-    echo " Test failed cleaning testbed"
+    echoerr " Test failed cleaning testbed"
     $TESTBED_CMD destroy kind
   fi
 }
 trap "quit" INT EXIT
 
-function run_test {
-  mode=$1
-  proxy=$2
-  args=$3
-  if [[ $proxy != "--proxy" ]]; then
-    proxy=""
-    args=$2
-  fi
+mode=""
+proxy=false
+np=false
+args=""
+while [[ $# -gt 0 ]]
+do
+key="$1"
+
+case $key in
+    --proxy)
+    proxy=true
+    shift
+    ;;
+    --np)
+    np=true
+    shift
+    ;;
+    --encap-mode)
+    mode="$2"
+    shift 2
+    ;;
+    -h|--help)
+    print_usage
+    exit 0
+    ;;
+    *)    # unknown option
+    args="$args $1"
+    shift 1
+    ;;
+esac
+done
+
+manifest_args=""
+if $proxy; then
+    manifest_args="$manifest_args --proxy"
+fi
+if $np; then
+    # See https://github.com/vmware-tanzu/antrea/issues/897
+    manifest_args="$manifest_args --np --tun vxlan"
+fi
 
-  echo "create test bed with args $args"
+function run_test {
+  echo "creating test bed with args $args"
   eval "timeout 600 $TESTBED_CMD create kind --antrea-cni false $args"
 
-  $YML_CMD --kind --encap-mode $mode $proxy | docker exec -i kind-control-plane dd of=/root/antrea.yml
+  $YML_CMD --kind --encap-mode $mode $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea.yml
   sleep 1
   go test -v -timeout=30m github.com/vmware-tanzu/antrea/test/e2e -provider=kind
   $TESTBED_CMD destroy kind
@@ -51,17 +100,17 @@ function run_test {
 docker pull busybox
 docker pull nginx
 
-if [[ $# == 0 ]] || [[ $1 == "encap" ]]; then
+if [[ "$mode" == "" ]] || [[ "$mode" == "encap" ]]; then
   echo "======== Test encap mode =========="
-  run_test encap $2 "--images \"$COMMON_IMAGES\""
+  run_test encap "$mode" "--images \"$COMMON_IMAGES\""
 fi
-if [[ $# == 0 ]] || [[ $1 == "noEncap" ]]; then
+if [[ "$mode" == "" ]] || [[ "$mode" == "noEncap" ]]; then
   echo "======== Test noencap mode =========="
-  run_test noEncap $2 "--images \"$COMMON_IMAGES\""
+  run_test noEncap "$mode" "--images \"$COMMON_IMAGES\""
 fi
-if [[ $# == 0 ]] || [[ $1 == "hybrid" ]]; then
+if [[ "$mode" == "" ]] || [[ "$mode" == "hybrid" ]]; then
   echo "======== Test hybrid mode =========="
-  run_test hybrid $2 "--subnets \"20.20.20.0/24\" --images \"$COMMON_IMAGES\""
+  run_test hybrid "$mode" "--subnets \"20.20.20.0/24\" --images \"$COMMON_IMAGES\""
 fi
 exit 0
 
diff --git a/test/e2e/clusternetworkpolicy_test.go b/test/e2e/clusternetworkpolicy_test.go
@@ -15,15 +15,13 @@
 package e2e
 
 import (
-	"context"
 	"fmt"
 	"strings"
 	"testing"
 	"time"
 
 	log "github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	secv1alpha1 "github.com/vmware-tanzu/antrea/pkg/apis/security/v1alpha1"
 	. "github.com/vmware-tanzu/antrea/test/e2e/utils"
@@ -79,33 +77,31 @@ func initialize(t *testing.T, data *TestData) {
 			allPods = append(allPods, NewPod(ns, podName))
 		}
 	}
-	err := enableCNP(data)
-	failOnError(err, t)
-	k8sUtils, err = NewKubernetesUtils(data)
+	skipIfCNPDisabled(t, data)
+	k8sUtils, err := NewKubernetesUtils(data)
 	failOnError(err, t)
 	ips, err := k8sUtils.Bootstrap(namespaces, pods)
 	failOnError(err, t)
 	podIPs = *ips
 }
 
-// TODO: skip restarting controller and only run the test when feature is detected to be enabled in configmap
-// https://github.com/vmware-tanzu/antrea/issues/893
-func enableCNP(data *TestData) error {
+func isCNPEnabled(data *TestData) (bool, error) {
 	configMap, err := data.GetAntreaConfigMap(antreaNamespace)
 	if err != nil {
-		return fmt.Errorf("failed to get ConfigMap: %v", err)
+		return false, fmt.Errorf("failed to get ConfigMap: %v", err)
 	}
 	antreaControllerConf, _ := configMap.Data["antrea-controller.conf"]
-	antreaControllerConf = strings.Replace(antreaControllerConf, "# ClusterNetworkPolicy: false", "  ClusterNetworkPolicy: true", 1)
-	configMap.Data["antrea-controller.conf"] = antreaControllerConf
-	if _, err := data.clientset.CoreV1().ConfigMaps(antreaNamespace).Update(context.TODO(), configMap, metav1.UpdateOptions{}); err != nil {
-		return fmt.Errorf("failed to update ConfigMap %s: %v", configMap.Name, err)
-	}
-	_, err = data.restartAntreaControllerPod(defaultTimeout)
+	return strings.Contains(antreaControllerConf, "ClusterNetworkPolicy: true"), nil
+}
+
+func skipIfCNPDisabled(tb testing.TB, data *TestData) {
+	enabled, err := isCNPEnabled(data)
 	if err != nil {
-		return fmt.Errorf("error when restarting antrea-controller Pod: %v", err)
+		tb.Fatalf("Cannot determine if CNP enabled: %v", err)
+	}
+	if !enabled {
+		tb.Skipf("Skipping test as it required CNP to be enabled")
 	}
-	return nil
 }
 
 func applyDefaultDenyToAllNamespaces(k8s *KubernetesUtils, namespaces []string) error {