Skip to content

Commit

Permalink
Fix NetworkPolicy failures in Kind clusters
Browse files Browse the repository at this point in the history 
When running NetworkPolicy tests in Kind, we "force" the tunnel type to
VXLAN as we observe some failures with Geneve (see #897).

We also change the Github CI jobs so that CNP tests are run in their own
job. This is to address the issue above, and is also a partial
implementation of #893.
  • Loading branch information
@antoninbas
antoninbas committed Jul 2, 2020
1 parent 8e07f0d commit d423421
Show file tree
Hide file tree
Showing 3 changed files with 114 additions and 37 deletions.
43 changes: 37 additions & 6 deletions .github/workflows/kind.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ jobs:
sudo mv kind /usr/local/bin
- name: Run e2e tests
run: |
./ci/kind/test-e2e-kind.sh encap
./ci/kind/test-e2e-kind.sh --encap-mode encap
test-e2e-encap-proxy:
name: E2e tests on a Kind cluster on Linux with proxy enabled
Expand Down Expand Up @@ -83,7 +83,7 @@ jobs:
sudo mv kind /usr/local/bin
- name: Run e2e tests
run: |
./ci/kind/test-e2e-kind.sh encap --proxy
./ci/kind/test-e2e-kind.sh --encap-mode encap --proxy
test-e2e-noencap:
name: E2e tests on a Kind cluster on Linux (noEncap)
Expand Down Expand Up @@ -114,7 +114,7 @@ jobs:
sudo mv kind /usr/local/bin
- name: Run e2e tests
run: |
./ci/kind/test-e2e-kind.sh noEncap
./ci/kind/test-e2e-kind.sh --encap-mode noEncap
test-e2e-noencap-proxy:
name: E2e tests on a Kind cluster on Linux (noEncap) with proxy enabled
Expand Down Expand Up @@ -145,7 +145,7 @@ jobs:
sudo mv kind /usr/local/bin
- name: Run e2e tests
run: |
./ci/kind/test-e2e-kind.sh noEncap --proxy
./ci/kind/test-e2e-kind.sh --encap-mode noEncap --proxy
test-e2e-hybrid:
name: E2e tests on a Kind cluster on Linux (hybrid)
Expand Down Expand Up @@ -176,7 +176,7 @@ jobs:
sudo mv kind /usr/local/bin
- name: Run e2e tests
run: |
./ci/kind/test-e2e-kind.sh hybrid
./ci/kind/test-e2e-kind.sh --encap-mode hybrid
test-e2e-hybrid-proxy:
name: E2e tests on a Kind cluster on Linux (hybrid) with proxy enabled
Expand Down Expand Up @@ -207,7 +207,38 @@ jobs:
sudo mv kind /usr/local/bin
- name: Run e2e tests
run: |
./ci/kind/test-e2e-kind.sh hybrid --proxy
./ci/kind/test-e2e-kind.sh --encap-mode hybrid --proxy
test-e2e-encap-np:
name: E2e tests on a Kind cluster on Linux with Antrea NetworkPolicies enabled
needs: build-antrea-image
runs-on: [ubuntu-18.04]
steps:
- name: Free disk space
# https://github.com/actions/virtual-environments/issues/709
run: |
sudo apt-get clean
df -h
- uses: actions/checkout@v2
- uses: actions/setup-go@v1
with:
go-version: 1.13
- name: Download Antrea image from previous job
uses: actions/download-artifact@v1
with:
name: antrea-ubuntu
- name: Load Antrea image
run: docker load -i antrea-ubuntu/antrea-ubuntu.tar
- name: Install Kind
env:
KIND_VERSION: v0.7.0
run: |
curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
chmod +x ./kind
sudo mv kind /usr/local/bin
- name: Run e2e tests
run: |
./ci/kind/test-e2e-kind.sh --encap-mode encap --np
test-netpol-tmp:
name: Run experimental network policy tests (netpol) on Kind cluster
Expand Down
82 changes: 65 additions & 17 deletions ci/kind/test-e2e-kind.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,31 +18,79 @@

set -eo pipefail

function echoerr {
>&2 echo "$@"
}

_usage="Usage: $0 [--encap-mode <mode>] [--proxy] [--np] [--help|-h]
--encap-mode Traffic encapsulation mode. (default is 'encap')
--proxy Enables Antrea proxy.
--np Enables Namespaced Antrea NetworkPolicy CRDs and ClusterNetworkPolicy related CRDs.
--help, -h Print this message and exit
"

function print_usage {
echoerr "$_usage"
}


TESTBED_CMD=$(dirname $0)"/kind-setup.sh"
YML_CMD=$(dirname $0)"/../../hack/generate-manifest.sh"
COMMON_IMAGES="busybox nginx antrea/antrea-ubuntu:latest"

function quit {
if [[ $? != 0 ]]; then
echo " Test failed cleaning testbed"
echoerr " Test failed cleaning testbed"
$TESTBED_CMD destroy kind
fi
}
trap "quit" INT EXIT

function run_test {
mode=$1
proxy=$2
args=$3
if [[ $proxy != "--proxy" ]]; then
proxy=""
args=$2
fi
mode=""
proxy=false
np=false
args=""
while [[ $# -gt 0 ]]
do
key="$1"

case $key in
--proxy)
proxy=true
shift
;;
--np)
np=true
shift
;;
--encap-mode)
mode="$2"
shift 2
;;
-h|--help)
print_usage
exit 0
;;
*) # unknown option
args="$args $1"
;;
esac
done

manifest_args=""
if $proxy; then
# See https://github.com/vmware-tanzu/antrea/issues/897
manifest_args="$manifest_args --proxy --tun vxlan"
fi
if $np; then
manifest_args="$manifest_args --np"
fi

echo "create test bed with args $args"
function run_test {
echo "creating test bed with args $args"
eval "timeout 600 $TESTBED_CMD create kind --antrea-cni false $args"

$YML_CMD --kind --encap-mode $mode $proxy | docker exec -i kind-control-plane dd of=/root/antrea.yml
$YML_CMD --kind --encap-mode $mode "$manifest_args" | docker exec -i kind-control-plane dd of=/root/antrea.yml
sleep 1
go test -v -timeout=30m github.com/vmware-tanzu/antrea/test/e2e -provider=kind
$TESTBED_CMD destroy kind
Expand All @@ -51,17 +99,17 @@ function run_test {
docker pull busybox
docker pull nginx

if [[ $# == 0 ]] || [[ $1 == "encap" ]]; then
if [[ "$mode" == "" ]] || [[ "$mode" == "encap" ]]; then
echo "======== Test encap mode =========="
run_test encap $2 "--images \"$COMMON_IMAGES\""
run_test encap "$mode" "--images \"$COMMON_IMAGES\""
fi
if [[ $# == 0 ]] || [[ $1 == "noEncap" ]]; then
if [[ "$mode" == "" ]] || [[ "$mode" == "noEncap" ]]; then
echo "======== Test noencap mode =========="
run_test noEncap $2 "--images \"$COMMON_IMAGES\""
run_test noEncap "$mode" "--images \"$COMMON_IMAGES\""
fi
if [[ $# == 0 ]] || [[ $1 == "hybrid" ]]; then
if [[ "$mode" == "" ]] || [[ "$mode" == "hybrid" ]]; then
echo "======== Test hybrid mode =========="
run_test hybrid $2 "--subnets \"20.20.20.0/24\" --images \"$COMMON_IMAGES\""
run_test hybrid "$mode" "--subnets \"20.20.20.0/24\" --images \"$COMMON_IMAGES\""
fi
exit 0

26 changes: 12 additions & 14 deletions test/e2e/clusternetworkpolicy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,33 +79,31 @@ func initialize(t *testing.T, data *TestData) {
allPods = append(allPods, NewPod(ns, podName))
}
}
err := enableCNP(data)
failOnError(err, t)
skipIfCNPDisabled(t, data)
k8sUtils, err = NewKubernetesUtils(data)
failOnError(err, t)
ips, err := k8sUtils.Bootstrap(namespaces, pods)
failOnError(err, t)
podIPs = *ips
}

// TODO: skip restarting controller and only run the test when feature is detected to be enabled in configmap
// https://github.com/vmware-tanzu/antrea/issues/893
func enableCNP(data *TestData) error {
func isCNPEnabled(data *TestData) (bool, error) {
configMap, err := data.GetAntreaConfigMap(antreaNamespace)
if err != nil {
return fmt.Errorf("failed to get ConfigMap: %v", err)
return false, fmt.Errorf("failed to get ConfigMap: %v", err)
}
antreaControllerConf, _ := configMap.Data["antrea-controller.conf"]
antreaControllerConf = strings.Replace(antreaControllerConf, "# ClusterNetworkPolicy: false", " ClusterNetworkPolicy: true", 1)
configMap.Data["antrea-controller.conf"] = antreaControllerConf
if _, err := data.clientset.CoreV1().ConfigMaps(antreaNamespace).Update(context.TODO(), configMap, metav1.UpdateOptions{}); err != nil {
return fmt.Errorf("failed to update ConfigMap %s: %v", configMap.Name, err)
}
_, err = data.restartAntreaControllerPod(defaultTimeout)
return strings.Contains("ClusterNetworkPolicy: true"), nil
}

func skipIfCNPDisabled(tb testing.TB, data *TestData) {
enabled, err := isCNPEnabled(data)
if err != nil {
return fmt.Errorf("error when restarting antrea-controller Pod: %v", err)
t.Fatalf("Cannot determine if CNP enabled: %v", err)
}
if !enabled {
t.Skipf("Skipping test as it required CNP to be enabled")
}
return nil
}

func applyDefaultDenyToAllNamespaces(k8s *KubernetesUtils, namespaces []string) error {
Expand Down

0 comments on commit d423421

Please sign in to comment.