Add e2e tests for NEG asm mode.

The e2e tests will test the configmap based asm mode on/off logic and
the DestinationRule/Serivce NEG creation logic in the asm mode.

cmd/e2e-test/asm_test.go

@@ -0,0 +1,278 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	istioV1alpha3 "istio.io/api/networking/v1alpha3"
+	apiappsv1 "k8s.io/api/apps/v1"
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/ingress-gce/pkg/e2e"
+	"k8s.io/klog"
+)
+
+const (
+	asmConfigNamespace          = "kube-system"
+	asmConfigName               = "ingress-controller-asm-cm-config"
+	negControllerRestartTimeout = 5 * time.Minute
+)
+
+// TestASMConfig tests the ASM enable/disable, it can't run parallel with other tests.
+func TestASMConfig(t *testing.T) {
+	Framework.RunWithSandbox("TestASMConfig", t, func(t *testing.T, s *e2e.Sandbox) {
+		for _, tc := range []struct {
+			desc                string
+			configMap           map[string]string
+			wantConfigMapEvents []string
+		}{
+			{
+				desc:                "Invalid ConfigMap value equals to disable",
+				configMap:           map[string]string{"enable-asm": "INVALID"},
+				wantConfigMapEvents: []string{"The map provided a unvalid value for field: enable-asm, value: INVALID"},
+			},
+			{
+				desc:                "Invalid ConfigMap filed equals to disable",
+				configMap:           map[string]string{"enable-unknow-feild": "INVALID1"},
+				wantConfigMapEvents: []string{"The map contains a unknown key-value pair: enable-unknow-feild:INVALID1"},
+			},
+			{
+				desc:                "Set enable-asm to true should restart the controller",
+				configMap:           map[string]string{"enable-asm": "true"},
+				wantConfigMapEvents: []string{"ConfigMapConfigController: Get a update on the ConfigMapConfig, Restarting Ingress controller"},
+			},
+			// TODO(koonwah): The below case is not fully tested, should update the neg controller to include a enable-asm event to the target CM.
+			{
+				desc:      "Invalid ConfigMap value equals to disable",
+				configMap: map[string]string{"enable-asm": "INVALID2"},
+				wantConfigMapEvents: []string{"The map provided a unvalid value for field: enable-asm, value: INVALID2",
+					"ConfigMapConfigController: Get a update on the ConfigMapConfig, Restarting Ingress controller"},
+			},
+		} {
+			if err := e2e.UpdateCreateConfigMap(s, asmConfigNamespace, asmConfigName, tc.configMap); err != nil {
+				t.Error(err)
+			}
+			if err := e2e.WaitConfigMapEvents(s, asmConfigNamespace, asmConfigName, tc.wantConfigMapEvents, negControllerRestartTimeout); err != nil {
+				t.Fatalf("Failed to get events: %v; Error %e", strings.Join(tc.wantConfigMapEvents, ";"), err)
+			}
+		}
+	})
+}
+
+func TestASMServiceAndDestinationRule(t *testing.T) {
+	_ = istioV1alpha3.DestinationRule{}
+	_ = apiv1.ComponentStatus{}
+
+	// This test case will need two namespaces, one will in asm-skip-namespaces.
+	Framework.RunWithSandbox("TestASMServiceAndDestinationRule", t, func(t *testing.T, sSkip *e2e.Sandbox) {
+		Framework.RunWithSandbox("TestASMServiceAndDestinationRule", t, func(t *testing.T, s *e2e.Sandbox) {
+			// Enable ASM mode
+			ctx := context.Background()
+
+			asmConfig := map[string]string{"enable-asm": "true",
+				"asm-skip-namespaces": fmt.Sprintf("kube-system,istio-system,%s", sSkip.Namespace)}
+			if err := e2e.UpdateCreateConfigMap(s, asmConfigNamespace, asmConfigName,
+				asmConfig); err != nil {
+				t.Error(err)
+			}
+
+			var porterPort int32
+			porterPort = 80
+			svcName := "service"
+			svcSkipName := "service-skip"
+
+			for _, deployment := range []*apiappsv1.Deployment{
+				createPorterDeployment(s.Namespace, "deployment-v1", 1, map[string]string{"app": "porter", "version": "v1"}, porterPort),
+				createPorterDeployment(s.Namespace, "deployment-v2", 2, map[string]string{"app": "porter", "version": "v2"}, porterPort),
+				createPorterDeployment(s.Namespace, "deployment-v3", 3, map[string]string{"app": "porter", "version": "v3"}, porterPort),
+			} {
+				if err := e2e.CrateDeployment(s, deployment); err != nil {
+					t.Errorf("Failed to create deployment, Error: %s", err)
+				}
+			}
+
+			service := createService(s.Namespace, svcName, map[string]string{"app": "porter"}, porterPort)
+			serviceSkip := createService(sSkip.Namespace, svcSkipName, map[string]string{"app": "porter"}, porterPort)
+
+			for _, tc := range []struct {
+				desc            string
+				svc             *apiv1.Service
+				inSkipNamespace bool
+			}{
+				{desc: "NEG Controller should create NEGs for all ports for a service by default", svc: service, inSkipNamespace: false},
+				{desc: "NEG Controller shouldn't create NEGs for all ports for a service if it's in a skip namespace", svc: serviceSkip, inSkipNamespace: true},
+			} {
+				sandbox := s
+				timeout := 5 * time.Minute
+				NEGCount := 1 // This equals to the service port count.
+				if tc.inSkipNamespace {
+					NEGCount = 0
+					sandbox = sSkip
+					timeout = 1 * time.Minute
+				}
+				if err := e2e.CrateService(sandbox, tc.svc); err != nil {
+					t.Errorf("Failed to create service, Error: %s", err)
+				}
+
+				// Test the Service Annotations
+				negStatus, err := e2e.WaitServiceNEGAnnotation(sandbox, tc.svc.Namespace, tc.svc.Name, NEGCount, timeout)
+				if err != nil {
+					if !(tc.inSkipNamespace && err == wait.ErrWaitTimeout) {
+						t.Errorf("Failed to wait for Service NEGAnnotation, error: %s", err)
+					}
+				}
+				if tc.inSkipNamespace {
+					if negStatus != nil {
+						t.Errorf("Service: %s/%s is in the ASM skip namespace, shoudln't have NEG Status. ASM Config: %v, NEGStatus got: %v",
+							tc.svc.Namespace, tc.svc.Name, asmConfig, negStatus)
+					}
+				} else {
+					if negName, ok := negStatus.NetworkEndpointGroups[strconv.Itoa(int(porterPort))]; ok {
+						// No backend pod exists, so the NEG has 0 endpoint.
+						if err := e2e.WaitForNegs(ctx, Framework.Cloud, negName, negStatus.Zones, false, 0); err != nil {
+							t.Errorf("Failed to wait Negs, error: %s", err)
+						}
+					} else {
+						t.Fatalf("Service annotation doesn't contain the desired NEG status, want: %d, have: %v", porterPort, negStatus.NetworkEndpointGroups)
+					}
+				}
+			}
+
+			for _, tc := range []struct {
+				desc                   string
+				destinationRuleName    string
+				subsetEndpointCountMap map[string]int
+				crossNamespace         bool
+			}{
+				{desc: "NEG controller should create NEGs for destinationrule", destinationRuleName: "porter-destinationrule", subsetEndpointCountMap: map[string]int{"v1": 1, "v2": 2, "v3": 3}, crossNamespace: false},
+				{desc: "NEG controller should update NEGs for destinationrule", destinationRuleName: "porter-destinationrule", subsetEndpointCountMap: map[string]int{"v1": 1, "v2": 2}, crossNamespace: false},
+				{desc: "NEG controller should update NEGs for destinationrule", destinationRuleName: "porter-destinationrule-1", subsetEndpointCountMap: map[string]int{"v1": 1}, crossNamespace: true},
+			} {
+				sandbox := s
+				namespace := s.Namespace
+				destinationRule := istioV1alpha3.DestinationRule{}
+				subset := []*istioV1alpha3.Subset{}
+				for v := range tc.subsetEndpointCountMap {
+					subset = append(subset, &istioV1alpha3.Subset{Name: v, Labels: map[string]string{"version": v}})
+				}
+				destinationRule.Subsets = subset
+
+				// crossNamespace will test DestinationRules that refering a serive located in a different namespace
+				if tc.crossNamespace {
+					sandbox = sSkip
+					namespace = sSkip.Namespace
+					destinationRule.Host = fmt.Sprintf("%s.%s.svc.cluster.local", svcName, s.Namespace)
+				} else {
+					destinationRule.Host = svcName
+				}
+
+				if err := e2e.UpdateCreateDestinationRule(sandbox, namespace, tc.destinationRuleName, &destinationRule); err != nil {
+					klog.Errorf("Failed to create destinationRule, error: %s", err)
+				}
+
+				// One DestinationRule should have count(NEGs) = count(subset)* count(port)
+				dsNEGStatus, err := e2e.WaitDestinationRuleAnnotation(sandbox, namespace, tc.destinationRuleName, len(subset)*1, 5*time.Minute)
+				if err != nil {
+					klog.Errorf("Failed to validate the NEG count. Error: %s", err)
+				}
+
+				zones := dsNEGStatus.Zones
+				for subsetVersion, endpointCount := range tc.subsetEndpointCountMap {
+					negNames, ok := dsNEGStatus.NetworkEndpointGroups[subsetVersion]
+					if !ok {
+						t.Fatalf("DestinationRule annotation doesn't contain the desired NEG status, want: %s, have: %v", subsetVersion, dsNEGStatus.NetworkEndpointGroups)
+					}
+					negName, ok := negNames[strconv.Itoa(int(porterPort))]
+					if !ok {
+						t.Fatalf("DestinationRule annotation doesn't contain the desired NEG status, want: %d, have: %v", porterPort, negNames)
+					}
+					if err := e2e.WaitForNegs(ctx, Framework.Cloud, negName, zones, false, endpointCount); err != nil {
+						t.Errorf("Failed to wait Negs, error: %s", err)
+					}
+
+				}
+
+			}
+
+		})
+	})
+}
+
+func TestNoIstioASM(t *testing.T) {
+
+	Framework.RunWithSandbox("TestASMConfigOnNoIstioCluster", t, func(t *testing.T, s *e2e.Sandbox) {
+
+		cm := map[string]string{"enable-asm": "true"}
+		wantConfigMapEvents := []string{"ConfigMapConfigController: Get a update on the ConfigMapConfig, Restarting Ingress controller",
+			"Cannot find DestinationRule CRD, disabling ASM Mode, please check Istio setup."}
+
+		if err := e2e.UpdateCreateConfigMap(s, asmConfigNamespace, asmConfigName, cm); err != nil {
+			t.Error(err)
+		}
+		defer e2e.DeleteConfigMap(s, asmConfigNamespace, asmConfigName)
+		if err := e2e.WaitConfigMapEvents(s, asmConfigNamespace, asmConfigName, wantConfigMapEvents, negControllerRestartTimeout); err != nil {
+			t.Fatalf("Failed to get events: %v; Error %e", wantConfigMapEvents, err)
+		}
+
+		if err := wait.Poll(5*time.Second, 1*time.Minute, func() (bool, error) {
+			cmData, err := e2e.GetConfigMap(s, asmConfigNamespace, asmConfigName)
+			if err != nil {
+				return false, err
+			}
+			if val, ok := cmData["enable-asm"]; ok && val == "false" {
+				return true, nil
+			}
+			return false, fmt.Errorf("ConfigMap: %s/%s, nable-asm is not false. Value: %v", asmConfigNamespace, asmConfigName, cmData)
+
+		}); err != nil {
+			t.Fatalf("Failed to validate enable-asm = false. Error: %s", err)
+		}
+
+	})
+}
+
+func createPorterDeployment(namespace, name string, replics int32, lables map[string]string, port int32) *apiappsv1.Deployment {
+	env := fmt.Sprintf("SERVE_PORT_%d", port)
+	deployment := apiappsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
+		Spec: apiappsv1.DeploymentSpec{
+			Replicas: &replics,
+			Selector: &metav1.LabelSelector{MatchLabels: lables},
+			Template: apiv1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{Labels: lables},
+				Spec: apiv1.PodSpec{
+					Containers: []apiv1.Container{
+						{
+							Name:  "hostname",
+							Image: "gcr.io/kubernetes-e2e-test-images/porter-alpine:1.0",
+							Env:   []apiv1.EnvVar{{Name: env, Value: env}},
+							Ports: []apiv1.ContainerPort{{Name: "server", ContainerPort: port}},
+						},
+					},
+				},
+			},
+		},
+	}
+	return &deployment
+}
+
+func createService(namespace, name string, selector map[string]string, port int32) *apiv1.Service {
+	svc := apiv1.Service{
+		ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
+		Spec: apiv1.ServiceSpec{
+			Selector: selector,
+			Ports: []apiv1.ServicePort{
+				{
+					Port: port,
+					Name: "http",
+				},
+			},
+		},
+	}
+	return &svc
+}

cmd/e2e-test/main_test.go

@@ -49,6 +49,7 @@ var (
 		handleSIGINT        bool
 		gceEndpointOverride string
 		createILBSubnet     bool
+		enableIstio         bool
 	}
 
 	Framework *e2e.Framework
@@ -71,6 +72,7 @@ func init() {
 	flag.BoolVar(&flags.handleSIGINT, "handleSIGINT", true, "catch SIGINT to perform clean")
 	flag.StringVar(&flags.gceEndpointOverride, "gce-endpoint-override", "", "If set, talks to a different GCE API Endpoint. By default it talks to https://www.googleapis.com/compute/v1/")
 	flag.BoolVar(&flags.createILBSubnet, "createILBSubnet", false, "If set, creates a proxy subnet for the L7 ILB")
+	flag.BoolVar(&flags.enableIstio, "enable-istio", false, "set to true if Istio is enabled.")
 }
 
 // TestMain is the entrypoint for the end-to-end test suite. This is where
@@ -126,6 +128,7 @@ func TestMain(m *testing.M) {
 		DestroySandboxes:    flags.destroySandboxes,
 		GceEndpointOverride: flags.gceEndpointOverride,
 		CreateILBSubnet:     flags.createILBSubnet,
+		EnableIstio:         flags.enableIstio,
 	})
 	if flags.handleSIGINT {
 		Framework.CatchSIGINT()

cmd/e2e-test/neg_test.go

@@ -21,7 +21,7 @@ import (
 	"testing"
 
 	apps "k8s.io/api/apps/v1"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/api/networking/v1beta1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/sets"