-
Notifications
You must be signed in to change notification settings - Fork 303
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #980 from cadmuxe/asm-e2e
Add e2e tests for NEG asm mode.
- Loading branch information
Showing
7 changed files
with
470 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,233 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "strconv" | ||
| "strings" | ||
| "testing" | ||
| "time" | ||
|
|
||
| istioV1alpha3 "istio.io/api/networking/v1alpha3" | ||
| apiv1 "k8s.io/api/core/v1" | ||
| "k8s.io/apimachinery/pkg/util/wait" | ||
| "k8s.io/ingress-gce/pkg/e2e" | ||
| "k8s.io/klog" | ||
| ) | ||
|
|
||
| const ( | ||
| asmConfigNamespace = "kube-system" | ||
| asmConfigName = "ingress-controller-asm-cm-config" | ||
| negControllerRestartTimeout = 5 * time.Minute | ||
| ) | ||
|
|
||
| // TestASMConfig tests the ASM enable/disable, it can't run parallel with other tests. | ||
| func TestASMConfig(t *testing.T) { | ||
| Framework.RunWithSandbox("TestASMConfig", t, func(t *testing.T, s *e2e.Sandbox) { | ||
| for _, tc := range []struct { | ||
| desc string | ||
| configMap map[string]string | ||
| wantConfigMapEvents []string | ||
| }{ | ||
| { | ||
| desc: "Invalid ConfigMap value equals to disable", | ||
| configMap: map[string]string{"enable-asm": "INVALID"}, | ||
| wantConfigMapEvents: []string{"The map provided a unvalid value for field: enable-asm, value: INVALID"}, | ||
| }, | ||
| { | ||
| desc: "Invalid ConfigMap filed equals to disable", | ||
| configMap: map[string]string{"enable-unknow-feild": "INVALID1"}, | ||
| wantConfigMapEvents: []string{"The map contains a unknown key-value pair: enable-unknow-feild:INVALID1"}, | ||
| }, | ||
| { | ||
| desc: "Set enable-asm to true should restart the controller", | ||
| configMap: map[string]string{"enable-asm": "true"}, | ||
| wantConfigMapEvents: []string{"ConfigMapConfigController: Get a update on the ConfigMapConfig, Restarting Ingress controller"}, | ||
| }, | ||
| // TODO(koonwah): The below case is not fully tested, should update the neg controller to include a enable-asm event to the target CM. | ||
| { | ||
| desc: "Invalid ConfigMap value equals to disable", | ||
| configMap: map[string]string{"enable-asm": "INVALID2"}, | ||
| wantConfigMapEvents: []string{"The map provided a unvalid value for field: enable-asm, value: INVALID2", | ||
| "ConfigMapConfigController: Get a update on the ConfigMapConfig, Restarting Ingress controller"}, | ||
| }, | ||
| } { | ||
| t.Logf("Running test case: %s", tc.desc) | ||
| if err := e2e.EnsureConfigMap(s, asmConfigNamespace, asmConfigName, tc.configMap); err != nil { | ||
| t.Errorf("Failed to ensure ConfigMap, error: %s", err) | ||
| } | ||
| if err := e2e.WaitConfigMapEvents(s, asmConfigNamespace, asmConfigName, tc.wantConfigMapEvents, negControllerRestartTimeout); err != nil { | ||
| t.Fatalf("Failed to get events: %v; Error %e", strings.Join(tc.wantConfigMapEvents, ";"), err) | ||
| } | ||
| } | ||
| }) | ||
| } | ||
|
|
||
| // TestASMServiceAndDestinationRule tests the service/destinationrule, it can't run parallel with other tests. | ||
| func TestASMServiceAndDestinationRule(t *testing.T) { | ||
| _ = istioV1alpha3.DestinationRule{} | ||
| _ = apiv1.ComponentStatus{} | ||
|
|
||
| // This test case will need two namespaces, one will in asm-skip-namespaces. | ||
| Framework.RunWithSandbox("TestASMServiceAndDestinationRule", t, func(t *testing.T, sSkip *e2e.Sandbox) { | ||
| Framework.RunWithSandbox("TestASMServiceAndDestinationRule", t, func(t *testing.T, s *e2e.Sandbox) { | ||
| // Enable ASM mode | ||
| ctx := context.Background() | ||
|
|
||
| asmConfig := map[string]string{"enable-asm": "true", | ||
| "asm-skip-namespaces": fmt.Sprintf("kube-system,istio-system,%s", sSkip.Namespace)} | ||
| if err := e2e.EnsureConfigMap(s, asmConfigNamespace, asmConfigName, | ||
| asmConfig); err != nil { | ||
| t.Error(err) | ||
| } | ||
|
|
||
| var porterPort int32 | ||
| porterPort = 80 | ||
| svcName := "service" | ||
| svcSkipName := "service-skip" | ||
|
|
||
| // Create deployments used by the Service and DestinationRules. | ||
| // The deployment will contain two label pairs: {"app": "porter", "version": "*"} | ||
| // Different versions will be used as DestinationRule: subset | ||
| for _, deployment := range []struct { | ||
| deploymentName string | ||
| replics int32 | ||
| version string | ||
| }{ | ||
| {deploymentName: "deployment-v1", replics: 1, version: "v1"}, | ||
| {deploymentName: "deployment-v2", replics: 2, version: "v2"}, | ||
| {deploymentName: "deployment-v3", replics: 3, version: "v3"}, | ||
| } { | ||
| if err := e2e.CreatePorterDeployment(s, deployment.deploymentName, deployment.replics, deployment.version); err != nil { | ||
| t.Errorf("Failed to create deployment, Error: %s", err) | ||
| } | ||
| } | ||
|
|
||
| // Create and validate DestinationRules level NEGs, NEG controller shouldn't create DestinationRules level NEGs for those services in the skip namespace. | ||
| for _, svc := range []struct { | ||
| desc string | ||
| svcName string | ||
| inSkipNamespace bool | ||
| }{ | ||
| {desc: "NEG Controller should create NEGs for all ports for a service by default", svcName: svcName, inSkipNamespace: false}, | ||
| {desc: "NEG Controller shouldn't create NEGs for all ports for a service if it's in a skip namespace", svcName: svcSkipName, inSkipNamespace: true}, | ||
| } { | ||
| t.Logf("Running test case: %s", svc.desc) | ||
| sandbox := s | ||
| noPresentTest := false | ||
| if svc.inSkipNamespace { | ||
| sandbox = sSkip | ||
| noPresentTest = true | ||
| } | ||
| if err := e2e.CreatePorterService(sandbox, svc.svcName); err != nil { | ||
| t.Errorf("Failed to create service, Error: %s", err) | ||
| } | ||
|
|
||
| // Test the Service Annotations | ||
| negStatus, err := e2e.WaitForNegStatus(sandbox, svc.svcName, []string{strconv.Itoa(int(porterPort))}, noPresentTest) | ||
| if err != nil { | ||
| t.Errorf("Failed to wait for Service NEGAnnotation, error: %s", err) | ||
| } | ||
| if svc.inSkipNamespace { | ||
| if negStatus != nil { | ||
| t.Errorf("Service: %s/%s is in the ASM skip namespace, shoudln't have NEG Status. ASM Config: %v, NEGStatus got: %v", | ||
| sandbox.Namespace, svc.svcName, asmConfig, negStatus) | ||
| } | ||
| } else { | ||
| if negName, ok := negStatus.NetworkEndpointGroups[strconv.Itoa(int(porterPort))]; ok { | ||
| // No backend pod exists, so the NEG has 0 endpoint. | ||
| if err := e2e.WaitForNegs(ctx, Framework.Cloud, negName, negStatus.Zones, false, 0); err != nil { | ||
| t.Errorf("Failed to wait Negs, error: %s", err) | ||
| } | ||
| } else { | ||
| t.Fatalf("Service annotation doesn't contain the desired NEG status, want: %d, have: %v", porterPort, negStatus.NetworkEndpointGroups) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| for _, tc := range []struct { | ||
| desc string | ||
| destinationRuleName string | ||
| subsetEndpointCountMap map[string]int | ||
| crossNamespace bool // If crossNamespace set, the DestinationRule will use full Host(Service) name to refer a service in a different namespace. | ||
| }{ | ||
| {desc: "NEG controller should create NEGs for destinationrule", destinationRuleName: "porter-destinationrule", subsetEndpointCountMap: map[string]int{"v1": 1, "v2": 2, "v3": 3}, crossNamespace: false}, | ||
| {desc: "NEG controller should update NEGs for destinationrule", destinationRuleName: "porter-destinationrule", subsetEndpointCountMap: map[string]int{"v1": 1, "v2": 2}, crossNamespace: false}, | ||
| {desc: "NEG controller should create NEGs for cross namespace destinationrule", destinationRuleName: "porter-destinationrule-1", subsetEndpointCountMap: map[string]int{"v1": 1}, crossNamespace: true}, | ||
| } { | ||
| t.Logf("Running test case: %s", tc.desc) | ||
| sandbox := s | ||
| drHost := svcName | ||
| // crossNamespace will test DestinationRules that refering a serive located in a different namespace | ||
| if tc.crossNamespace { | ||
| sandbox = sSkip | ||
| drHost = fmt.Sprintf("%s.%s.svc.cluster.local", svcName, s.Namespace) | ||
| } | ||
|
|
||
| versions := []string{} | ||
| for k := range tc.subsetEndpointCountMap { | ||
| versions = append(versions, k) | ||
| } | ||
| if err := e2e.EnsurePorterDestinationRule(sandbox, tc.destinationRuleName, drHost, versions); err != nil { | ||
| klog.Errorf("Failed to create destinationRule, error: %s", err) | ||
| } | ||
|
|
||
| // One DestinationRule should have count(NEGs) = count(subset)* count(port) | ||
| dsNEGStatus, err := e2e.WaitDestinationRuleAnnotation(sandbox, sandbox.Namespace, tc.destinationRuleName, len(tc.subsetEndpointCountMap)*1, 5*time.Minute) | ||
| if err != nil { | ||
| klog.Errorf("Failed to validate the NEG count. Error: %s", err) | ||
| } | ||
|
|
||
| zones := dsNEGStatus.Zones | ||
| for subsetVersion, endpointCount := range tc.subsetEndpointCountMap { | ||
| negNames, ok := dsNEGStatus.NetworkEndpointGroups[subsetVersion] | ||
| if !ok { | ||
| t.Fatalf("DestinationRule annotation doesn't contain the desired NEG status, want: %s, have: %v", subsetVersion, dsNEGStatus.NetworkEndpointGroups) | ||
| } | ||
| negName, ok := negNames[strconv.Itoa(int(porterPort))] | ||
| if !ok { | ||
| t.Fatalf("DestinationRule annotation doesn't contain the desired NEG status, want: %d, have: %v", porterPort, negNames) | ||
| } | ||
| if err := e2e.WaitForNegs(ctx, Framework.Cloud, negName, zones, false, endpointCount); err != nil { | ||
| t.Errorf("Failed to wait Negs, error: %s", err) | ||
| } | ||
| } | ||
|
|
||
| } | ||
|
|
||
| }) | ||
| }) | ||
| } | ||
|
|
||
| func TestNoIstioASM(t *testing.T) { | ||
|
|
||
| Framework.RunWithSandbox("TestASMConfigOnNoIstioCluster", t, func(t *testing.T, s *e2e.Sandbox) { | ||
|
|
||
| cm := map[string]string{"enable-asm": "true"} | ||
| wantConfigMapEvents := []string{"ConfigMapConfigController: Get a update on the ConfigMapConfig, Restarting Ingress controller", | ||
| "Cannot find DestinationRule CRD, disabling ASM Mode, please check Istio setup."} | ||
|
|
||
| if err := e2e.EnsureConfigMap(s, asmConfigNamespace, asmConfigName, cm); err != nil { | ||
| t.Error(err) | ||
| } | ||
| defer e2e.DeleteConfigMap(s, asmConfigNamespace, asmConfigName) | ||
| if err := e2e.WaitConfigMapEvents(s, asmConfigNamespace, asmConfigName, wantConfigMapEvents, negControllerRestartTimeout); err != nil { | ||
| t.Fatalf("Failed to get events: %v; Error %e", wantConfigMapEvents, err) | ||
| } | ||
|
|
||
| if err := wait.Poll(5*time.Second, 1*time.Minute, func() (bool, error) { | ||
| cmData, err := e2e.GetConfigMap(s, asmConfigNamespace, asmConfigName) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| if val, ok := cmData["enable-asm"]; ok && val == "false" { | ||
| return true, nil | ||
| } | ||
| return false, fmt.Errorf("ConfigMap: %s/%s, nable-asm is not false. Value: %v", asmConfigNamespace, asmConfigName, cmData) | ||
|
|
||
| }); err != nil { | ||
| t.Fatalf("Failed to validate enable-asm = false. Error: %s", err) | ||
| } | ||
|
|
||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.