From 21e456702c4f261ff7dbb9d671ba750c0aaf745a Mon Sep 17 00:00:00 2001 From: Chinmay Gadgil Date: Thu, 6 May 2021 13:21:24 -0700 Subject: [PATCH 1/8] Added binary support to validate MTU value and Veth prefix --- test/agent/cmd/aws-node-config/main.go | 92 ++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 test/agent/cmd/aws-node-config/main.go diff --git a/test/agent/cmd/aws-node-config/main.go b/test/agent/cmd/aws-node-config/main.go new file mode 100644 index 0000000000..942d5805dc --- /dev/null +++ b/test/agent/cmd/aws-node-config/main.go @@ -0,0 +1,92 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. +package main + +import ( + "flag" + "fmt" + "log" + "net" + "strings" +) + +const ( + MTU_CHECK = "mtu" + VETH_PREFIX_CHECK = "veth" +) + +func main() { + validate := "" + prefix := "eni" + vethCount := 1 + mtuVal := 9001 + + flag.StringVar(&validate, "validate", "", "name of the test, valid values: mtu, veth") + flag.StringVar(&prefix, "prefix", "eni", "veth prefix to be used, specify when validating veth") + flag.IntVar(&vethCount, "veth-count", 1, "Number of expected veth links to have the new prefix. specify when validating veth") + flag.IntVar(&mtuVal, "mtu-val", 9001, "Expected MTU values, specify when validating mtu") + flag.Parse() + + switch validate { + case MTU_CHECK: + validateMTUValue(mtuVal) + case VETH_PREFIX_CHECK: + validateVethPairCount(prefix, vethCount) + default: + fmt.Println("Invalid test-name choice") + } +} + +func validateVethPairCount(prefix string, expected int) { + interfaces, err := net.Interfaces() + if err != nil { + log.Fatal(err) + } + + actual := 0 + for _, ni := range interfaces { + if strings.HasPrefix(ni.Name, prefix) { + actual++ + } + } + + if actual >= expected { + log.Println("Veth Check: Passed") + } else { + log.Println("Veth Check: Failed") + } +} + +func validateMTUValue(expected int) { + interfaces, err := net.Interfaces() + if err != nil { + log.Fatal(err) + } + + ok := false + for _, ni := range interfaces { + if strings.HasPrefix(ni.Name, "eth") { + if ni.MTU != expected { + log.Fatal("MTU Check: Failed") + } else { + ok = true + } + } + } + if !ok { + log.Println("MTU Check: Failed") + log.Println("Could not find eth interface, check ifconfig on the node") + } else { + log.Println("MTU Check: Passed") + } +} From b294acf586e2c86001bcfd7003146a0f023418c2 Mon Sep 17 00:00:00 2001 From: Chinmay Gadgil Date: Thu, 6 May 2021 13:25:49 -0700 Subject: [PATCH 2/8] Updated Dockerfile to include aws-node-config binary --- test/agent/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/agent/Dockerfile b/test/agent/Dockerfile index ad614e9f9a..f72ee6a7e6 100644 --- a/test/agent/Dockerfile +++ b/test/agent/Dockerfile @@ -26,6 +26,9 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build \ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build \ -a -o metric-server cmd/metric-server/main.go +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build \ + -a -o aws-node-config cmd/aws-node-config/main.go + FROM public.ecr.aws/amazonlinux/amazonlinux:2 RUN yum update -y && \ yum clean all From 3b4c92f2e11f197bfaaf9b80dd15ce242b12dccd Mon Sep 17 00:00:00 2001 From: Chinmay Gadgil Date: Wed, 12 May 2021 11:26:27 -0700 Subject: [PATCH 3/8] Added base64 deserialization logic Improved logic to test veth pairs --- test/agent/cmd/aws-node-config/main.go | 102 ++++++++++++++++++++----- 1 file changed, 83 insertions(+), 19 deletions(-) diff --git a/test/agent/cmd/aws-node-config/main.go b/test/agent/cmd/aws-node-config/main.go index 942d5805dc..d164866ab7 100644 --- a/test/agent/cmd/aws-node-config/main.go +++ b/test/agent/cmd/aws-node-config/main.go @@ -13,11 +13,17 @@ package main import ( + "crypto/sha1" + "encoding/base64" + "encoding/hex" + "encoding/json" "flag" "fmt" "log" "net" "strings" + + "github.com/aws/amazon-vpc-cni-k8s/test/agent/pkg/input" ) const ( @@ -25,68 +31,126 @@ const ( VETH_PREFIX_CHECK = "veth" ) +type MTU_CONFIG struct { + ExpectedVal int +} + func main() { validate := "" - prefix := "eni" - vethCount := 1 - mtuVal := 9001 + config := "" flag.StringVar(&validate, "validate", "", "name of the test, valid values: mtu, veth") - flag.StringVar(&prefix, "prefix", "eni", "veth prefix to be used, specify when validating veth") - flag.IntVar(&vethCount, "veth-count", 1, "Number of expected veth links to have the new prefix. specify when validating veth") - flag.IntVar(&mtuVal, "mtu-val", 9001, "Expected MTU values, specify when validating mtu") + flag.StringVar(&config, "config", "", "Base64 encoded json object for the given test") flag.Parse() + decoded, err := base64.StdEncoding.DecodeString(config) + if err != nil { + log.Fatalf("failed to descode base64 string %s: %v", config, err) + } + switch validate { case MTU_CHECK: - validateMTUValue(mtuVal) + validateMTUValue(decoded) case VETH_PREFIX_CHECK: - validateVethPairCount(prefix, vethCount) + validateVethPairCount(decoded) default: fmt.Println("Invalid test-name choice") } } -func validateVethPairCount(prefix string, expected int) { +func validateVethPairCount(config []byte) { + var podNetworkingValidationInput input.PodNetworkingValidationInput + err := json.Unmarshal(config, &podNetworkingValidationInput) + if err != nil { + log.Fatalf("failed to unmarshall json string %s: %v", config, err) + } + + log.Printf("podNetworkingValidationInput: %+v\n", podNetworkingValidationInput) + + if podNetworkingValidationInput.VethPrefix == "" { + podNetworkingValidationInput.VethPrefix = "eni" + } + + actual := 0 + vethPairs := make(map[string]bool, len(podNetworkingValidationInput.PodList)) + + for _, pod := range podNetworkingValidationInput.PodList { + // Get the veth pair for pod in host network namespace + hostVethName := getHostVethPairName(pod, podNetworkingValidationInput.VethPrefix) + log.Printf("hostVethName: %v\n", hostVethName) + vethPairs[hostVethName] = false + } + interfaces, err := net.Interfaces() if err != nil { log.Fatal(err) } - actual := 0 for _, ni := range interfaces { - if strings.HasPrefix(ni.Name, prefix) { - actual++ + if checked, ok := vethPairs[ni.Name]; ok { + if !checked { + actual++ + } } } - if actual >= expected { + if actual == len(podNetworkingValidationInput.PodList) { log.Println("Veth Check: Passed") } else { log.Println("Veth Check: Failed") + log.Println("Missing Veth pairs") + for name, val := range vethPairs { + if !val { + log.Println(name) + } + } } } -func validateMTUValue(expected int) { +func validateMTUValue(config []byte) { + mtuConfig := MTU_CONFIG{} + err := json.Unmarshal(config, &mtuConfig) + + if err != nil { + log.Fatalf("failed to unmarshall json string %s: %v", config, err) + } + + log.Printf("MTU_CONFIG: %+v\n", mtuConfig) + + if mtuConfig.ExpectedVal == 0 { + mtuConfig.ExpectedVal = 9001 + } + interfaces, err := net.Interfaces() if err != nil { log.Fatal(err) } ok := false + failedInterfaces := make([]string, 0) for _, ni := range interfaces { if strings.HasPrefix(ni.Name, "eth") { - if ni.MTU != expected { - log.Fatal("MTU Check: Failed") - } else { - ok = true + if ni.MTU != mtuConfig.ExpectedVal { + failedInterfaces = append(failedInterfaces, ni.Name) } + ok = true } } if !ok { log.Println("MTU Check: Failed") log.Println("Could not find eth interface, check ifconfig on the node") } else { - log.Println("MTU Check: Passed") + if len(failedInterfaces) != 0 { + log.Println("MTU Check: Failed") + log.Printf("Failed Interfaces: %+v\n", failedInterfaces) + } else { + log.Println("MTU Check: Passed") + } } } + +func getHostVethPairName(input input.Pod, vethPrefix string) string { + h := sha1.New() + h.Write([]byte(fmt.Sprintf("%s.%s", input.PodNamespace, input.PodName))) + return fmt.Sprintf("%s%s", vethPrefix, hex.EncodeToString(h.Sum(nil))[:11]) +} From 75aaebc423c44f1f9f8edd45e6fc9240b1f1b461 Mon Sep 17 00:00:00 2001 From: Chinmay Gadgil Date: Thu, 13 May 2021 02:14:59 -0700 Subject: [PATCH 4/8] Removed Binary as it was redundant Migrated MTU and Veth check to host networking binary instead Updated env_vars_test file --- test/agent/cmd/aws-node-config/main.go | 156 ------------------ test/agent/cmd/networking/tester/network.go | 6 + test/agent/pkg/input/input.go | 2 + test/framework/utils/utils.go | 16 ++ .../cni/host_networking_test.go | 31 +++- test/integration-new/ipamd/env_vars_test.go | 134 ++------------- 6 files changed, 65 insertions(+), 280 deletions(-) delete mode 100644 test/agent/cmd/aws-node-config/main.go diff --git a/test/agent/cmd/aws-node-config/main.go b/test/agent/cmd/aws-node-config/main.go deleted file mode 100644 index d164866ab7..0000000000 --- a/test/agent/cmd/aws-node-config/main.go +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"). You may -// not use this file except in compliance with the License. A copy of the -// License is located at -// -// http://aws.amazon.com/apache2.0/ -// -// or in the "license" file accompanying this file. This file is distributed -// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language governing -// permissions and limitations under the License. -package main - -import ( - "crypto/sha1" - "encoding/base64" - "encoding/hex" - "encoding/json" - "flag" - "fmt" - "log" - "net" - "strings" - - "github.com/aws/amazon-vpc-cni-k8s/test/agent/pkg/input" -) - -const ( - MTU_CHECK = "mtu" - VETH_PREFIX_CHECK = "veth" -) - -type MTU_CONFIG struct { - ExpectedVal int -} - -func main() { - validate := "" - config := "" - - flag.StringVar(&validate, "validate", "", "name of the test, valid values: mtu, veth") - flag.StringVar(&config, "config", "", "Base64 encoded json object for the given test") - flag.Parse() - - decoded, err := base64.StdEncoding.DecodeString(config) - if err != nil { - log.Fatalf("failed to descode base64 string %s: %v", config, err) - } - - switch validate { - case MTU_CHECK: - validateMTUValue(decoded) - case VETH_PREFIX_CHECK: - validateVethPairCount(decoded) - default: - fmt.Println("Invalid test-name choice") - } -} - -func validateVethPairCount(config []byte) { - var podNetworkingValidationInput input.PodNetworkingValidationInput - err := json.Unmarshal(config, &podNetworkingValidationInput) - if err != nil { - log.Fatalf("failed to unmarshall json string %s: %v", config, err) - } - - log.Printf("podNetworkingValidationInput: %+v\n", podNetworkingValidationInput) - - if podNetworkingValidationInput.VethPrefix == "" { - podNetworkingValidationInput.VethPrefix = "eni" - } - - actual := 0 - vethPairs := make(map[string]bool, len(podNetworkingValidationInput.PodList)) - - for _, pod := range podNetworkingValidationInput.PodList { - // Get the veth pair for pod in host network namespace - hostVethName := getHostVethPairName(pod, podNetworkingValidationInput.VethPrefix) - log.Printf("hostVethName: %v\n", hostVethName) - vethPairs[hostVethName] = false - } - - interfaces, err := net.Interfaces() - if err != nil { - log.Fatal(err) - } - - for _, ni := range interfaces { - if checked, ok := vethPairs[ni.Name]; ok { - if !checked { - actual++ - } - } - } - - if actual == len(podNetworkingValidationInput.PodList) { - log.Println("Veth Check: Passed") - } else { - log.Println("Veth Check: Failed") - log.Println("Missing Veth pairs") - for name, val := range vethPairs { - if !val { - log.Println(name) - } - } - } -} - -func validateMTUValue(config []byte) { - mtuConfig := MTU_CONFIG{} - err := json.Unmarshal(config, &mtuConfig) - - if err != nil { - log.Fatalf("failed to unmarshall json string %s: %v", config, err) - } - - log.Printf("MTU_CONFIG: %+v\n", mtuConfig) - - if mtuConfig.ExpectedVal == 0 { - mtuConfig.ExpectedVal = 9001 - } - - interfaces, err := net.Interfaces() - if err != nil { - log.Fatal(err) - } - - ok := false - failedInterfaces := make([]string, 0) - for _, ni := range interfaces { - if strings.HasPrefix(ni.Name, "eth") { - if ni.MTU != mtuConfig.ExpectedVal { - failedInterfaces = append(failedInterfaces, ni.Name) - } - ok = true - } - } - if !ok { - log.Println("MTU Check: Failed") - log.Println("Could not find eth interface, check ifconfig on the node") - } else { - if len(failedInterfaces) != 0 { - log.Println("MTU Check: Failed") - log.Printf("Failed Interfaces: %+v\n", failedInterfaces) - } else { - log.Println("MTU Check: Passed") - } - } -} - -func getHostVethPairName(input input.Pod, vethPrefix string) string { - h := sha1.New() - h.Write([]byte(fmt.Sprintf("%s.%s", input.PodNamespace, input.PodName))) - return fmt.Sprintf("%s%s", vethPrefix, hex.EncodeToString(h.Sum(nil))[:11]) -} diff --git a/test/agent/cmd/networking/tester/network.go b/test/agent/cmd/networking/tester/network.go index de9ccfa929..864f13198f 100644 --- a/test/agent/cmd/networking/tester/network.go +++ b/test/agent/cmd/networking/tester/network.go @@ -69,6 +69,12 @@ func TestNetworkingSetupForRegularPod(podNetworkingValidationInput input.PodNetw continue } + // Validate MTU value + if link.Attrs().MTU != podNetworkingValidationInput.MTU { + validationErrors = append(validationErrors, + fmt.Errorf("failed to match expected MTU value %v for veth pair: %s", hostVethName, link.Attrs().MTU)) + } + // Verify IP Link for the Pod is UP isLinkUp := strings.Contains(link.Attrs().Flags.String(), "up") if !isLinkUp { diff --git a/test/agent/pkg/input/input.go b/test/agent/pkg/input/input.go index 1fb187b475..698e4d6dcf 100644 --- a/test/agent/pkg/input/input.go +++ b/test/agent/pkg/input/input.go @@ -32,6 +32,8 @@ type PodNetworkingValidationInput struct { VethPrefix string // List of pod to validate the networking PodList []Pod + // Expected MTU value + MTU int } type Pod struct { diff --git a/test/framework/utils/utils.go b/test/framework/utils/utils.go index 6fea0f723f..1670ebd669 100644 --- a/test/framework/utils/utils.go +++ b/test/framework/utils/utils.go @@ -14,10 +14,16 @@ package utils import ( + appsV1 "k8s.io/api/apps/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) +const ( + NAMESPACE = "kube-system" + DAEMONSET = "aws-node" +) + // NamespacedName returns the namespaced name for k8s objects func NamespacedName(obj v1.Object) types.NamespacedName { return types.NamespacedName{ @@ -25,3 +31,13 @@ func NamespacedName(obj v1.Object) types.NamespacedName { Name: obj.GetName(), } } + +func GetEnvValueForKeyFromDaemonSet(key string, ds *appsV1.DaemonSet) string { + envVar := ds.Spec.Template.Spec.Containers[0].Env + for _, env := range envVar { + if env.Name == key { + return env.Value + } + } + return "" +} diff --git a/test/integration-new/cni/host_networking_test.go b/test/integration-new/cni/host_networking_test.go index 71cd292818..50ccd53247 100644 --- a/test/integration-new/cni/host_networking_test.go +++ b/test/integration-new/cni/host_networking_test.go @@ -16,12 +16,14 @@ package cni import ( "encoding/json" "fmt" + "strconv" "time" "github.com/aws/amazon-vpc-cni-k8s/test/agent/pkg/input" v1 "k8s.io/api/core/v1" "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" . "github.com/onsi/ginkgo" @@ -39,6 +41,13 @@ const ( NetworkingSetupFails ) +const ( + AWS_VPC_ENI_MTU = "AWS_VPC_ENI_MTU" + AWS_VPC_K8S_CNI_VETHPREFIX = "AWS_VPC_K8S_CNI_VETHPREFIX" + NEW_MTU_VAL = 1300 + NEW_VETH_PREFIX = "veth" +) + var _ = Describe("test host networking", func() { var err error var podLabelKey = "app" @@ -54,6 +63,18 @@ var _ = Describe("test host networking", func() { NodeName(primaryNode.Name). Build() + By("Configuring Veth Prefix and MTU value on aws-node daemonset") + ds, err := f.K8sResourceManagers.DaemonSetManager().GetDaemonSet(utils.NAMESPACE, utils.DAEMONSET) + Expect(err).NotTo(HaveOccurred()) + + oldMTU := utils.GetEnvValueForKeyFromDaemonSet(AWS_VPC_ENI_MTU, ds) + oldVethPrefix := utils.GetEnvValueForKeyFromDaemonSet(AWS_VPC_K8S_CNI_VETHPREFIX, ds) + + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.DAEMONSET, utils.NAMESPACE, utils.DAEMONSET, map[string]string{ + AWS_VPC_ENI_MTU: strconv.Itoa(NEW_MTU_VAL), + AWS_VPC_K8S_CNI_VETHPREFIX: NEW_VETH_PREFIX, + }) + By("creating a deployment to launch pod using primary and secondary ENI IP") deployment, err = f.K8sResourceManagers.DeploymentManager(). CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout) @@ -86,6 +107,13 @@ var _ = Describe("test host networking", func() { By("validating host networking is teared down correctly") ValidateHostNetworking(NetworkingTearDownSucceeds, input) + + // Restore MTU and Veth Prefix + By("Restoring MTU value and Veth Prefix to old values") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.DAEMONSET, utils.NAMESPACE, utils.DAEMONSET, map[string]string{ + AWS_VPC_ENI_MTU: oldMTU, + AWS_VPC_K8S_CNI_VETHPREFIX: oldVethPrefix, + }) }) }) @@ -198,8 +226,9 @@ func GetPodNetworkingValidationInput(interfaceTypeToPodList InterfaceTypeToPodLi ip := input.PodNetworkingValidationInput{ VPCCidrRange: vpcCIDRs, - VethPrefix: "eni", + VethPrefix: NEW_VETH_PREFIX, PodList: []input.Pod{}, + MTU: NEW_MTU_VAL, } for _, primaryENIPod := range interfaceTypeToPodList.PodsOnPrimaryENI { diff --git a/test/integration-new/ipamd/env_vars_test.go b/test/integration-new/ipamd/env_vars_test.go index 32f0de50e2..ae4f6c1786 100644 --- a/test/integration-new/ipamd/env_vars_test.go +++ b/test/integration-new/ipamd/env_vars_test.go @@ -1,8 +1,6 @@ package ipamd import ( - "regexp" - "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" @@ -13,89 +11,19 @@ import ( ) var ( - ds *appsV1.DaemonSet - hostNetworkDeploymentSpec *appsV1.Deployment - hostNetworkPod v1.Pod + ds *appsV1.DaemonSet ) const ( - AWS_VPC_ENI_MTU = "AWS_VPC_ENI_MTU" - AWS_VPC_K8S_CNI_LOG_FILE = "AWS_VPC_K8S_CNI_LOG_FILE" - AWS_VPC_K8S_CNI_VETHPREFIX = "AWS_VPC_K8S_CNI_VETHPREFIX" - POD_VOL_LABEL_KEY = "MountVolume" - POD_VOL_LABEL_VAL = "true" - VOLUME_NAME = "ipamd-logs" - VOLUME_MOUNT_PATH = "/var/log/aws-routed-eni/" + AWS_VPC_K8S_CNI_LOG_FILE = "AWS_VPC_K8S_CNI_LOG_FILE" + POD_VOL_LABEL_KEY = "MountVolume" + POD_VOL_LABEL_VAL = "true" + VOLUME_NAME = "ipamd-logs" + VOLUME_MOUNT_PATH = "/var/log/aws-routed-eni/" ) var _ = Describe("cni env test", func() { Context("CNI Environment Variables", func() { - It("Changing AWS_VPC_ENI_MTU and AWS_VPC_K8S_CNI_VETHPREFIX", func() { - setupHostNetworkPod() - - currMTUVal := getEnvValueForKey(AWS_VPC_ENI_MTU) - Expect(currMTUVal).NotTo(Equal("")) - - currVETHPrefix := getEnvValueForKey(AWS_VPC_K8S_CNI_VETHPREFIX) - Expect(currVETHPrefix).NotTo(Equal("")) - - k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, DAEMONSET, NAMESPACE, DAEMONSET, map[string]string{ - AWS_VPC_ENI_MTU: "1300", - AWS_VPC_K8S_CNI_VETHPREFIX: "veth", - }) - - By("Deploying a BusyBox deployment") - deploymentSpec := manifest.NewBusyBoxDeploymentBuilder(). - Namespace("default"). - Name("busybox"). - Replicas(1). - NodeName(primaryNode.Name). - Build() - - _, err := f.K8sResourceManagers. - DeploymentManager(). - CreateAndWaitTillDeploymentIsReady(deploymentSpec, utils.DefaultDeploymentReadyTimeout) - Expect(err).ToNot(HaveOccurred()) - - stdout, _, err := f.K8sResourceManagers.PodManager().PodExec("default", hostNetworkPod.Name, []string{"ifconfig"}) - Expect(err).NotTo(HaveOccurred()) - - re := regexp.MustCompile(`\n`) - input := re.ReplaceAllString(stdout, "") - - re = regexp.MustCompile(`eth.*lo`) - eth := re.FindStringSubmatch(input)[0] - - re = regexp.MustCompile(`MTU:[0-9]*`) - mtus := re.FindAllStringSubmatch(eth, -1) - - By("Validating new MTU value") - // Validate MTU - for _, m := range mtus { - Expect(m[0]).To(Equal("MTU:1300")) - } - - By("Validating new VETH Prefix") - // Validate VETH Prefix - // Adding the new MTU value to below regex ensures that we are checking the recently created - // veth and not any older entries - re = regexp.MustCompile(`veth.*MTU:1300`) - veth := re.FindAllString(input, -1) - - Expect(len(veth)).NotTo(Equal(0)) - - By("Deleting BusyBox Deployment") - err = f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(deploymentSpec) - Expect(err).NotTo(HaveOccurred()) - - By("Restoring old value on daemonset") - restoreOldValues(map[string]string{ - AWS_VPC_ENI_MTU: currMTUVal, - AWS_VPC_K8S_CNI_VETHPREFIX: currVETHPrefix, - }) - cleanupHostNetworkPod() - }) - It("Changing AWS_VPC_K8S_CNI_LOG_FILE", func() { By("Deploying a host network deployment with Volume mount") curlContainer := manifest.NewBusyBoxContainerBuilder().Image("curlimages/curl:7.76.1").Name("curler").Build() @@ -139,7 +67,11 @@ var _ = Describe("cni env test", func() { Expect(len(pods.Items)).Should(BeNumerically(">", 0)) podWithVol := pods.Items[0] - currLogFilepath := getEnvValueForKey(AWS_VPC_K8S_CNI_LOG_FILE) + + ds, err = f.K8sResourceManagers.DaemonSetManager().GetDaemonSet(NAMESPACE, DAEMONSET) + Expect(err).NotTo(HaveOccurred()) + + currLogFilepath := utils.GetEnvValueForKeyFromDaemonSet(AWS_VPC_K8S_CNI_LOG_FILE, ds) Expect(currLogFilepath).NotTo(Equal("")) newLogFile := "ipamd_test.log" @@ -163,50 +95,6 @@ var _ = Describe("cni env test", func() { }) }) -func getEnvValueForKey(key string) string { - ds, err = f.K8sResourceManagers.DaemonSetManager().GetDaemonSet(NAMESPACE, DAEMONSET) - Expect(err).NotTo(HaveOccurred()) - - envVar := ds.Spec.Template.Spec.Containers[0].Env - for _, env := range envVar { - if env.Name == key { - return env.Value - } - } - return "" -} - -func setupHostNetworkPod() { - By("Deploying a Host Network Pod") - curlContainer := manifest.NewBusyBoxContainerBuilder().Image("curlimages/curl:7.76.1").Name("curler").Build() - - hostNetworkDeploymentSpec = manifest.NewDefaultDeploymentBuilder(). - Namespace("default"). - Name("host-network"). - Replicas(1). - HostNetwork(true). - Container(curlContainer). - PodLabel(HOST_POD_LABEL_KEY, HOST_POD_LABEL_VAL). - NodeName(primaryNode.Name). - Build() - - _, err := f.K8sResourceManagers. - DeploymentManager(). - CreateAndWaitTillDeploymentIsReady(hostNetworkDeploymentSpec, utils.DefaultDeploymentReadyTimeout) - Expect(err).NotTo(HaveOccurred()) - - pods, err := f.K8sResourceManagers.PodManager().GetPodsWithLabelSelector(HOST_POD_LABEL_KEY, HOST_POD_LABEL_VAL) - Expect(err).NotTo(HaveOccurred()) - - hostNetworkPod = pods.Items[0] -} - -func cleanupHostNetworkPod() { - err = f.K8sResourceManagers.DeploymentManager(). - DeleteAndWaitTillDeploymentIsDeleted(hostNetworkDeploymentSpec) - Expect(err).ToNot(HaveOccurred()) -} - func restoreOldValues(oldVals map[string]string) { k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, DAEMONSET, NAMESPACE, DAEMONSET, oldVals) } From be4c7e0fc985fe50c282fdff80ef1be794116e94 Mon Sep 17 00:00:00 2001 From: Chinmay Gadgil Date: Thu, 13 May 2021 11:22:15 -0700 Subject: [PATCH 5/8] Separate MTU check from the default behavior --- test/agent/cmd/networking/tester/network.go | 4 +- test/agent/pkg/input/input.go | 12 +++ .../cni/host_networking_test.go | 96 +++++++++++++------ 3 files changed, 82 insertions(+), 30 deletions(-) diff --git a/test/agent/cmd/networking/tester/network.go b/test/agent/cmd/networking/tester/network.go index 864f13198f..14eb450e08 100644 --- a/test/agent/cmd/networking/tester/network.go +++ b/test/agent/cmd/networking/tester/network.go @@ -69,8 +69,8 @@ func TestNetworkingSetupForRegularPod(podNetworkingValidationInput input.PodNetw continue } - // Validate MTU value - if link.Attrs().MTU != podNetworkingValidationInput.MTU { + // Validate MTU value if it is set to true + if podNetworkingValidationInput.ValidateMTU && link.Attrs().MTU != podNetworkingValidationInput.MTU { validationErrors = append(validationErrors, fmt.Errorf("failed to match expected MTU value %v for veth pair: %s", hostVethName, link.Attrs().MTU)) } diff --git a/test/agent/pkg/input/input.go b/test/agent/pkg/input/input.go index 698e4d6dcf..f31f1ec596 100644 --- a/test/agent/pkg/input/input.go +++ b/test/agent/pkg/input/input.go @@ -13,6 +13,8 @@ package input +import "encoding/json" + type TestStatus struct { SuccessCount int FailureCount int @@ -32,6 +34,8 @@ type PodNetworkingValidationInput struct { VethPrefix string // List of pod to validate the networking PodList []Pod + // Should Validate MTU value, by default it will false + ValidateMTU bool // Expected MTU value MTU int } @@ -47,3 +51,11 @@ type Pod struct { // from the Secondary ENI IsIPFromSecondaryENI bool } + +func (ip PodNetworkingValidationInput) Serialize() (string, error) { + inputBytes, err := json.Marshal(ip) + if err != nil { + return "", err + } + return string(inputBytes), nil +} diff --git a/test/integration-new/cni/host_networking_test.go b/test/integration-new/cni/host_networking_test.go index 50ccd53247..f0d0e1ee67 100644 --- a/test/integration-new/cni/host_networking_test.go +++ b/test/integration-new/cni/host_networking_test.go @@ -14,7 +14,6 @@ package cni import ( - "encoding/json" "fmt" "strconv" "time" @@ -63,18 +62,6 @@ var _ = Describe("test host networking", func() { NodeName(primaryNode.Name). Build() - By("Configuring Veth Prefix and MTU value on aws-node daemonset") - ds, err := f.K8sResourceManagers.DaemonSetManager().GetDaemonSet(utils.NAMESPACE, utils.DAEMONSET) - Expect(err).NotTo(HaveOccurred()) - - oldMTU := utils.GetEnvValueForKeyFromDaemonSet(AWS_VPC_ENI_MTU, ds) - oldVethPrefix := utils.GetEnvValueForKeyFromDaemonSet(AWS_VPC_K8S_CNI_VETHPREFIX, ds) - - k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.DAEMONSET, utils.NAMESPACE, utils.DAEMONSET, map[string]string{ - AWS_VPC_ENI_MTU: strconv.Itoa(NEW_MTU_VAL), - AWS_VPC_K8S_CNI_VETHPREFIX: NEW_VETH_PREFIX, - }) - By("creating a deployment to launch pod using primary and secondary ENI IP") deployment, err = f.K8sResourceManagers.DeploymentManager(). CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout) @@ -92,9 +79,10 @@ var _ = Describe("test host networking", func() { Should(BeNumerically(">", 0)) By("generating the pod networking validation input to be passed to tester") - input := GetPodNetworkingValidationInput(interfaceTypeToPodList) + input, err := GetPodNetworkingValidationInput(interfaceTypeToPodList).Serialize() + Expect(err).NotTo(HaveOccurred()) - By("validating host networking setup is setup correctly") + By("validating host, networking setup is setup correctly") ValidateHostNetworking(NetworkingSetupSucceeds, input) By("deleting the deployment to test teardown") @@ -107,6 +95,46 @@ var _ = Describe("test host networking", func() { By("validating host networking is teared down correctly") ValidateHostNetworking(NetworkingTearDownSucceeds, input) + }) + + It("Validate Host Networking setup after changing MTU and Veth Prefix", func() { + deployment := manifest.NewBusyBoxDeploymentBuilder(). + Replicas(6). + PodLabel(podLabelKey, podLabelVal). + NodeName(primaryNode.Name). + Build() + + By("creating a deployment to launch pods") + deployment, err = f.K8sResourceManagers.DeploymentManager(). + CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout) + Expect(err).ToNot(HaveOccurred()) + + By("Configuring Veth Prefix and MTU value on aws-node daemonset") + ds, err := f.K8sResourceManagers.DaemonSetManager().GetDaemonSet(utils.NAMESPACE, utils.DAEMONSET) + Expect(err).NotTo(HaveOccurred()) + + oldMTU := utils.GetEnvValueForKeyFromDaemonSet(AWS_VPC_ENI_MTU, ds) + oldVethPrefix := utils.GetEnvValueForKeyFromDaemonSet(AWS_VPC_K8S_CNI_VETHPREFIX, ds) + + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.DAEMONSET, utils.NAMESPACE, utils.DAEMONSET, map[string]string{ + AWS_VPC_ENI_MTU: strconv.Itoa(NEW_MTU_VAL), + AWS_VPC_K8S_CNI_VETHPREFIX: NEW_VETH_PREFIX, + }) + + By("getting the list of pods using IP from primary and secondary ENI") + interfaceTypeToPodList := + GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal) + + By("generating the pod networking validation input to be passed to tester") + podNetworkingValidationInput := GetPodNetworkingValidationInput(interfaceTypeToPodList) + podNetworkingValidationInput.VethPrefix = NEW_VETH_PREFIX + podNetworkingValidationInput.ValidateMTU = true + podNetworkingValidationInput.MTU = NEW_MTU_VAL + input, err := podNetworkingValidationInput.Serialize() + Expect(err).NotTo(HaveOccurred()) + + By("validating host networking setup is setup correctly with MTU check as well") + ValidateHostNetworking(NetworkingSetupSucceedsAfterChangingMTU, input) // Restore MTU and Veth Prefix By("Restoring MTU value and Veth Prefix to old values") @@ -114,6 +142,17 @@ var _ = Describe("test host networking", func() { AWS_VPC_ENI_MTU: oldMTU, AWS_VPC_K8S_CNI_VETHPREFIX: oldVethPrefix, }) + + By("deleting the deployment to test teardown") + err = f.K8sResourceManagers.DeploymentManager(). + DeleteAndWaitTillDeploymentIsDeleted(deployment) + Expect(err).ToNot(HaveOccurred()) + + By("waiting to allow CNI to tear down networking for terminated pods") + time.Sleep(time.Second * 60) + + By("validating host networking is teared down correctly") + ValidateHostNetworking(NetworkingTearDownSucceeds, input) }) }) @@ -131,9 +170,10 @@ var _ = Describe("test host networking", func() { CreatAndWaitTillRunning(parkingPod) Expect(err).ToNot(HaveOccurred()) - validInput := GetPodNetworkingValidationInput(InterfaceTypeToPodList{ + validInput, err := GetPodNetworkingValidationInput(InterfaceTypeToPodList{ PodsOnPrimaryENI: []v1.Pod{*parkingPod}, - }) + }).Serialize() + Expect(err).NotTo(HaveOccurred()) By("first validating the tester work on valid input") ValidateHostNetworking(NetworkingSetupSucceeds, validInput) @@ -142,9 +182,11 @@ var _ = Describe("test host networking", func() { invalidPod := parkingPod.DeepCopy() invalidPod.Status.PodIP = "1.1.1.1" - invalidInput := GetPodNetworkingValidationInput(InterfaceTypeToPodList{ + invalidInput, err := GetPodNetworkingValidationInput(InterfaceTypeToPodList{ PodsOnPrimaryENI: []v1.Pod{*invalidPod}, - }) + }).Serialize() + Expect(err).NotTo(HaveOccurred()) + ValidateHostNetworking(NetworkingSetupFails, invalidInput) By("validating the tester fails when invalid namespace is passed") @@ -152,9 +194,10 @@ var _ = Describe("test host networking", func() { // veth pair name is generated using namespace+name so the test should fail invalidPod.Namespace = "different" - invalidInput = GetPodNetworkingValidationInput(InterfaceTypeToPodList{ + invalidInput, err = GetPodNetworkingValidationInput(InterfaceTypeToPodList{ PodsOnPrimaryENI: []v1.Pod{*invalidPod}, - }) + }).Serialize() + Expect(err).NotTo(HaveOccurred()) ValidateHostNetworking(NetworkingSetupFails, invalidInput) @@ -222,13 +265,13 @@ func ValidateHostNetworking(testType TestType, podValidationInputString string) // GetPodNetworkingValidationInput returns input string containing the list of pods for which // the host networking has to be tested -func GetPodNetworkingValidationInput(interfaceTypeToPodList InterfaceTypeToPodList) string { +func GetPodNetworkingValidationInput(interfaceTypeToPodList InterfaceTypeToPodList) input.PodNetworkingValidationInput { ip := input.PodNetworkingValidationInput{ VPCCidrRange: vpcCIDRs, - VethPrefix: NEW_VETH_PREFIX, + VethPrefix: "eni", PodList: []input.Pod{}, - MTU: NEW_MTU_VAL, + ValidateMTU: false, } for _, primaryENIPod := range interfaceTypeToPodList.PodsOnPrimaryENI { @@ -249,8 +292,5 @@ func GetPodNetworkingValidationInput(interfaceTypeToPodList InterfaceTypeToPodLi }) } - inputBytes, err := json.Marshal(ip) - Expect(err).ToNot(HaveOccurred()) - - return string(inputBytes) + return ip } From f132ff4e73b909a2b36017e1dee4065a66d72900 Mon Sep 17 00:00:00 2001 From: Chinmay Gadgil Date: Thu, 13 May 2021 11:25:28 -0700 Subject: [PATCH 6/8] Reverted Dockerfile changes --- test/agent/Dockerfile | 3 --- test/integration-new/cni/host_networking_test.go | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/test/agent/Dockerfile b/test/agent/Dockerfile index f72ee6a7e6..ad614e9f9a 100644 --- a/test/agent/Dockerfile +++ b/test/agent/Dockerfile @@ -26,9 +26,6 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build \ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build \ -a -o metric-server cmd/metric-server/main.go -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build \ - -a -o aws-node-config cmd/aws-node-config/main.go - FROM public.ecr.aws/amazonlinux/amazonlinux:2 RUN yum update -y && \ yum clean all diff --git a/test/integration-new/cni/host_networking_test.go b/test/integration-new/cni/host_networking_test.go index f0d0e1ee67..03e63db87f 100644 --- a/test/integration-new/cni/host_networking_test.go +++ b/test/integration-new/cni/host_networking_test.go @@ -82,7 +82,7 @@ var _ = Describe("test host networking", func() { input, err := GetPodNetworkingValidationInput(interfaceTypeToPodList).Serialize() Expect(err).NotTo(HaveOccurred()) - By("validating host, networking setup is setup correctly") + By("validating host networking setup is setup correctly") ValidateHostNetworking(NetworkingSetupSucceeds, input) By("deleting the deployment to test teardown") From 43e44880749ff8407fd09536fcc272499f4b3f09 Mon Sep 17 00:00:00 2001 From: Chinmay Gadgil Date: Mon, 17 May 2021 13:08:39 -0700 Subject: [PATCH 7/8] Create deployment after changing MTU and Veth prefix --- test/framework/utils/utils.go | 5 ----- .../cni/host_networking_test.go | 18 +++++++++--------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/test/framework/utils/utils.go b/test/framework/utils/utils.go index 1670ebd669..34b105283e 100644 --- a/test/framework/utils/utils.go +++ b/test/framework/utils/utils.go @@ -19,11 +19,6 @@ import ( "k8s.io/apimachinery/pkg/types" ) -const ( - NAMESPACE = "kube-system" - DAEMONSET = "aws-node" -) - // NamespacedName returns the namespaced name for k8s objects func NamespacedName(obj v1.Object) types.NamespacedName { return types.NamespacedName{ diff --git a/test/integration-new/cni/host_networking_test.go b/test/integration-new/cni/host_networking_test.go index 03e63db87f..f9761a1e22 100644 --- a/test/integration-new/cni/host_networking_test.go +++ b/test/integration-new/cni/host_networking_test.go @@ -104,23 +104,23 @@ var _ = Describe("test host networking", func() { NodeName(primaryNode.Name). Build() - By("creating a deployment to launch pods") - deployment, err = f.K8sResourceManagers.DeploymentManager(). - CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout) - Expect(err).ToNot(HaveOccurred()) - By("Configuring Veth Prefix and MTU value on aws-node daemonset") - ds, err := f.K8sResourceManagers.DaemonSetManager().GetDaemonSet(utils.NAMESPACE, utils.DAEMONSET) + ds, err := f.K8sResourceManagers.DaemonSetManager().GetDaemonSet(utils.AwsNodeNamespace, utils.AwsNodeName) Expect(err).NotTo(HaveOccurred()) oldMTU := utils.GetEnvValueForKeyFromDaemonSet(AWS_VPC_ENI_MTU, ds) oldVethPrefix := utils.GetEnvValueForKeyFromDaemonSet(AWS_VPC_K8S_CNI_VETHPREFIX, ds) - k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.DAEMONSET, utils.NAMESPACE, utils.DAEMONSET, map[string]string{ + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ AWS_VPC_ENI_MTU: strconv.Itoa(NEW_MTU_VAL), AWS_VPC_K8S_CNI_VETHPREFIX: NEW_VETH_PREFIX, }) + By("creating a deployment to launch pods") + deployment, err = f.K8sResourceManagers.DeploymentManager(). + CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout) + Expect(err).ToNot(HaveOccurred()) + By("getting the list of pods using IP from primary and secondary ENI") interfaceTypeToPodList := GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal) @@ -134,11 +134,11 @@ var _ = Describe("test host networking", func() { Expect(err).NotTo(HaveOccurred()) By("validating host networking setup is setup correctly with MTU check as well") - ValidateHostNetworking(NetworkingSetupSucceedsAfterChangingMTU, input) + ValidateHostNetworking(NetworkingSetupSucceeds, input) // Restore MTU and Veth Prefix By("Restoring MTU value and Veth Prefix to old values") - k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.DAEMONSET, utils.NAMESPACE, utils.DAEMONSET, map[string]string{ + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ AWS_VPC_ENI_MTU: oldMTU, AWS_VPC_K8S_CNI_VETHPREFIX: oldVethPrefix, }) From 3994d6e46789e803027f3311f2c332b174623bf7 Mon Sep 17 00:00:00 2001 From: Chinmay Gadgil Date: Mon, 17 May 2021 17:20:28 -0700 Subject: [PATCH 8/8] Minor change to MTU validation logic --- test/agent/cmd/networking/tester/network.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/agent/cmd/networking/tester/network.go b/test/agent/cmd/networking/tester/network.go index 14eb450e08..6320693dd0 100644 --- a/test/agent/cmd/networking/tester/network.go +++ b/test/agent/cmd/networking/tester/network.go @@ -70,9 +70,13 @@ func TestNetworkingSetupForRegularPod(podNetworkingValidationInput input.PodNetw } // Validate MTU value if it is set to true - if podNetworkingValidationInput.ValidateMTU && link.Attrs().MTU != podNetworkingValidationInput.MTU { - validationErrors = append(validationErrors, - fmt.Errorf("failed to match expected MTU value %v for veth pair: %s", hostVethName, link.Attrs().MTU)) + if podNetworkingValidationInput.ValidateMTU { + if link.Attrs().MTU != podNetworkingValidationInput.MTU { + validationErrors = append(validationErrors, + fmt.Errorf("MTU value %v for pod: %s on veth pair: %s failed to match the expected value: %v", link.Attrs().MTU, pod.PodName, hostVethName, podNetworkingValidationInput.MTU)) + } else { + log.Printf("Found Valid MTU value:%d for pod: %s on veth Pair: %s\n", link.Attrs().MTU, pod.PodName, hostVethName) + } } // Verify IP Link for the Pod is UP