From 104b5d1b66284b1f2c57644832646420662bb60a Mon Sep 17 00:00:00 2001 From: Jeff Nelson Date: Wed, 28 Feb 2024 17:54:30 +0000 Subject: [PATCH] integration test suite for Custom Networking + Security Groups for Pods --- .../resources/agent/traffic_tester.go | 2 +- test/integration/README.md | 21 +- .../custom_networking_sgpp_suite_test.go | 219 +++++++++++ .../custom_networking_sgpp_test.go | 355 ++++++++++++++++++ .../custom-networking-sgpp/trunk_test.go | 46 +++ test/integration/ipamd/eni_tag_test.go | 1 - .../security_group_per_pod_suite_test.go | 2 - 7 files changed, 635 insertions(+), 11 deletions(-) create mode 100644 test/integration/custom-networking-sgpp/custom_networking_sgpp_suite_test.go create mode 100644 test/integration/custom-networking-sgpp/custom_networking_sgpp_test.go create mode 100644 test/integration/custom-networking-sgpp/trunk_test.go diff --git a/test/framework/resources/agent/traffic_tester.go b/test/framework/resources/agent/traffic_tester.go index 1be40be92c..b01d56acec 100644 --- a/test/framework/resources/agent/traffic_tester.go +++ b/test/framework/resources/agent/traffic_tester.go @@ -93,7 +93,7 @@ func (t *TrafficTest) TestTraffic() (float64, error) { if t.ValidateServerPods != nil { err = t.ValidateServerPods(podList) if err != nil { - return 0, fmt.Errorf("pod list %v validation failed %v", podList, err) + return 0, fmt.Errorf("pod list validation failed %v", err) } fmt.Fprintln(GinkgoWriter, "successfully validated the server pod list") } diff --git a/test/integration/README.md b/test/integration/README.md index dc0dc92938..72d20d343c 100644 --- a/test/integration/README.md +++ b/test/integration/README.md @@ -74,22 +74,21 @@ In order to test a custom image you need pass the following tags along with the *IMPORTANT*: Should use an IPv6 cluster with Prefix Delegation enabled. VPC CNI only supports IPv6 mode with Prefix Delegation. -### Custom Networking +### Custom Networking tests (custom_networking) Custom networking tests validate use of the `AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG` environment variable. Test info: - - No existing node group should be present. The test creates a self managed node group with the reduced MAX_POD value. - Pass `custom-networking-cidr-range` flag with *allowed* VPC CIDR that does not conflict with an existing one. So if existing VPC CIDR is `192.168.0.0/16`, you can use `custom-networking-cidr-range=100.64.0.0/16`. You can go to your cluster VPC to check existing/allowed CIDRs. -### SNAT tests +### SNAT tests (snat) SNAT tests cover pod source NAT behavior with various deployment scenarios. Test info: - EKS Cluster should have at least one private subnet and at least one public subnet. These tests modify the SNAT related variables in `aws-node` pod, validate the IP table SNAT rules, and check for Internet Connectivity. -### Calico tests +### Calico tests (calico) `calico` helps validate compatibility with calico network policies. It does so by running the Calico Stars policy demo. @@ -99,10 +98,18 @@ Test info: Test info: - Requires at least one Nitro-based instance. - - EKS Cluster should be v1.16+. This tests creates an additional Trunk ENI on all Nitro-based instances present in the cluster. This could interfere with running integration tests that expect specific values of `WARM_ENI_TARGET`. - - For this reason, the test should either be run without any node groups present in the cluster or at the very end. + - EKS Cluster should be v1.16+. This tests creates an additional Trunk ENI on all Nitro-based instances present in the cluster. -### Multus tests +### Custom Networking and Security Groups for Pods tests (custom_networking_sgpp) + +`custom_networking_sgpp` test suite validates the combination of Custom Networking and Security Groups for Pods. + +Test info: + - Pass `custom-networking-cidr-range` flag with *allowed* VPC CIDR that does not conflict with an existing one. So if existing VPC CIDR is `192.168.0.0/16`, you can use `custom-networking-cidr-range=100.64.0.0/16`. You can go to your cluster VPC to check existing/allowed CIDRs. + - Requires at least one Nitro-based instance. + - EKS Cluster should be v1.16+. This tests creates an additional Trunk ENI on all Nitro-based instances present in the cluster. + +### Multus tests (multus) These tests require multus to be deployed to your cluster using the [manifest](https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/master/config/multus/v3.9.2-eksbuild.1/aws-k8s-multus.yaml) file. Instead test can be triggered by running `run-multus-tests.sh` located under scripts directory. This script installs the multus manifest first and then runs the the ginkgo test suite. You can optionally provide multus tag to install the manifest. If not provided then it will use the default tag diff --git a/test/integration/custom-networking-sgpp/custom_networking_sgpp_suite_test.go b/test/integration/custom-networking-sgpp/custom_networking_sgpp_suite_test.go new file mode 100644 index 0000000000..e681f0e73c --- /dev/null +++ b/test/integration/custom-networking-sgpp/custom_networking_sgpp_suite_test.go @@ -0,0 +1,219 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package custom_networking_sgpp + +import ( + "flag" + "fmt" + "net" + "testing" + + "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" + "github.com/aws/amazon-vpc-cni-k8s/test/framework" + awsUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/vpc" + "github.com/prometheus/client_golang/prometheus" + corev1 "k8s.io/api/core/v1" + + "github.com/apparentlymart/go-cidr/cidr" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestCustomNetworkingSGPP(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "CNI Custom Networking + Security Groups for Pods Test Suite") +} + +var ( + f *framework.Framework + // VPC Configuration with the details of public subnet and availability zone present in the cluster's subnets + clusterVPCConfig *awsUtils.ClusterVPCConfig + // The CIDR Range that will be associated with the VPC to create new subnet for Custom Networking + cidrRangeString string + cidrRange *net.IPNet + cidrBlockAssociationID string + // Security Group that will be used in ENIConfig + customNetworkingSGID string + customNetworkingSubnetIDList []string + // List of ENIConfig per Availability Zone + eniConfigList []*v1alpha1.ENIConfig + eniConfigBuilderList []*manifest.ENIConfigBuilder + // Security Group that will be used to create Security Group Policy + podEniSGID string + // Port that will be opened for Security Groups for Pods testing + podEniOpenPort = 80 + metricsPort = 8080 + // Maximum number of branch interfaces that can be created across all nodes + totalBranchInterface int + // Cluster security group ID for node to node communication + clusterSGID string + + targetNode corev1.Node + v4Zero = "0.0.0.0/0" + v6Zero = "::/0" + numNodes int // number of nodes in cluster +) + +// Parse test specific variable from flag +func init() { + flag.StringVar(&cidrRangeString, "custom-networking-cidr-range", "100.64.0.0/16", "custom networking cidr range to be associated with the VPC") +} + +var _ = BeforeSuite(func() { + f = framework.New(framework.GlobalOptions) + + _, cidrRange, err = net.ParseCIDR(cidrRangeString) + Expect(err).ToNot(HaveOccurred()) + + By("getting the cluster VPC Config") + clusterVPCConfig, err = awsUtils.GetClusterVPCConfig(f) + Expect(err).ToNot(HaveOccurred()) + + By("Getting Cluster Security Group ID") + clusterRes, err := f.CloudServices.EKS().DescribeCluster(f.Options.ClusterName) + Expect(err).NotTo(HaveOccurred()) + clusterSGID = *(clusterRes.Cluster.ResourcesVpcConfig.ClusterSecurityGroupId) + fmt.Fprintf(GinkgoWriter, "cluster security group is %s\n", clusterSGID) + + // Custom Networking setup + // TODO: Ideally, we would clone the Custom Networking SG from the cluster SG. Unfortunately, the EC2 API does not support this. + By("creating security group to be used by custom networking") + createSecurityGroupOutput, err := f.CloudServices.EC2(). + CreateSecurityGroup("custom-networking-test", "custom networking", f.Options.AWSVPCID) + Expect(err).ToNot(HaveOccurred()) + customNetworkingSGID = *createSecurityGroupOutput.GroupId + + By("authorizing egress and ingress for security group in ENIConfig") + f.CloudServices.EC2().AuthorizeSecurityGroupEgress(customNetworkingSGID, "-1", -1, -1, v4Zero) + f.CloudServices.EC2().AuthorizeSecurityGroupIngress(customNetworkingSGID, "-1", -1, -1, v4Zero) + + By("associating cidr range to the VPC") + association, err := f.CloudServices.EC2().AssociateVPCCIDRBlock(f.Options.AWSVPCID, cidrRange.String()) + Expect(err).ToNot(HaveOccurred()) + cidrBlockAssociationID = *association.CidrBlockAssociation.AssociationId + + for i, az := range clusterVPCConfig.AvailZones { + By(fmt.Sprintf("creating the subnet in %s", az)) + + subnetCidr, err := cidr.Subnet(cidrRange, 8, 5*i) + Expect(err).ToNot(HaveOccurred()) + + createSubnetOutput, err := f.CloudServices.EC2(). + CreateSubnet(subnetCidr.String(), f.Options.AWSVPCID, az) + Expect(err).ToNot(HaveOccurred()) + + subnetID := *createSubnetOutput.Subnet.SubnetId + + By("associating the route table with the newly created subnet") + err = f.CloudServices.EC2().AssociateRouteTableToSubnet(clusterVPCConfig.PublicRouteTableID, subnetID) + Expect(err).ToNot(HaveOccurred()) + + eniConfigBuilder := manifest.NewENIConfigBuilder(). + Name(az). + SubnetID(subnetID). + SecurityGroup([]string{customNetworkingSGID}) + eniConfig, err := eniConfigBuilder.Build() + Expect(err).ToNot(HaveOccurred()) + + // For updating/deleting later + customNetworkingSubnetIDList = append(customNetworkingSubnetIDList, subnetID) + eniConfigBuilderList = append(eniConfigBuilderList, eniConfigBuilder) + eniConfigList = append(eniConfigList, eniConfig.DeepCopy()) + + By("creating the ENIConfig with az name") + err = f.K8sResourceManagers.CustomResourceManager().CreateResource(eniConfig) + Expect(err).ToNot(HaveOccurred()) + } + + // Security Groups for Pods setup + // Note that Custom Networking only supports IPv4 clusters, so IPv4 setup can be assumed. + By("creating a new security group for use in Security Group Policy") + podEniSGName := "pod-eni-automation-v4" + securityGroupOutput, err := f.CloudServices.EC2().CreateSecurityGroup(podEniSGName, + "test created by vpc cni automation test suite", f.Options.AWSVPCID) + Expect(err).ToNot(HaveOccurred()) + podEniSGID = *securityGroupOutput.GroupId + + By("authorizing egress and ingress on security group for client-server communication") + f.CloudServices.EC2().AuthorizeSecurityGroupEgress(podEniSGID, "tcp", podEniOpenPort, podEniOpenPort, v4Zero) + f.CloudServices.EC2().AuthorizeSecurityGroupIngress(podEniSGID, "tcp", podEniOpenPort, podEniOpenPort, v4Zero) + + By("getting branch ENI limits") + nodeList, err := f.K8sResourceManagers.NodeManager().GetNodes(f.Options.NgNameLabelKey, f.Options.NgNameLabelVal) + Expect(err).ToNot(HaveOccurred()) + numNodes = len(nodeList.Items) + Expect(numNodes).Should(BeNumerically(">=", 1)) + + node := nodeList.Items[0] + instanceID := k8sUtils.GetInstanceIDFromNode(node) + nodeInstance, err := f.CloudServices.EC2().DescribeInstance(instanceID) + instanceType := *nodeInstance.InstanceType + totalBranchInterface = vpc.Limits[instanceType].BranchInterface * numNodes + + By("enabling custom networking and sgpp on aws-node DaemonSet") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, + utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ + "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG": "true", + "ENI_CONFIG_LABEL_DEF": "topology.kubernetes.io/zone", + "ENABLE_POD_ENI": "true", + }) + + By("terminating instances") + err = awsUtils.TerminateInstances(f) + Expect(err).ToNot(HaveOccurred()) + + By("getting target node") + nodeList, err = f.K8sResourceManagers.NodeManager().GetNodes(f.Options.NgNameLabelKey, f.Options.NgNameLabelVal) + Expect(err).ToNot(HaveOccurred()) + targetNode = nodeList.Items[0] +}) + +var _ = AfterSuite(func() { + var errs prometheus.MultiError + for _, eniConfig := range eniConfigList { + By("deleting ENIConfig") + errs.Append(f.K8sResourceManagers.CustomResourceManager().DeleteResource(eniConfig)) + } + + By("disabling custom networking and pod eni on aws-node DaemonSet") + k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, + utils.AwsNodeNamespace, utils.AwsNodeName, map[string]struct{}{ + "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG": {}, + "ENI_CONFIG_LABEL_DEF": {}, + "ENABLE_POD_ENI": {}, + }) + + By("terminating instances") + errs.Append(awsUtils.TerminateInstances(f)) + + By("deleting Custom Networking security group") + errs.Append(f.CloudServices.EC2().DeleteSecurityGroup(customNetworkingSGID)) + + By("deleting pod ENI security group") + errs.Append(f.CloudServices.EC2().DeleteSecurityGroup(podEniSGID)) + + for _, subnet := range customNetworkingSubnetIDList { + By(fmt.Sprintf("deleting the subnet %s", subnet)) + errs.Append(f.CloudServices.EC2().DeleteSubnet(subnet)) + } + + By("disassociating the CIDR range to the VPC") + errs.Append(f.CloudServices.EC2().DisAssociateVPCCIDRBlock(cidrBlockAssociationID)) + + Expect(errs.MaybeUnwrap()).ToNot(HaveOccurred()) +}) diff --git a/test/integration/custom-networking-sgpp/custom_networking_sgpp_test.go b/test/integration/custom-networking-sgpp/custom_networking_sgpp_test.go new file mode 100644 index 0000000000..acf61e4e5d --- /dev/null +++ b/test/integration/custom-networking-sgpp/custom_networking_sgpp_test.go @@ -0,0 +1,355 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package custom_networking_sgpp + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/aws/amazon-vpc-cni-k8s/test/agent/pkg/input" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/agent" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + + "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1beta1" + vpcControllerFW "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/manifest" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type TestType int + +var err error + +const ( + NetworkingTearDownSucceeds TestType = iota + NetworkingSetupSucceeds + // Custom Networking does not support IPv6 clusters + isIPv4Cluster = true +) + +// NOTE: This file is a near identical copy of $PROJECT_ROOT/test/integration/pod-eni/security_group_per_pod_test.go, but it excludes the DISABLE_TCP_EARLY_DEMUX tests. + +var _ = Describe("Custom Networking + Security Groups for Pods Test", func() { + var ( + // The Pod labels for client and server in order to retrieve the + // client and server Pods belonging to a Deployment/Jobs + labelKey = "app" + serverPodLabelVal = "server-pod" + clientPodLabelVal = "client-pod" + busyboxPodLabelVal = "busybox-pod" + // The Security Group Policy take list of Pod Label Value and if the + // Pod has any label in the list, it should get Branch ENI + branchPodLabelVal []string + serverDeploymentBuilder *manifest.DeploymentBuilder + securityGroupPolicy *v1beta1.SecurityGroupPolicy + ) + + JustBeforeEach(func() { + By("creating test namespace") + f.K8sResourceManagers.NamespaceManager(). + CreateNamespace(utils.DefaultTestNamespace) + + serverDeploymentBuilder = manifest.NewDefaultDeploymentBuilder(). + Name("traffic-server") + + securityGroupPolicy, err = vpcControllerFW.NewSGPBuilder(). + Namespace(utils.DefaultTestNamespace). + Name("test-sgp"). + SecurityGroup([]string{podEniSGID}). + PodMatchExpression(labelKey, metaV1.LabelSelectorOpIn, branchPodLabelVal...). + Build() + Expect(err).ToNot(HaveOccurred()) + + By("creating the Security Group Policy") + err = f.K8sResourceManagers.CustomResourceManager().CreateResource(securityGroupPolicy) + Expect(err).ToNot(HaveOccurred()) + }) + + JustAfterEach(func() { + By("deleting test namespace") + f.K8sResourceManagers.NamespaceManager(). + DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace) + + By("Deleting Security Group Policy") + f.K8sResourceManagers.CustomResourceManager().DeleteResource(securityGroupPolicy) + + By("waiting for the branch ENI to be cooled down") + time.Sleep(time.Second * 60) + }) + + Context("when testing traffic between branch ENI pods and regular pods", func() { + BeforeEach(func() { + // Only the Server Pods will get Branch ENI + branchPodLabelVal = []string{serverPodLabelVal} + }) + + It("should have 99%+ success rate", func() { + trafficTester := agent.TrafficTest{ + Framework: f, + TrafficServerDeploymentBuilder: serverDeploymentBuilder, + ServerPort: podEniOpenPort, + ServerProtocol: "tcp", + ClientCount: 20, + ServerCount: totalBranchInterface, + ServerPodLabelKey: labelKey, + ServerPodLabelVal: serverPodLabelVal, + ClientPodLabelKey: labelKey, + ClientPodLabelVal: clientPodLabelVal, + ValidateServerPods: ValidatePodsHaveBranchENI, + IsV6Enabled: !isIPv4Cluster, + } + + By("performing traffic test") + successRate, err := trafficTester.TestTraffic() + Expect(err).ToNot(HaveOccurred()) + Expect(successRate).Should(BeNumerically(">=", float64(99))) + }) + }) + + Context("when testing traffic between branch ENI and branch ENI pods", func() { + BeforeEach(func() { + // Both the Server and Client Pods will get Branch ENI + branchPodLabelVal = []string{serverPodLabelVal, clientPodLabelVal} + + // Allow Ingress on cluster security group so client pods can communicate with metric pod + // 8080: metric-pod listener port + // TODO: uncomment after Custom Networking SGID clones cluster SGID + //By("Adding an additional Ingress Rule on NodeSecurityGroupID to allow client-to-metric traffic") + //if isIPv4Cluster { + // err := f.CloudServices.EC2().AuthorizeSecurityGroupIngress(customNetworkingSGID, "TCP", metricsPort, metricsPort, v4Zero) + // Expect(err).ToNot(HaveOccurred()) + //} else { + // err := f.CloudServices.EC2().AuthorizeSecurityGroupIngress(customNetworkingSGID, "TCP", metricsPort, metricsPort, v6Zero) + // Expect(err).ToNot(HaveOccurred()) + //} + }) + + It("should have 99%+ success rate", func() { + t := agent.TrafficTest{ + Framework: f, + TrafficServerDeploymentBuilder: serverDeploymentBuilder, + ServerPort: podEniOpenPort, + ServerProtocol: "tcp", + ClientCount: totalBranchInterface / 2, + ServerCount: totalBranchInterface / 2, + ServerPodLabelKey: labelKey, + ServerPodLabelVal: serverPodLabelVal, + ClientPodLabelKey: labelKey, + ClientPodLabelVal: clientPodLabelVal, + ValidateServerPods: ValidatePodsHaveBranchENI, + ValidateClientPods: ValidatePodsHaveBranchENI, + IsV6Enabled: !isIPv4Cluster, + } + + successRate, err := t.TestTraffic() + Expect(err).ToNot(HaveOccurred()) + Expect(successRate).Should(BeNumerically(">=", float64(99))) + }) + + // TODO: uncomment after Custom Networking SGID clones cluster SGID + //AfterEach(func() { + // // Revoke the Ingress rule for traffic from client pods added to Node Security Group + // By("Revoking the additional Ingress rule added to allow client-to-metric traffic") + // if isIPv4Cluster { + // err := f.CloudServices.EC2().RevokeSecurityGroupIngress(clusterSGID, "TCP", metricsPort, metricsPort, v4Zero) + // Expect(err).ToNot(HaveOccurred()) + // } else { + // err := f.CloudServices.EC2().RevokeSecurityGroupIngress(clusterSGID, "TCP", metricsPort, metricsPort, v6Zero) + // Expect(err).ToNot(HaveOccurred()) + // } + //}) + }) + + Context("when testing traffic to a port on Branch ENI that is not open", func() { + BeforeEach(func() { + // Only the Server Pods will get Branch ENI + branchPodLabelVal = []string{serverPodLabelVal} + }) + + It("should have 0% success rate", func() { + t := agent.TrafficTest{ + Framework: f, + TrafficServerDeploymentBuilder: serverDeploymentBuilder, + ServerPort: 2271, + ServerProtocol: "tcp", + ClientCount: 2, + ServerCount: 5, + ServerPodLabelKey: labelKey, + ServerPodLabelVal: serverPodLabelVal, + ClientPodLabelKey: labelKey, + ClientPodLabelVal: clientPodLabelVal, + ValidateServerPods: ValidatePodsHaveBranchENI, + IsV6Enabled: !isIPv4Cluster, + } + + successRate, err := t.TestTraffic() + Expect(err).ToNot(HaveOccurred()) + Expect(successRate).Should(Equal(float64(0))) + }) + }) + + Context("Verify HostNetworking", func() { + BeforeEach(func() { + // BusyBox Pods will get Branch ENI + branchPodLabelVal = []string{busyboxPodLabelVal} + }) + It("Deploy BusyBox Pods with branch ENI and verify HostNetworking", func() { + // Pin deployment to primary node + deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry). + Replicas(totalBranchInterface/numNodes). + PodLabel(labelKey, busyboxPodLabelVal). + NodeName(targetNode.Name). + Build() + + By("creating a deployment to launch pod using Branch ENI") + _, err = f.K8sResourceManagers.DeploymentManager(). + CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout) + Expect(err).ToNot(HaveOccurred()) + + By("getting the list of pods using BranchENI") + podList, err := f.K8sResourceManagers. + PodManager(). + GetPodsWithLabelSelector(labelKey, busyboxPodLabelVal) + Expect(err).ToNot(HaveOccurred()) + + By("generating the pod networking validation input to be passed to tester") + input, err := GetPodNetworkingValidationInput(podList).Serialize() + Expect(err).NotTo(HaveOccurred()) + + By("validating host networking setup is setup correctly") + ValidateHostNetworking(NetworkingSetupSucceeds, input) + + By("deleting the deployment to test teardown") + err = f.K8sResourceManagers.DeploymentManager(). + DeleteAndWaitTillDeploymentIsDeleted(deployment) + Expect(err).ToNot(HaveOccurred()) + + By("waiting to allow CNI to tear down networking for terminated pods") + time.Sleep(time.Second * 60) + + By("validating host networking is teared down correctly") + ValidateHostNetworking(NetworkingTearDownSucceeds, input) + }) + }) +}) + +func GetPodNetworkingValidationInput(podList v1.PodList) input.PodNetworkingValidationInput { + var ipFamily string + if isIPv4Cluster { + ipFamily = "IPv4" + } else { + ipFamily = "IPv6" + } + ip := input.PodNetworkingValidationInput{ + IPFamily: ipFamily, + VethPrefix: "vlan", + PodList: []input.Pod{}, + ValidateMTU: true, + MTU: 9001, + } + + for _, pod := range podList.Items { + if isIPv4Cluster { + ip.PodList = append(ip.PodList, input.Pod{ + PodName: pod.Name, + PodNamespace: pod.Namespace, + PodIPv4Address: pod.Status.PodIP, + }) + } else { + ip.PodList = append(ip.PodList, input.Pod{ + PodName: pod.Name, + PodNamespace: pod.Namespace, + PodIPv6Address: pod.Status.PodIP, + }) + + } + } + return ip +} + +func ValidateHostNetworking(testType TestType, podValidationInputString string) { + testerArgs := []string{fmt.Sprintf("-pod-networking-validation-input=%s", + podValidationInputString)} + + if NetworkingSetupSucceeds == testType { + testerArgs = append(testerArgs, "-test-setup=true", "-test-ppsg=true") + } else if NetworkingTearDownSucceeds == testType { + testerArgs = append(testerArgs, "-test-cleanup=true", "-test-ppsg=true") + } + + testContainer := manifest.NewTestHelperContainer(f.Options.TestImageRegistry). + Command([]string{"./networking"}). + Args(testerArgs). + Build() + + // Pin pod to primary node + testPod := manifest.NewDefaultPodBuilder(). + Container(testContainer). + NodeName(targetNode.Name). + HostNetwork(true). + Build() + + By("creating pod to test host networking setup") + testPod, err := f.K8sResourceManagers.PodManager(). + CreateAndWaitTillPodCompleted(testPod) + Expect(err).ToNot(HaveOccurred()) + + logs, errLogs := f.K8sResourceManagers.PodManager(). + PodLogs(testPod.Namespace, testPod.Name) + Expect(errLogs).ToNot(HaveOccurred()) + + fmt.Fprintln(GinkgoWriter, logs) + + By("deleting the host networking setup pod") + err = f.K8sResourceManagers.PodManager(). + DeleteAndWaitTillPodDeleted(testPod) + Expect(err).ToNot(HaveOccurred()) +} + +func ValidatePodsHaveBranchENI(podList v1.PodList) error { + for _, pod := range podList.Items { + if val, ok := pod.Annotations["vpc.amazonaws.com/pod-eni"]; ok { + type ENIDetails struct { + IPV4Addr string `json:"privateIp"` + IPV6Addr string `json:"ipv6addr"` + ID string `json:"eniId"` + } + var eniList []ENIDetails + err := json.Unmarshal([]byte(val), &eniList) + if err != nil { + return fmt.Errorf("failed to unmarshall the branch ENI annotation %v", err) + } + + if isIPv4Cluster { + if eniList[0].IPV4Addr != pod.Status.PodIP { + return fmt.Errorf("expected the pod to have IP %s but recieved %s", + eniList[0].IPV4Addr, pod.Status.PodIP) + } + } else { + if eniList[0].IPV6Addr != pod.Status.PodIP { + return fmt.Errorf("expected the pod to have IP %s but recieved %s", + eniList[0].IPV6Addr, pod.Status.PodIP) + } + } + By(fmt.Sprintf("validating pod %s has branch ENI %s", pod.Name, eniList[0].ID)) + } else { + return fmt.Errorf("failed to validate pod %v", pod) + } + } + return nil +} diff --git a/test/integration/custom-networking-sgpp/trunk_test.go b/test/integration/custom-networking-sgpp/trunk_test.go new file mode 100644 index 0000000000..f96d972c10 --- /dev/null +++ b/test/integration/custom-networking-sgpp/trunk_test.go @@ -0,0 +1,46 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package custom_networking_sgpp + +import ( + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Trunk ENI Security Group Test", func() { + Context("when validating security group on trunk ENI", func() { + It("should match security group in ENIConfig", func() { + instanceID := k8sUtils.GetInstanceIDFromNode(targetNode) + instance, err := f.CloudServices.EC2().DescribeInstance(instanceID) + Expect(err).ToNot(HaveOccurred()) + + trunkSGMatch := false + for _, nwInterface := range instance.NetworkInterfaces { + if *nwInterface.InterfaceType == "trunk" { + for _, group := range nwInterface.Groups { + if *group.GroupId == customNetworkingSGID { + trunkSGMatch = true + break + } + } + if trunkSGMatch { + break + } + } + } + Expect(trunkSGMatch).To(BeTrue()) + }) + }) +}) diff --git a/test/integration/ipamd/eni_tag_test.go b/test/integration/ipamd/eni_tag_test.go index 3bfa75fb90..661f900c5e 100644 --- a/test/integration/ipamd/eni_tag_test.go +++ b/test/integration/ipamd/eni_tag_test.go @@ -39,7 +39,6 @@ var _ = Describe("test tags are created on Secondary ENI", func() { // sets the desired environment variables and gets the list of new ENIs created after setting // the environment variables JustBeforeEach(func() { - // To re-initialize for each test case newENIs = []string{} diff --git a/test/integration/pod-eni/security_group_per_pod_suite_test.go b/test/integration/pod-eni/security_group_per_pod_suite_test.go index 6e248c0cf7..a5aae892bb 100644 --- a/test/integration/pod-eni/security_group_per_pod_suite_test.go +++ b/test/integration/pod-eni/security_group_per_pod_suite_test.go @@ -28,8 +28,6 @@ import ( . "github.com/onsi/gomega" ) -const AmazonEKSVPCResourceControllerARN = "arn:aws:iam::aws:policy/AmazonEKSVPCResourceController" - var ( f *framework.Framework err error