[e2e tests] Wait for secondary IPs before running ping test

ci/kind/test-secondary-network-kind.sh

@@ -112,6 +112,8 @@ function run_test {
 
   # Wait for antrea-controller start to make sure the IPPool validation webhook is ready.
   kubectl rollout status --timeout=1m deployment.apps/antrea-controller -n kube-system
+  # An extra small delay to reduce the possibility of failure in CI.
+  sleep 5
   kubectl apply -f $ATTACHMENT_DEFINITION_YAML
   kubectl apply -f $SECONDARY_NETWORKS_YAML
 

test/e2e-secondary-network/secondary_network_test.go

@@ -105,74 +105,117 @@ func (data *testData) createPodForSecondaryNetwork(ns string, pod *testPodInfo)
 	return podBuilder.Create(data.e2eTestData)
 }
 
-// getSecondaryInterface checks the secondary interfaces created for the specific Pod and returns its IPv4 address.
-func (data *testData) getSecondaryInterface(targetPod *testPodInfo, interfaceName string) (string, error) {
-	cmd := []string{"/bin/sh", "-c", fmt.Sprintf("ip addr show %s | grep 'inet ' | awk '{print $2}' | cut -d/ -f1", interfaceName)}
+// listPodIPs returns a map of Pod IPs, indexed by the interface name. All interfaces are included
+// and only IPv4 addresses are considered. If an interface is not assigned an IPv4 address, it will
+// be included in the map, with a nil value.
+func (data *testData) listPodIPs(targetPod *testPodInfo) (map[string]net.IP, error) {
+	cmd := []string{"ip", "addr", "show"}
 	stdout, _, err := data.e2eTestData.RunCommandFromPod(data.e2eTestData.GetTestNamespace(), targetPod.podName, containerName, cmd)
-	stdout = strings.TrimSuffix(stdout, "\n")
-	if err != nil || stdout == "" {
-		return "", fmt.Errorf("interface %s not found on %s. err: %v", interfaceName, targetPod.podName, err)
+	if err != nil {
+		return nil, fmt.Errorf("error when listing interfaces for %s: %w", targetPod.podName, err)
 	}
-	return stdout, nil
-}
-
-// checkSubnet checks if the IP address to be pinged has the same subnet as the Pod from which the IP Address is pinged.
-func (data *testData) checkSubnet(t *testing.T, sourcePod, targetPod *testPodInfo, targetNetwork string) (bool, error) {
-	for i, n := range sourcePod.interfaceNetworks {
-		if n == targetNetwork {
-			_, err := data.getSecondaryInterface(sourcePod, i)
-			if err != nil {
-				return false, err
+	result := make(map[string]net.IP)
+	var currentInterface string
+	lines := strings.Split(stdout, "\n")
+	for _, line := range lines {
+		fields := strings.Fields(line)
+		if len(fields) >= 2 && strings.HasSuffix(fields[0], ":") {
+			// first field is ifindex, second field is interface name
+			currentInterface = strings.Split(strings.TrimSuffix(fields[1], ":"), "@")[0]
+			result[currentInterface] = nil
+		} else if len(fields) >= 2 && fields[0] == "inet" {
+			ipStr := strings.Split(fields[1], "/")[0]
+			ip := net.ParseIP(ipStr)
+			if ip == nil {
+				return nil, fmt.Errorf("failed to parse IP (%s) for interface %s of Pod %s", ipStr, currentInterface, targetPod.podName)
 			}
-			return true, nil
+			result[currentInterface] = ip
 		}
 	}
-	return false, nil
+	return result, nil
 }
 
 // pingBetweenInterfaces parses through all the created Pods and pings the other Pod if the two Pods
 // both have a secondary network interface on the same network.
 func (data *testData) pingBetweenInterfaces(t *testing.T) error {
 	e2eTestData := data.e2eTestData
 	namespace := e2eTestData.GetTestNamespace()
-	for _, sourcePod := range data.pods {
-		for _, targetPod := range data.pods {
-			if targetPod.podName == sourcePod.podName {
-				continue
+
+	type attachment struct {
+		network string
+		iface   string
+		ip      net.IP
+	}
+	type network struct {
+		// maps each Pod to its attachments in this network (typically just one)
+		podAttachments map[*testPodInfo][]*attachment
+	}
+	networks := make(map[string]*network)
+	addPodNetworkAttachments := func(pod *testPodInfo, podAttachments []*attachment) {
+		for _, pa := range podAttachments {
+			if _, ok := networks[pa.network]; !ok {
+				networks[pa.network] = &network{
+					podAttachments: make(map[*testPodInfo][]*attachment),
+				}
 			}
-			for i, n := range targetPod.interfaceNetworks {
-				_, err := e2eTestData.PodWaitFor(defaultTimeout, targetPod.podName, namespace, func(pod *corev1.Pod) (bool, error) {
-					return pod.Status.Phase == corev1.PodRunning, nil
-				})
-				if err != nil {
-					return fmt.Errorf("error when waiting for Pod %s: %v", targetPod.podName, err)
+			networks[pa.network].podAttachments[pod] = append(networks[pa.network].podAttachments[pod], pa)
+		}
+	}
+
+	// Collect all secondary network IPs when they are available.
+	for _, testPod := range data.pods {
+		_, err := e2eTestData.PodWaitFor(defaultTimeout, testPod.podName, namespace, func(pod *corev1.Pod) (bool, error) {
+			if pod.Status.Phase != corev1.PodRunning {
+				return false, nil
+			}
+			var podNetworkAttachments []*attachment
+			podIPs, err := data.listPodIPs(testPod)
+			if err != nil {
+				return false, err
+			}
+			for iface, net := range testPod.interfaceNetworks {
+				if podIPs[iface] == nil {
+					return false, nil
 				}
+				podNetworkAttachments = append(podNetworkAttachments, &attachment{
+					network: net,
+					iface:   iface,
+					ip:      podIPs[iface],
+				})
+			}
+			// we found all the expected secondary network interfaces / attachments
+			addPodNetworkAttachments(testPod, podNetworkAttachments)
+			return true, nil
+		})
+		if err != nil {
+			return fmt.Errorf("error when waiting for secondary IPs for Pod %+v: %v", testPod, err)
+		}
+	}
 
-				matched, _ := data.checkSubnet(t, sourcePod, targetPod, n)
-				if matched {
-					secondaryIPAddress, err := data.getSecondaryInterface(targetPod, i)
-					if err != nil {
-						return err
-					}
-					ip := net.ParseIP(secondaryIPAddress)
-					if ip == nil {
-						return fmt.Errorf("failed to parse IP (%s) for interface %s of Pod %s", secondaryIPAddress, i, targetPod.podName)
-					}
+	// Run ping-mesh test for each secondary network.
+	for _, network := range networks {
+		for sourcePod := range network.podAttachments {
+			for targetPod, targetPodAttachments := range network.podAttachments {
+				if sourcePod == targetPod {
+					continue
+				}
+				for _, targetAttachment := range targetPodAttachments {
 					var IPToPing antreae2e.PodIPs
-					if ip.To4() != nil {
-						IPToPing = antreae2e.PodIPs{IPv4: &ip}
+					if targetAttachment.ip.To4() != nil {
+						IPToPing = antreae2e.PodIPs{IPv4: &targetAttachment.ip}
 					} else {
-						IPToPing = antreae2e.PodIPs{IPv6: &ip}
+						IPToPing = antreae2e.PodIPs{IPv6: &targetAttachment.ip}
 					}
 					if err := e2eTestData.RunPingCommandFromTestPod(antreae2e.PodInfo{Name: sourcePod.podName, OS: osType, NodeName: sourcePod.nodeName, Namespace: namespace},
 						namespace, &IPToPing, containerName, pingCount, pingSize, false); err != nil {
-						return fmt.Errorf("ping '%s' -> '%s'(Interface: %s, IP Address: %s) failed: %v", sourcePod.podName, targetPod.podName, i, secondaryIPAddress, err)
+						return fmt.Errorf("ping '%s' -> '%s'(Interface: %s, IP Address: %s) failed: %v", sourcePod.podName, targetPod.podName, targetAttachment.iface, targetAttachment.ip, err)
 					}
-					logs.Infof("ping '%s' -> '%s'( Interface: %s, IP Address: %s): OK", sourcePod.podName, targetPod.podName, i, secondaryIPAddress)
+					logs.Infof("ping '%s' -> '%s'( Interface: %s, IP Address: %s): OK", sourcePod.podName, targetPod.podName, targetAttachment.iface, targetAttachment.ip)
 				}
 			}
 		}
 	}
+
 	return nil
 }