Merge pull request #922 from zhangzujian/underlay-e2e

update underlay e2e testing

Makefile

@@ -218,13 +218,13 @@ e2e:
 		done; \
 	fi
 
-	printf "package underlay\n\nvar nodeNetworks = map[string]string{\n" > test/e2e/underlay/network.go
+	printf "package e2e\n\nvar nodeNetworks = map[string]string{\n" > test/e2e/network.go
 	kind get nodes --name kube-ovn | while read node; do \
-		printf "\`$$node\`: \`" >> test/e2e/underlay/network.go; \
-		docker inspect -f '{{json .NetworkSettings.Networks.bridge}}' $$node >> test/e2e/underlay/network.go; \
-		printf "\`,\n" >> test/e2e/underlay/network.go; \
+		printf "    \`$$node\`: \`" >> test/e2e/network.go; \
+		docker inspect -f '{{json .NetworkSettings.Networks.bridge}}' $$node >> test/e2e/network.go; \
+		printf "\`,\n" >> test/e2e/network.go; \
 	done
-	echo "}" >> test/e2e/underlay/network.go
+	echo "}" >> test/e2e/network.go
 
 	docker pull kubeovn/pause:3.2
 	kind load docker-image --name kube-ovn kubeovn/pause:3.2

test/e2e-vlan-single-nic/e2e_suite_test.go

@@ -11,6 +11,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	_ "github.com/kubeovn/kube-ovn/test/e2e-vlan-single-nic/kubectl-ko"
 	_ "github.com/kubeovn/kube-ovn/test/e2e-vlan-single-nic/node"
 	"github.com/kubeovn/kube-ovn/test/e2e/framework"
 )

test/e2e-vlan-single-nic/kubectl-ko/ko.go

@@ -0,0 +1,39 @@
+package kubectl_ko
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	kubeovn "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
+	"github.com/kubeovn/kube-ovn/pkg/util"
+	"github.com/kubeovn/kube-ovn/test/e2e/framework"
+)
+
+var _ = Describe("[kubectl-ko]", func() {
+	f := framework.NewFramework("kubectl-ko", fmt.Sprintf("%s/.kube/config", os.Getenv("HOME")))
+
+	It("trace", func() {
+		pods, err := f.KubeClientSet.CoreV1().Pods("kube-system").List(context.Background(), metav1.ListOptions{LabelSelector: "app=kube-ovn-pinger"})
+		Expect(err).NotTo(HaveOccurred())
+		pod := pods.Items[0]
+		if util.CheckProtocol(pod.Status.PodIP) == kubeovn.ProtocolIPv6 {
+			return
+		}
+
+		output, err := exec.Command("kubectl", "ko", "trace", fmt.Sprintf("kube-system/%s", pod.Name), "114.114.114.114", "icmp").CombinedOutput()
+		Expect(err).NotTo(HaveOccurred(), string(output))
+
+		output, err = exec.Command("kubectl", "ko", "trace", fmt.Sprintf("kube-system/%s", pod.Name), "114.114.114.114", "tcp", "80").CombinedOutput()
+		Expect(err).NotTo(HaveOccurred(), string(output))
+
+		output, err = exec.Command("kubectl", "ko", "trace", fmt.Sprintf("kube-system/%s", pod.Name), "114.114.114.114", "udp", "53").CombinedOutput()
+		Expect(err).NotTo(HaveOccurred(), string(output))
+	})
+})

test/e2e-vlan-single-nic/node/node.go

@@ -36,10 +36,12 @@ var _ = Describe("[Vlan Node]", func() {
 	f := framework.NewFramework("node", fmt.Sprintf("%s/.kube/config", os.Getenv("HOME")))
 
 	var network *nodeNetwork
-	if len(networkJSON) != 0 {
-		network = new(nodeNetwork)
-		Expect(json.Unmarshal(networkJSON, network)).NotTo(HaveOccurred())
-	}
+	BeforeEach(func() {
+		if len(networkJSON) != 0 {
+			network = new(nodeNetwork)
+			Expect(json.Unmarshal(networkJSON, network)).NotTo(HaveOccurred())
+		}
+	})
 
 	It("Single NIC", func() {
 		nodes, err := f.KubeClientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})

test/e2e/e2e_suite_test.go

@@ -1,7 +1,8 @@
-package e2e_test
+package e2e
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"testing"
@@ -20,9 +21,19 @@ import (
 	_ "github.com/kubeovn/kube-ovn/test/e2e/node"
 	_ "github.com/kubeovn/kube-ovn/test/e2e/service"
 	_ "github.com/kubeovn/kube-ovn/test/e2e/subnet"
-	_ "github.com/kubeovn/kube-ovn/test/e2e/underlay"
+	"github.com/kubeovn/kube-ovn/test/e2e/underlay"
 )
 
+type nodeNetwork struct {
+	Gateway             string
+	IPAddress           string
+	IPPrefixLen         int
+	IPv6Gateway         string
+	GlobalIPv6Address   string
+	GlobalIPv6PrefixLen int
+	MacAddress          string
+}
+
 func TestE2e(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "Kube-OVN E2E Suite")
@@ -47,6 +58,16 @@ var _ = SynchronizedAfterSuite(func() {}, func() {
 	if err != nil {
 		Fail(err.Error())
 	}
+
+	err = f.OvnClientSet.KubeovnV1().Vlans().DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: "e2e=true"})
+	if err != nil {
+		Fail(err.Error())
+	}
+
+	err = f.OvnClientSet.KubeovnV1().ProviderNetworks().DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: "e2e=true"})
+	if err != nil {
+		Fail(err.Error())
+	}
 })
 
 var _ = SynchronizedBeforeSuite(func() []byte {
@@ -80,5 +101,147 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	if err != nil {
 		Fail(err.Error())
 	}
+
+	// underlay
+	var underlayNodeIPs []string
+	var underlayCIDR, underlayGateway string
+	for node, network := range nodeNetworks {
+		var info nodeNetwork
+		if err = json.Unmarshal([]byte(network), &info); err != nil {
+			Fail("invalid node network information: " + err.Error())
+		}
+
+		underlay.SetNodeMac(node, info.MacAddress)
+		if info.IPAddress != "" {
+			underlay.AddNodeIP(info.IPAddress)
+			underlayNodeIPs = append(underlayNodeIPs, info.IPAddress)
+			underlay.AddNodeAddrs(node, fmt.Sprintf("%s/%d", info.IPAddress, info.IPPrefixLen))
+			if underlayCIDR == "" {
+				underlayCIDR = fmt.Sprintf("%s/%d", info.IPAddress, info.IPPrefixLen)
+			}
+		}
+		if info.GlobalIPv6Address != "" {
+			underlay.AddNodeAddrs(node, fmt.Sprintf("%s/%d", info.GlobalIPv6Address, info.GlobalIPv6PrefixLen))
+		}
+		if info.Gateway != "" {
+			underlay.AddNodeRoutes(node, fmt.Sprintf("default via %s ", info.Gateway))
+			if underlayGateway == "" {
+				underlayGateway = info.Gateway
+			}
+		}
+		if info.IPv6Gateway != "" {
+			underlay.AddNodeRoutes(node, fmt.Sprintf("default via %s ", info.IPv6Gateway))
+		}
+	}
+	underlay.SetCIDR(underlayCIDR)
+
+	nodes, err := f.KubeClientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		Fail(err.Error())
+	}
+	cniPods, err := f.KubeClientSet.CoreV1().Pods("kube-system").List(context.Background(), metav1.ListOptions{LabelSelector: "app=kube-ovn-cni"})
+	if err != nil {
+		Fail(err.Error())
+	}
+
+	for i := range nodes.Items {
+		var nodeIP string
+		for _, addr := range nodes.Items[i].Status.Addresses {
+			if addr.Type == corev1.NodeInternalIP {
+				nodeIP = addr.Address
+				break
+			}
+		}
+		if nodeIP == "" {
+			Fail("failed to get IP of node " + nodes.Items[i].Name)
+		}
+
+		var cniPod *corev1.Pod
+		for _, pod := range cniPods.Items {
+			if pod.Status.HostIP == nodeIP {
+				cniPod = &pod
+				break
+			}
+		}
+		if cniPod == nil {
+			Fail("failed to get CNI pod on node " + nodes.Items[i].Name)
+		}
+
+		// change MTU
+		mtu := 1500 - (i+1)*5
+		cmd := fmt.Sprintf("ip link set %s mtu %d", underlay.ProviderInterface, mtu)
+		if _, _, err = f.ExecToPodThroughAPI(cmd, "cni-server", cniPod.Name, cniPod.Namespace, nil); err != nil {
+			Fail(fmt.Sprintf("failed to set MTU of %s on node %s: %v", underlay.ProviderInterface, nodes.Items[i].Name, err))
+		}
+		underlay.SetNodeMTU(nodes.Items[i].Name, mtu)
+	}
+
+	ns := corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   underlay.Namespace,
+			Labels: map[string]string{"e2e": "true"},
+		},
+	}
+	if _, err = f.KubeClientSet.CoreV1().Namespaces().Create(context.Background(), &ns, metav1.CreateOptions{}); err != nil {
+		Fail(err.Error())
+	}
+
+	// create provider network
+	pn := kubeovn.ProviderNetwork{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   underlay.ProviderNetwork,
+			Labels: map[string]string{"e2e": "true"},
+		},
+		Spec: kubeovn.ProviderNetworkSpec{
+			DefaultInterface: underlay.ProviderInterface,
+		},
+	}
+	if _, err = f.OvnClientSet.KubeovnV1().ProviderNetworks().Create(context.Background(), &pn, metav1.CreateOptions{}); err != nil {
+		Fail("failed to create provider network: " + err.Error())
+	}
+	if err = f.WaitProviderNetworkReady(pn.Name); err != nil {
+		Fail("provider network failed: " + err.Error())
+	}
+
+	// create vlan
+	vlan := kubeovn.Vlan{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   underlay.Vlan,
+			Labels: map[string]string{"e2e": "true"},
+		},
+		Spec: kubeovn.VlanSpec{
+			ID:       0,
+			Provider: pn.Name,
+		},
+	}
+	if _, err = f.OvnClientSet.KubeovnV1().Vlans().Create(context.Background(), &vlan, metav1.CreateOptions{}); err != nil {
+		Fail("failed to create vlan: " + err.Error())
+	}
+	if err = f.WaitProviderNetworkReady(pn.Name); err != nil {
+		Fail("provider network failed: " + err.Error())
+	}
+
+	// create subnet
+	subnet := kubeovn.Subnet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   underlay.Subnet,
+			Labels: map[string]string{"e2e": "true"},
+		},
+		Spec: kubeovn.SubnetSpec{
+			CIDRBlock:       underlayCIDR,
+			Gateway:         underlayGateway,
+			ExcludeIps:      underlayNodeIPs,
+			Vlan:            vlan.Name,
+			UnderlayGateway: true,
+			Namespaces:      []string{underlay.Namespace},
+		},
+	}
+	if _, err = f.OvnClientSet.KubeovnV1().Subnets().Create(context.Background(), &subnet, metav1.CreateOptions{}); err != nil {
+		Fail("failed to create subnet: " + err.Error())
+	}
+	if err = f.WaitSubnetReady(subnet.Name); err != nil {
+		Fail("subnet failed: " + err.Error())
+	}
+
 	return nil
 }, func(data []byte) {})

test/e2e/underlay/network.go → test/e2e/network.go

@@ -1,3 +1,3 @@
-package underlay
+package e2e
 
 var nodeNetworks = map[string]string{}