From 2d062ec65c46045fc1067b26a1df3c2c197e2962 Mon Sep 17 00:00:00 2001 From: Charles CAPORALI Date: Thu, 31 Oct 2024 17:37:42 +0100 Subject: [PATCH 1/3] Adding support of UDN usage Using the UDN ip from the Server pod annotations. Service and IPv6 are not supported Using --udn option create a UserDefinedNetwork object on the netperf ns --- cmd/k8s-netperf/k8s-netperf.go | 31 +++++++++++++++-- pkg/config/config.go | 1 + pkg/k8s/kubernetes.go | 61 ++++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 3 deletions(-) diff --git a/cmd/k8s-netperf/k8s-netperf.go b/cmd/k8s-netperf/k8s-netperf.go index 44fcbebf..8c4dca45 100644 --- a/cmd/k8s-netperf/k8s-netperf.go +++ b/cmd/k8s-netperf/k8s-netperf.go @@ -39,6 +39,7 @@ var ( netperf bool iperf3 bool uperf bool + udn bool acrossAZ bool full bool vm bool @@ -151,19 +152,36 @@ var rootCmd = &cobra.Command{ os.Exit(1) } - if vm { - s.VM = true + if udn { + s.Udn = true // Create a dynamic client dynClient, err := dynamic.NewForConfig(rconfig) if err != nil { log.Error(err) } + s.DClient = dynClient + err = k8s.DeployL2Udn(dynClient) + if err != nil { + log.Error(err) + os.Exit(1) + } + } + + if vm { + s.VM = true + // Create a dynamic client + if s.DClient == nil { + dynClient, err := dynamic.NewForConfig(rconfig) + if err != nil { + log.Error(err) + } + s.DClient = dynClient + } kclient, err := kubevirtv1.NewForConfig(rconfig) if err != nil { log.Error(err) } s.KClient = kclient - s.DClient = dynClient } // Build the SUT (Deployments) @@ -374,6 +392,7 @@ func executeWorkload(nc config.Config, hostNet bool, driverName string, virt bool) result.Data { serverIP := "" + var err error Client := s.Client var driver drivers.Driver if nc.Service { @@ -384,6 +403,11 @@ func executeWorkload(nc config.Config, } else { serverIP = s.NetperfService.Spec.ClusterIP } + } else if s.Udn { + serverIP, err = k8s.ExtractUdnIp(s) + if err != nil { + log.Fatal(err) + } } else { if hostNet { serverIP = s.ServerHost.Items[0].Status.PodIP @@ -481,6 +505,7 @@ func main() { rootCmd.Flags().BoolVar(&acrossAZ, "across", false, "Place the client and server across availability zones") rootCmd.Flags().BoolVar(&full, "all", false, "Run all tests scenarios - hostNet and podNetwork (if possible)") rootCmd.Flags().BoolVar(&debug, "debug", false, "Enable debug log") + rootCmd.Flags().BoolVar(&udn, "udn", false, "Create and use a UDN called 'udn-l2-primary' as primary network.") rootCmd.Flags().StringVar(&promURL, "prom", "", "Prometheus URL") rootCmd.Flags().StringVar(&id, "uuid", "", "User provided UUID") rootCmd.Flags().StringVar(&searchURL, "search", "", "OpenSearch URL, if you have auth, pass in the format of https://user:pass@url:port") diff --git a/pkg/config/config.go b/pkg/config/config.go index 8195d2c0..44a0cb62 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -38,6 +38,7 @@ type PerfScenarios struct { Configs []Config VM bool VMHost string + Udn bool ServerNodeInfo metrics.NodeInfo ClientNodeInfo metrics.NodeInfo Client apiv1.PodList diff --git a/pkg/k8s/kubernetes.go b/pkg/k8s/kubernetes.go index 8d681f8d..6cd1a99d 100644 --- a/pkg/k8s/kubernetes.go +++ b/pkg/k8s/kubernetes.go @@ -5,6 +5,8 @@ import ( "fmt" "strings" + "encoding/json" + "github.com/cloud-bulldozer/k8s-netperf/pkg/config" log "github.com/cloud-bulldozer/k8s-netperf/pkg/logging" "github.com/cloud-bulldozer/k8s-netperf/pkg/metrics" @@ -12,9 +14,12 @@ import ( corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/utils/pointer" ) @@ -72,6 +77,7 @@ const clientAcrossRole = "client-across" const hostNetServerRole = "host-server" const hostNetClientRole = "host-client" const k8sNetperfImage = "quay.io/cloud-bulldozer/k8s-netperf:latest" +const udnName = "udn-l2-primary" // BuildInfra will create the infra for the SUT func BuildInfra(client *kubernetes.Clientset) error { @@ -124,6 +130,39 @@ func BuildInfra(client *kubernetes.Clientset) error { return nil } +// Create a User Defined Network for the tests +func DeployL2Udn(dynamicClient *dynamic.DynamicClient) error { + udn := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "k8s.ovn.org/v1", + "kind": "UserDefinedNetwork", + "metadata": map[string]interface{}{ + "name": udnName, + "namespace": "netperf", + }, + "spec": map[string]interface{}{ + "topology": "Layer2", + "layer2": map[string]interface{}{ + "role": "Primary", + "subnets": []string{"10.0.0.0/24", "2001:db8::/60"}, + }, + }, + }, + } + + // Specify the GVR for UDN + gvr := schema.GroupVersionResource{ + Group: "k8s.ovn.org", + Version: "v1", + Resource: "userdefinednetworks", + } + _, err := dynamicClient.Resource(gvr).Namespace(namespace).Create(context.TODO(), udn, metav1.CreateOptions{}) + if err != nil { + return err + } + return nil +} + // BuildSUT Build the k8s env to run network performance tests func BuildSUT(client *kubernetes.Clientset, s *config.PerfScenarios) error { var netperfDataPorts []int32 @@ -449,6 +488,28 @@ func BuildSUT(client *kubernetes.Clientset, s *config.PerfScenarios) error { return nil } +// Extract the UDN Ip address of a pod from the annotations - Support only ipv4 +func ExtractUdnIp(s config.PerfScenarios) (string, error) { + podNetworksJson := s.Server.Items[0].Annotations["k8s.ovn.org/pod-networks"] + var podNetworks map[string]interface{} + err := json.Unmarshal([]byte(podNetworksJson), &podNetworks) + if err != nil { + return "", err + } + UdnJson := podNetworks[namespace+"/"+udnName].(map[string]interface{}) + UdnIpAddreses := UdnJson["ip_addresses"].([]interface{}) + // Extract the IPv4 address + var ipv4 string + for _, ip := range UdnIpAddreses { + ipStr := ip.(string) + if strings.Contains(ipStr, ".") { // Check if it's an IPv4 address + ipv4 = strings.Split(ipStr, "/")[0] // Extract the IP address part before the subnet + break + } + } + return ipv4, nil +} + // launchServerVM will create the ServerVM with the specific node and pod affinity. func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) error { _, err := CreateVMServer(perf.KClient, serverRole, serverRole, *podAff, *nodeAff) From b52028510563692cdebb1f69855fce5499d5386a Mon Sep 17 00:00:00 2001 From: Charles CAPORALI Date: Thu, 7 Nov 2024 15:20:49 +0100 Subject: [PATCH 2/3] updating the README.md with the latest features --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 5280e639..6fa1d79c 100644 --- a/README.md +++ b/README.md @@ -70,15 +70,19 @@ Flags: --clean Clean-up resources created by k8s-netperf (default true) --json Instead of human-readable output, return JSON to stdout --local Run network performance tests with Server-Pods/Client-Pods on the same Node + --vm Launch Virtual Machines instead of pods for client/servers --across Place the client and server across availability zones --all Run all tests scenarios - hostNet and podNetwork (if possible) --debug Enable debug log + --udn Create and use a UDN called 'udn-l2-primary' as a primary network. --prom string Prometheus URL --uuid string User provided UUID --search string OpenSearch URL, if you have auth, pass in the format of https://user:pass@url:port + --index string OpenSearch Index to save the results to, defaults to k8s-netperf --metrics Show all system metrics retrieved from prom --tcp-tolerance float Allowed %diff from hostNetwork to podNetwork, anything above tolerance will result in k8s-netperf exiting 1. (default 10) --version k8s-netperf version + --csv Archive results, cluster and benchmark metrics in CSV files (default true) -h, --help help for k8s-netperf From 909534252d0e6ef92ad35e3f8efb2fd9c24a7a7c Mon Sep 17 00:00:00 2001 From: Charles CAPORALI Date: Tue, 19 Nov 2024 18:11:28 +0100 Subject: [PATCH 3/3] Using a dedicated struct to extract NetworkData from the annotations --- pkg/k8s/kubernetes.go | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/pkg/k8s/kubernetes.go b/pkg/k8s/kubernetes.go index 6cd1a99d..36e953a1 100644 --- a/pkg/k8s/kubernetes.go +++ b/pkg/k8s/kubernetes.go @@ -2,11 +2,11 @@ package k8s import ( "context" + "encoding/json" "fmt" + "net" "strings" - "encoding/json" - "github.com/cloud-bulldozer/k8s-netperf/pkg/config" log "github.com/cloud-bulldozer/k8s-netperf/pkg/logging" "github.com/cloud-bulldozer/k8s-netperf/pkg/metrics" @@ -49,6 +49,13 @@ type ServiceParams struct { DataPorts []int32 } +type PodNetworksData struct { + IPAddresses []string `json:"ip_addresses"` + MacAddress string `json:"mac_address"` + GatewayIPs []string `json:"gateway_ips"` + Role string `json:"role"` +} + const sa string = "netperf" const namespace string = "netperf" @@ -132,6 +139,7 @@ func BuildInfra(client *kubernetes.Clientset) error { // Create a User Defined Network for the tests func DeployL2Udn(dynamicClient *dynamic.DynamicClient) error { + log.Infof("Deploying L2 Primary UDN in the NS : %s", namespace) udn := &unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "k8s.ovn.org/v1", @@ -491,23 +499,30 @@ func BuildSUT(client *kubernetes.Clientset, s *config.PerfScenarios) error { // Extract the UDN Ip address of a pod from the annotations - Support only ipv4 func ExtractUdnIp(s config.PerfScenarios) (string, error) { podNetworksJson := s.Server.Items[0].Annotations["k8s.ovn.org/pod-networks"] - var podNetworks map[string]interface{} - err := json.Unmarshal([]byte(podNetworksJson), &podNetworks) + // + var root map[string]json.RawMessage + err := json.Unmarshal([]byte(podNetworksJson), &root) + if err != nil { + fmt.Println("Error unmarshalling JSON:", err) + return "", err + } + // + var udnData PodNetworksData + err = json.Unmarshal(root["netperf/"+udnName], &udnData) if err != nil { return "", err } - UdnJson := podNetworks[namespace+"/"+udnName].(map[string]interface{}) - UdnIpAddreses := UdnJson["ip_addresses"].([]interface{}) // Extract the IPv4 address - var ipv4 string - for _, ip := range UdnIpAddreses { - ipStr := ip.(string) - if strings.Contains(ipStr, ".") { // Check if it's an IPv4 address - ipv4 = strings.Split(ipStr, "/")[0] // Extract the IP address part before the subnet - break + var ipv4 net.IP + for _, ip := range udnData.IPAddresses { + if strings.Contains(ip, ".") { // Check if it's an IPv4 address + ipv4, _, err = net.ParseCIDR(ip) + if err != nil { + return "", err + } } } - return ipv4, nil + return ipv4.String(), nil } // launchServerVM will create the ServerVM with the specific node and pod affinity.