Merge pull request #2664 from amorenoz/vdpa

Add support to virtio-vDPA devices

docs/ovs_offload.md

@@ -226,3 +226,53 @@ The following manufacturers are known to work:
 - [Mellanox Bluefield-2](https://www.mellanox.com/products/bluefield2-overview)
 
 Deployment guide can be found [here](https://docs.google.com/document/d/1hRke0cOCY84Ef8OU283iPg_PHiJ6O17aUkb9Vv-fWPQ/edit?usp=sharing).
+
+## vDPA
+
+vDPA (Virtio DataPath Acceleration) is a technology that enables the acceleration of virtIO devices while
+allowing the implementations of such devices (e.g: NIC vendors) to use their own control plane.
+
+vDPA can be combined with the SR-IOV OVS Hardware offloading setup to expose the workload to an
+open standard interface such as virtio-net.
+
+### Additional Prerequisites:
+* Linux Kernel >= 5.12
+* iproute >= 5.14
+
+### Supported Hardware:
+- Mellanox ConnectX-6DX NIC
+
+### Additional configuration
+In addition to all the steps listed above, insert the virtio-vdpa driver and the mlx-vdpa driver:
+
+    $ modprobe vdpa
+    $ modprobe virtio-vdpa
+    $ modprobe mlx5-vdpa
+
+The the `vdpa` tool (part of iproute package) is used to create a vdpa device on top
+of an existing VF:
+
+    $ vdpa mgmtdev show
+    pci/0000:65:00.2:
+      supported_classes net
+    $ vdpa dev add name vdpa2 mgmtdev pci/0000:65:00.2
+    $ vdpa dev list
+    vdpa2: type network mgmtdev pci/0000:65:00.2 vendor_id 5555 max_vqs 16 max_vq_size 256
+
+After a device has been created, the SR-IOV Device Plugin plugin configuration has to be modified for it
+to select and expose the vdpa device:
+
+```json
+{
+    "resourceList": [
+         {
+            "resourceName": "cx6_sriov_vpda_virtio",
+            "selectors": {
+               "vendors": ["15b3"],
+               "devices": ["101e"],
+               "vdpaType": "virtio"
+            }
+        }
+    ]
+}
+```

go-controller/go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/google/uuid v1.2.0
 	github.com/gorilla/mux v1.8.0
 	github.com/j-keck/arping v1.0.2
+	github.com/k8snetworkplumbingwg/govdpa v0.1.4
 	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.3.0
 	github.com/miekg/dns v1.1.31
 	github.com/mitchellh/copystructure v1.2.0

go-controller/go.sum

@@ -514,6 +514,8 @@ github.com/juju/utils v0.0.0-20180808125547-9dfc6dbfb02b/go.mod h1:6/KLg8Wz/y2KV
 github.com/juju/version v0.0.0-20161031051906-1f41e27e54f2/go.mod h1:kE8gK5X0CImdr7qpSKl3xB2PmpySSmfj7zVbkZFs81U=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/k8snetworkplumbingwg/govdpa v0.1.4 h1:e6mM7JFZkLVJeMQw3px96EigHAhnb4VUlqhNub/2Psk=
+github.com/k8snetworkplumbingwg/govdpa v0.1.4/go.mod h1:UQR1xu7A+nnRK1dkLEi12OnNL0OiBPpIKOYDuaQQkck=
 github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.3.0 h1:MjRRgZyTGo90G+UrwlDQjU+uG4Z7By65qvQxGoILT/8=
 github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.3.0/go.mod h1:nqCI7aelBJU61wiBeeZWJ6oi4bJy5nrjkM6lWIMA4j0=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=

go-controller/pkg/cni/cni.go

@@ -11,6 +11,7 @@ import (
 	utilnet "k8s.io/utils/net"
 
 	current "github.com/containernetworking/cni/pkg/types/100"
+	"github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
@@ -95,6 +96,13 @@ func (pr *PodRequest) checkOrUpdatePodUID(podUID string) error {
 
 func (pr *PodRequest) getVFNetdevName() (string, error) {
 	// Get the vf device Name
+
+	// If a vDPA device exists, it takes preference over the vendor device, steering-wize
+	vdpaDevice, err := util.GetVdpaOps().GetVdpaDeviceByPci(pr.CNIConf.DeviceID)
+	if err == nil && vdpaDevice.Driver() == kvdpa.VirtioVdpaDriver {
+		return vdpaDevice.VirtioNet().NetDev(), nil
+	}
+
 	vfNetdevices, err := util.GetSriovnetOps().GetNetDevicesFromPci(pr.CNIConf.DeviceID)
 	if err != nil {
 		return "", err

go-controller/pkg/util/vdpa_linux.go

@@ -0,0 +1,37 @@
+package util
+
+import (
+	"github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa"
+)
+
+type VdpaDevice interface {
+	kvdpa.VdpaDevice
+}
+
+type VdpaOps interface {
+	GetVdpaDeviceByPci(pciAddress string) (kvdpa.VdpaDevice, error)
+}
+
+type defaultVdpaOps struct {
+}
+
+var vdpaOps VdpaOps = &defaultVdpaOps{}
+
+// SetVdpaOpsInst method should be used by unit tests in
+func SetVdpaOpsInst(mockInst VdpaOps) {
+	vdpaOps = mockInst
+}
+
+// GetVdpaOps will be invoked by functions in other packages that would need access to the govdpa library methods.
+func GetVdpaOps() VdpaOps {
+	return vdpaOps
+}
+
+func (v *defaultVdpaOps) GetVdpaDeviceByPci(pciAddress string) (kvdpa.VdpaDevice, error) {
+	// the PCI prefix is required by the govdpa library
+	vdpaDevices, err := kvdpa.GetVdpaDevicesByPciAddress("pci/" + pciAddress)
+	if len(vdpaDevices) > 0 {
+		return vdpaDevices[0], nil
+	}
+	return nil, err
+}