From d51e6a1169db6a3b06ffbf92c516f7f849bd0c00 Mon Sep 17 00:00:00 2001 From: Adrian Moreno Date: Thu, 10 Dec 2020 17:55:28 +0100 Subject: [PATCH] docs: document vdpa device type Signed-off-by: Adrian Moreno --- README.md | 12 ++++++- docs/README.md | 1 + docs/vdpa/README.md | 71 ++++++++++++++++++++++++++++++++++++++++ docs/vdpa/configMap.yaml | 30 +++++++++++++++++ 4 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 docs/vdpa/README.md create mode 100644 docs/vdpa/configMap.yaml diff --git a/README.md b/README.md index 4dee8633e..f1d881bb5 100644 --- a/README.md +++ b/README.md @@ -181,6 +181,15 @@ This plugin creates device plugin endpoints based on the configurations given in "isRdma": true } }, + { + "resourceName": "ct6dx_vdpa_vhost", + "selectors": { + "vendors": ["15b3"], + "devices": ["101e"], + "drivers": ["mlx5_core"], + "vdpaType": "vhost" + } + }, { "resourceName": "intel_fpga", "deviceType": "accelerator", @@ -231,8 +240,9 @@ This selector is applicable when "deviceType" is "netDevice"(note: this is defau | "rootDevices" | N | VFs from PF matches list of PF PCI addresses | `string` list Default: `null` | "rootDevices": ["0000:86:00.0"] (See follow-up sections for some advance usage of "rootDevices") | | "linkTypes" | N | The link type of the net device associated with the PCI device | `string` list Default: `null` | "linkTypes": ["ether"] | | "ddpProfiles" | N | A map of device selectors | `string` list Default: `null` | "ddpProfiles": ["GTPv1-C/U IPv4/IPv6 payload"] | -| "isRdma" | N | Mount RDMA resources | `bool` values `true` or `false` Default: `false` | "isRdma": `true` | +| "isRdma" | N | Mount RDMA resources. Incompatible with vdpaType | `bool` values `true` or `false` Default: `false` | "isRdma": `true` | | "needVhostNet"| N | Share /dev/vhost-net and /dev/net/tun | `bool` values `true` or `false` Default: `false` | "needVhostNet": `true` | +| "vdpaType" | N | The type of vDPA device (virtio, vhost). Incompatible with isRdma = true | `string` values `vhost` or `virtio` Default: `null` | "vdpaType": "vhost" | [//]: # (The tables above generated using: https://ozh.github.io/ascii-tables/) diff --git a/docs/README.md b/docs/README.md index d463d4f74..7eeb6f1ce 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,3 +7,4 @@ This page contains supplimentary documention that users may find useful for vari * [Running RDMA application in Kubernetes](rdma/) * [SR-IOV Network Device Plugin with DDP](ddp/) * [Using node specific config file for running device plugin DaemonSet](config-file) +* [Using vDPA devices in Kubernetes](vdpa/) diff --git a/docs/vdpa/README.md b/docs/vdpa/README.md new file mode 100644 index 000000000..dcf0ca176 --- /dev/null +++ b/docs/vdpa/README.md @@ -0,0 +1,71 @@ +# Using vDPA devices in Kubernetes +## Introduction to vDPA +vDPA (Virtio DataPath Acceleration) is a technology that enables the acceleration +of virtIO devices while allowing the implementations of such devices +(e.g: NIC vendors) to use their own control plane. + +The consumers of the virtIO devices (VMs or containers) interact with the devices +using the standard virtIO datapath and virtio-compatible control paths (virtIO, vhost). +While the data-plane is mapped directly to the accelerator device, the control-plane +gets translated by the vDPA kernel framework. + +The vDPA kernel framework is composed of the vDPA bus (/sys/bus/vdpa), vDPA devices +(/sys/bus/vdpa/devices) and vDPA drivers (/sys/bus/vdpa/drivers). +Currently, two vDPA drivers are implemented: +* virtio_vdpa: Exposes the device as a virtio-net netdev +* vhost_vdpa: Exposes the device as a vhost-vdpa device. This device uses an extension +of the vhost-net protocol to allow userspace applications access the rings directly + +For more information about the vDPA framework, read the article on +[LWN.net](https://lwn.net/Articles/816063/) or the blog series written by one of the +main authors ([part 1](https://www.redhat.com/en/blog/vdpa-kernel-framework-part-1-vdpa-bus-abstracting-hardware), +[part 2](https://www.redhat.com/en/blog/vdpa-kernel-framework-part-2-vdpa-bus-drivers-kernel-subsystem-interactions), +[part3](https://www.redhat.com/en/blog/vdpa-kernel-framework-part-3-usage-vms-and-containers)) + +## vDPA Management +Currently, the management of vDPA devices is performed using the sysfs interface exposed +by the vDPA Framework. However, in order to decouple the management of vDPA devices from +the SR-IOV Device Plugin functionality, this low-level management is done in an external +library called [go-vdpa](https://github.com/k8snetworkplumbingwg/govdpa). + +In the context of the SR-IOV Device Plugin and the SR-IOV CNI, the current plan is to +support only 1:1 mappings between SR-IOV VFs and vDPA devices despite the fact that +the vDPA Framework might support 1:N mappings. + +Note that vDPA and RDMA are mutually exclusive modes. + +## Tested NICs: +* Mellanox ConnectX®-6 DX + +## Prerequisites +* Linux Kernel >= 5.12 +* iproute >= 5.14 + +## vDPA device creation +Insert the vDPA kernel modules if not present: + + $ modprobe vdpa + $ modprobe virtio-vdpa + $ modprobe vhost-vdpa + +Create a vDPA device using the `vdpa` management tool integrated into iproute2, e.g: + + $ vdpa mgmtdev show + pci/0000:65:00.2: + supported_classes net + $ vdpa dev add name vdpa2 mgmtdev pci/0000:65:00.2 + $ vdpa dev list + vdpa2: type network mgmtdev pci/0000:65:00.2 vendor_id 5555 max_vqs 16 max_vq_size 256 + +## Bind the desired vDPA driver +The vDPA bus works similar to the pci bus. To unbind a driver from a device, run: + + echo ${DEV_NAME} > /sys/bus/vdpa/devices/${DEV_NAME}/driver/unbind + +To bind a driver to a device, run: + + echo ${DEV_NAME} > /sys/bus/vdpa/drivers/${DRIVER_NAME}/bind + +## Configure the SR-IOV Device Plugin +See the sample [configMap](configMap.yaml) for an example of how to configure a vDPA device. + diff --git a/docs/vdpa/configMap.yaml b/docs/vdpa/configMap.yaml new file mode 100644 index 000000000..70e99444f --- /dev/null +++ b/docs/vdpa/configMap.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: sriovdp-config + namespace: kube-system +data: + config.json: | + { + "resourceList": [{ + { + "resourceName": "vdpa_mlx_virtio", + "selectors": { + "vendors": ["15b3"], + "devices": ["101e"], + "drivers": ["mlx5_core"], + "vdpaType": "virtio" + } + }, + { + "resourceName": "vdpa_mlx_vhost", + "selectors": { + "vendors": ["15b3"], + "devices": ["101e"], + "drivers": ["mlx5_core"], + "vdpaType": "vhost" + } + } + ] + } +