From 069d2ac2131bd59a25e18c3d4d6c2958aa603492 Mon Sep 17 00:00:00 2001 From: Adrian Moreno Date: Thu, 10 Dec 2020 17:55:28 +0100 Subject: [PATCH] document vdpa device type Signed-off-by: Adrian Moreno --- README.md | 1 + docs/README.md | 1 + docs/vdpa/README.md | 24 ++++++++++++++++++++++++ docs/vdpa/configMap.yaml | 30 ++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+) create mode 100644 docs/vdpa/README.md create mode 100644 docs/vdpa/configMap.yaml diff --git a/README.md b/README.md index f31a52bcf..15d1ae22f 100644 --- a/README.md +++ b/README.md @@ -258,6 +258,7 @@ This selector is applicable when "deviceType" is "netDevice"(note: this is defau | "ddpProfiles" | N | A map of device selectors | `string` list Default: `null` | "ddpProfiles": ["GTPv1-C/U IPv4/IPv6 payload"] | | "isRdma" | N | Mount RDMA resources | `bool` values `true` or `false` Default: `false` | "isRdma": `true` | | "needVhostNet"| N | Share /dev/vhost-net | `bool` values `true` or `false` Default: `false` | "needVhostNet": `true` | +| "vdpaType" | N | The type of vDPA device (virtio, vhost or `nil`) | `string` values `vhost` or `virtio` Default: `null` | "vdpaType": "vhost" | [//]: # (The tables above generated using: https://ozh.github.io/ascii-tables/) diff --git a/docs/README.md b/docs/README.md index 29d403c56..71d25a9de 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,3 +7,4 @@ This page contains supplimentary documention that users may find useful for vari * [Running RDMA application in Kubernetes](rdma/) * [SR-IOV network device plugin with DDP](ddp/) * [Using node specific config file for running device plugin DaemonSet](config-file) +* [Using vDPA devices in Kuberenets](vdpa/) diff --git a/docs/vdpa/README.md b/docs/vdpa/README.md new file mode 100644 index 000000000..2daef719b --- /dev/null +++ b/docs/vdpa/README.md @@ -0,0 +1,24 @@ +# Using vDPA devices in Kuberenets + +vDPA-capable devices can be used to expose an accelerated, yet standard interface to pods. These devices implement the virtIO ring layout that is used for the data-plane. +The control plane, however, is translated by the vDPA kernel framework which takes care of configuring the virtio devices and virtqueues. + +The vDPA kernel framework is composed of a vdpa bus (/sys/bus/vdpa), vdpa devices (/sys/bus/vdpa/devices) and vdpa drivers (/sys/bus/vdpa/drivers). Currently, two vdpa drivers are implemented: +* virtio_vdpa: Exposes the device as a virtio-net netdev +* virtio_vdpa: Exposes the device as a vhost-vdpa device. This device uses an extension of the vhost-net protocol to allow userspace applications access the rings directly + +## Supported NICs: +* Mellanox ConnectX®-6 DX + +## vDPA device creation +Each NIC might require diffent steps to create vDPA devices on top of the VFs. + +## Bind the desired vdpa driver +The vdpa bus works similar to the pci bus. To unbind a driver from a device, run: +``echo ${DEV_NAME} > /sys/bus/vdpa/devices/${DEV_NAME}/driver/unbind`` + +To bind a driver to a device, run: +``echo ${DEV_NAME} > /sys/bus/vdpa/drivers/${DRIVER_NAME}/bind`` + +## Priviledges +IPC_LOCK capability privilege is required for "vhost" mode to be used in a Kubernetes Pod. diff --git a/docs/vdpa/configMap.yaml b/docs/vdpa/configMap.yaml new file mode 100644 index 000000000..70e99444f --- /dev/null +++ b/docs/vdpa/configMap.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: sriovdp-config + namespace: kube-system +data: + config.json: | + { + "resourceList": [{ + { + "resourceName": "vdpa_mlx_virtio", + "selectors": { + "vendors": ["15b3"], + "devices": ["101e"], + "drivers": ["mlx5_core"], + "vdpaType": "virtio" + } + }, + { + "resourceName": "vdpa_mlx_vhost", + "selectors": { + "vendors": ["15b3"], + "devices": ["101e"], + "drivers": ["mlx5_core"], + "vdpaType": "vhost" + } + } + ] + } +