Support multiple SR-IOV networks

This patch adopts changes from the latest SR-IOV device plugin master [1] that
allow us to attach a VMI to multiple SR-IOV Multus networks. (Before the
change, devices allocated for multiple networks could end up in different boot
order than configured.)

Among other things, this patch adds read access to network attachment
definitions to virt-controller role. This is to allow the controller to fetch
network CRD objects and determine their respective resource names. For this
same reason, it extends KubevirtClient object to include NetworkClient().

We do *not* maintain backwards compatibility with older SR-IOV device plugin
releases. This is as intended because SR-IOV support is still considered
experimental, and there is no good reason for us to support the device plugin
release that does not support multiple networks.

Updated documentation to reflect the fact that kubevirt is now compatible with
device plugin master (and *not* compatible with v1.0).

todo: get rid of vendor/* change

[1] k8snetworkplumbingwg/sriov-network-device-plugin#26

docs/sriov.md

@@ -159,17 +159,24 @@ $ ./cluster/kubectl.sh create -f $GOPATH/src/github.com/intel/multus-cni/images/
 
 Now, deploy SR-IOV device plugin.
 
-Note: as of the time of writing, latest master of SR-IOV device plugin is not
-compatible with kubevirt. Please check out the latest version supported by
-kubevirt before building the plugin image:
-
-
 ```
 $ go get -u -d github.com/intel/sriov-network-device-plugin/
 $ cd $GOPATH/src/github.com/intel/sriov-network-device-plugin/images
-$ git checkout v1.0
 $ ./build_docker.sh
 $ vi images/sriovdp-daemonset.yaml # change to refer to local sriov-network-device-plugin:latest
+$ cat <<EOF > /etc/pcidp/config.json
+{
+    "resourceList":
+    [
+        {
+            "resourceName": "sriov",
+            "rootDevices": ["05:00.0", "05:00.1"],
+            "sriovMode": true,
+            "deviceType": "vfio"
+        }
+    ]
+}
+EOF
 $ ./cluster/kubectl.sh create -f $GOPATH/src/github.com/intel/sriov-network-device-plugin/images/sriovdp-daemonset.yaml
 ```
 

manifests/dev/rbac.authorization.k8s.yaml.in

@@ -218,6 +218,14 @@ rules:
       - '*'
     verbs:
       - '*'
+  - apiGroups:
+      - k8s.cni.cncf.io
+    resources:
+      - network-attachment-definitions
+    verbs:
+      - get
+      - list
+      - watch
 ---
 apiVersion: v1
 kind: ServiceAccount

manifests/release/kubevirt.yaml.in

@@ -314,6 +314,14 @@ rules:
       - '*'
     verbs:
       - '*'
+  - apiGroups:
+      - k8s.cni.cncf.io
+    resources:
+      - network-attachment-definitions
+    verbs:
+      - get
+      - list
+      - watch
 ---
 apiVersion: v1
 kind: ServiceAccount

pkg/kubecli/generated_mock_kubevirt.go

@@ -7,6 +7,7 @@ import (
 	time "time"
 
 	gomock "github.com/golang/mock/gomock"
+	versioned "github.com/phoracek/network-attachment-definition-client/pkg/client/clientset/versioned"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	types "k8s.io/apimachinery/pkg/types"
 	discovery "k8s.io/client-go/discovery"
@@ -41,7 +42,7 @@ import (
 	v1beta110 "k8s.io/client-go/kubernetes/typed/storage/v1beta1"
 	rest "k8s.io/client-go/rest"
 
-	versioned "kubevirt.io/containerized-data-importer/pkg/client/clientset/versioned"
+	versioned0 "kubevirt.io/containerized-data-importer/pkg/client/clientset/versioned"
 	v19 "kubevirt.io/kubevirt/pkg/api/v1"
 )
 
@@ -126,16 +127,26 @@ func (_mr *_MockKubevirtClientRecorder) RestClient() *gomock.Call {
 	return _mr.mock.ctrl.RecordCall(_mr.mock, "RestClient")
 }
 
-func (_m *MockKubevirtClient) CdiClient() versioned.Interface {
+func (_m *MockKubevirtClient) CdiClient() versioned0.Interface {
 	ret := _m.ctrl.Call(_m, "CdiClient")
-	ret0, _ := ret[0].(versioned.Interface)
+	ret0, _ := ret[0].(versioned0.Interface)
 	return ret0
 }
 
 func (_mr *_MockKubevirtClientRecorder) CdiClient() *gomock.Call {
 	return _mr.mock.ctrl.RecordCall(_mr.mock, "CdiClient")
 }
 
+func (_m *MockKubevirtClient) NetworkClient() versioned.Interface {
+	ret := _m.ctrl.Call(_m, "NetworkClient")
+	ret0, _ := ret[0].(versioned.Interface)
+	return ret0
+}
+
+func (_mr *_MockKubevirtClientRecorder) NetworkClient() *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "NetworkClient")
+}
+
 func (_m *MockKubevirtClient) Discovery() discovery.DiscoveryInterface {
 	ret := _m.ctrl.Call(_m, "Discovery")
 	ret0, _ := ret[0].(discovery.DiscoveryInterface)

pkg/kubecli/kubecli.go

@@ -33,6 +33,8 @@ import (
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 
+	networkclient "github.com/phoracek/network-attachment-definition-client/pkg/client/clientset/versioned"
+
 	cdiclient "kubevirt.io/containerized-data-importer/pkg/client/clientset/versioned"
 	"kubevirt.io/kubevirt/pkg/api/v1"
 )
@@ -73,12 +75,18 @@ func GetKubevirtSubresourceClientFromFlags(master string, kubeconfig string) (Ku
 		return nil, err
 	}
 
+	networkClient, err := networkclient.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
 	return &kubevirt{
 		master,
 		kubeconfig,
 		restClient,
 		config,
 		cdiClient,
+		networkClient,
 		coreClient,
 	}, nil
 }
@@ -177,12 +185,18 @@ func GetKubevirtClientFromRESTConfig(config *rest.Config) (KubevirtClient, error
 		return nil, err
 	}
 
+	networkClient, err := networkclient.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
 	return &kubevirt{
 		master,
 		kubeconfig,
 		restClient,
 		config,
 		cdiClient,
+		networkClient,
 		coreClient,
 	}, nil
 }

pkg/kubecli/kubevirt.go

@@ -34,6 +34,8 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 
+	networkclient "github.com/phoracek/network-attachment-definition-client/pkg/client/clientset/versioned"
+
 	cdiclient "kubevirt.io/containerized-data-importer/pkg/client/clientset/versioned"
 	"kubevirt.io/kubevirt/pkg/api/v1"
 )
@@ -46,22 +48,28 @@ type KubevirtClient interface {
 	ServerVersion() *ServerVersion
 	RestClient() *rest.RESTClient
 	CdiClient() cdiclient.Interface
+	NetworkClient() networkclient.Interface
 	kubernetes.Interface
 }
 
 type kubevirt struct {
-	master     string
-	kubeconfig string
-	restClient *rest.RESTClient
-	config     *rest.Config
-	cdiClient  *cdiclient.Clientset
+	master        string
+	kubeconfig    string
+	restClient    *rest.RESTClient
+	config        *rest.Config
+	cdiClient     *cdiclient.Clientset
+	networkClient *networkclient.Clientset
 	*kubernetes.Clientset
 }
 
 func (k kubevirt) CdiClient() cdiclient.Interface {
 	return k.cdiClient
 }
 
+func (k kubevirt) NetworkClient() networkclient.Interface {
+	return k.networkClient
+}
+
 func (k kubevirt) RestClient() *rest.RESTClient {
 	return k.restClient
 }

pkg/virt-api/rest/subresource_test.go

@@ -84,7 +84,7 @@ var _ = Describe("VirtualMachineInstance Subresources", func() {
 			vmi := v1.NewMinimalVMI("testvmi")
 			vmi.Status.Phase = v1.Running
 			vmi.ObjectMeta.SetUID(uuid.NewUUID())
-			templateService := services.NewTemplateService("whatever", "whatever", "whatever", "whatever", configCache, pvcCache)
+			templateService := services.NewTemplateService("whatever", "whatever", "whatever", "whatever", configCache, pvcCache, app.VirtCli)
 
 			pod, err := templateService.RenderLaunchManifest(vmi)
 			Expect(err).ToNot(HaveOccurred())
@@ -256,7 +256,7 @@ var _ = Describe("VirtualMachineInstance Subresources", func() {
 			vmi.Status.Phase = v1.Running
 			vmi.ObjectMeta.SetUID(uuid.NewUUID())
 
-			templateService := services.NewTemplateService("whatever", "whatever", "whatever", "whatever", configCache, pvcCache)
+			templateService := services.NewTemplateService("whatever", "whatever", "whatever", "whatever", configCache, pvcCache, app.VirtCli)
 
 			pod, err := templateService.RenderLaunchManifest(vmi)
 			Expect(err).ToNot(HaveOccurred())

pkg/virt-controller/services/template.go

@@ -30,9 +30,12 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/tools/cache"
 
+	networkv1 "github.com/phoracek/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+
 	"kubevirt.io/kubevirt/pkg/api/v1"
 	"kubevirt.io/kubevirt/pkg/config"
 	"kubevirt.io/kubevirt/pkg/hooks"
+	"kubevirt.io/kubevirt/pkg/kubecli"
 	"kubevirt.io/kubevirt/pkg/log"
 	"kubevirt.io/kubevirt/pkg/precond"
 	registrydisk "kubevirt.io/kubevirt/pkg/registry-disk"
@@ -52,6 +55,8 @@ const VhostNetDevice = "devices.kubevirt.io/vhost-net"
 const CAP_NET_ADMIN = "NET_ADMIN"
 const CAP_SYS_NICE = "SYS_NICE"
 
+const MULTUS_RESOURCE_NAME_ANNOTATION = "k8s.v1.cni.cncf.io/resourceName"
+
 type TemplateService interface {
 	RenderLaunchManifest(*v1.VirtualMachineInstance) (*k8sv1.Pod, error)
 }
@@ -63,6 +68,7 @@ type templateService struct {
 	imagePullSecret            string
 	configMapStore             cache.Store
 	persistentVolumeClaimStore cache.Store
+	virtClient                 kubecli.KubevirtClient
 }
 
 type PvcNotFoundError error
@@ -592,6 +598,15 @@ func (t *templateService) RenderLaunchManifest(vmi *v1.VirtualMachineInstance) (
 	podLabels[v1.AppLabel] = "virt-launcher"
 	podLabels[v1.CreatedByLabel] = string(vmi.UID)
 
+	networkToResourceMap, err := getNetworkToResourceMap(t.virtClient, vmi)
+	if err != nil {
+		return nil, err
+	}
+	for networkName, resourceName := range networkToResourceMap {
+		varName := fmt.Sprintf("KUBEVIRT_RESOURCE_NAME_%s", networkName)
+		container.Env = append(container.Env, k8sv1.EnvVar{Name: varName, Value: resourceName})
+
+	}
 	containers = append(containers, container)
 
 	for i, requestedHookSidecar := range requestedHookSidecarList {
@@ -796,6 +811,30 @@ func getPortsFromVMI(vmi *v1.VirtualMachineInstance) []k8sv1.ContainerPort {
 	return ports
 }
 
+func getResourceNameForNetwork(network *networkv1.NetworkAttachmentDefinition) string {
+	resourceName, ok := network.Annotations[MULTUS_RESOURCE_NAME_ANNOTATION]
+	if ok {
+		return resourceName
+	}
+	return "" // meaning the network is not served by resources
+}
+
+func getNetworkToResourceMap(virtClient kubecli.KubevirtClient, vmi *v1.VirtualMachineInstance) (networkToResourceMap map[string]string, err error) {
+	networkToResourceMap = make(map[string]string)
+	for _, network := range vmi.Spec.Networks {
+		if network.Multus != nil {
+			networkName := network.Multus.NetworkName
+			namespace := precond.MustNotBeEmpty(vmi.GetObjectMeta().GetNamespace())
+			crd, err := virtClient.NetworkClient().K8sCniCncfIo().NetworkAttachmentDefinitions(namespace).Get(networkName, metav1.GetOptions{})
+			if err != nil {
+				return map[string]string{}, fmt.Errorf("Failed to locate network attachment definition %s", networkName)
+			}
+			networkToResourceMap[network.Name] = getResourceNameForNetwork(crd)
+		}
+	}
+	return
+}
+
 func getCniInterfaceList(vmi *v1.VirtualMachineInstance) (ifaceListString string, cniAnnotation string) {
 	ifaceList := make([]string, 0)
 
@@ -824,7 +863,8 @@ func NewTemplateService(launcherImage string,
 	ephemeralDiskDir string,
 	imagePullSecret string,
 	configMapCache cache.Store,
-	persistentVolumeClaimCache cache.Store) TemplateService {
+	persistentVolumeClaimCache cache.Store,
+	virtClient kubecli.KubevirtClient) TemplateService {
 
 	precond.MustNotBeEmpty(launcherImage)
 	svc := templateService{
@@ -834,6 +874,7 @@ func NewTemplateService(launcherImage string,
 		imagePullSecret:            imagePullSecret,
 		configMapStore:             configMapCache,
 		persistentVolumeClaimStore: persistentVolumeClaimCache,
+		virtClient:                 virtClient,
 	}
 	return &svc
 }