From 131f7325dcde8050c4c2a6b607743bc5a82a632d Mon Sep 17 00:00:00 2001 From: Adrian Moreno Date: Fri, 23 Apr 2021 09:51:22 +0200 Subject: [PATCH] sriov: support virtio vdpa devices Add support to sriov-based virtio-vdpa devices. When a sr-iov device has a vdpa device created, and it is bound to virtio_vdpa driver, use the associated virtio-net device instead of the vendor one. In order to determine whether a device is vdpa or not as well as its device path, use external library github.com/k8snetworkplumbingwg/govdpa Signed-off-by: Adrian Moreno --- go-controller/go.mod | 1 + go-controller/go.sum | 2 + go-controller/pkg/cni/cni.go | 8 + go-controller/pkg/util/mocks/VdpaDevice.go | 66 +++++ go-controller/pkg/util/mocks/VdpaOps.go | 36 +++ go-controller/pkg/util/vdpa_linux.go | 32 +++ .../k8snetworkplumbingwg/govdpa/LICENSE | 201 +++++++++++++++ .../govdpa/pkg/kvdpa/kvdpa.go | 242 ++++++++++++++++++ go-controller/vendor/modules.txt | 2 + 9 files changed, 590 insertions(+) create mode 100644 go-controller/pkg/util/mocks/VdpaDevice.go create mode 100644 go-controller/pkg/util/mocks/VdpaOps.go create mode 100644 go-controller/pkg/util/vdpa_linux.go create mode 100644 go-controller/vendor/github.com/k8snetworkplumbingwg/govdpa/LICENSE create mode 100644 go-controller/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/kvdpa.go diff --git a/go-controller/go.mod b/go-controller/go.mod index dec222dc13b..0f8e2f66d83 100644 --- a/go-controller/go.mod +++ b/go-controller/go.mod @@ -15,6 +15,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/juju/errors v0.0.0-20200330140219-3fe23663418f // indirect github.com/juju/testing v0.0.0-20200706033705-4c23f9c453cd // indirect + github.com/k8snetworkplumbingwg/govdpa v0.1.3 github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200626054723-37f83d1996bc github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/miekg/dns v1.1.31 diff --git a/go-controller/go.sum b/go-controller/go.sum index 68378451a88..2e3d0a90206 100644 --- a/go-controller/go.sum +++ b/go-controller/go.sum @@ -269,6 +269,8 @@ github.com/juju/utils v0.0.0-20180808125547-9dfc6dbfb02b/go.mod h1:6/KLg8Wz/y2KV github.com/juju/version v0.0.0-20161031051906-1f41e27e54f2/go.mod h1:kE8gK5X0CImdr7qpSKl3xB2PmpySSmfj7zVbkZFs81U= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k8snetworkplumbingwg/govdpa v0.1.3 h1:FZRhTMB1e3yWwSEy+l4eS73WioyMaL+vmFZ8JNwn+Uk= +github.com/k8snetworkplumbingwg/govdpa v0.1.3/go.mod h1:Jx2rlMquENdCd8M5Oc51xHCt10bQIXTloDU8F4nS4T4= github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200626054723-37f83d1996bc h1:M7bj0RX9dc79YIyptmHm0tPmC/WuIbn+HVeTBDK2KZw= github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200626054723-37f83d1996bc/go.mod h1:+1DpV8uIwteAhxNO0lgRox8gHkTG6w3OeDfAlg+qqjA= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/go-controller/pkg/cni/cni.go b/go-controller/pkg/cni/cni.go index 6fa0dbd059d..310f6f829b0 100644 --- a/go-controller/pkg/cni/cni.go +++ b/go-controller/pkg/cni/cni.go @@ -11,6 +11,7 @@ import ( utilnet "k8s.io/utils/net" "github.com/containernetworking/cni/pkg/types/current" + "github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" @@ -95,6 +96,13 @@ func (pr *PodRequest) checkOrUpdatePodUID(podUID string) error { func (pr *PodRequest) getVFNetdevName() (string, error) { // Get the vf device Name + + // If a vDPA device exists, it takes preference over the vendor device, steering-wize + vdpaDevice, err := util.GetVdpaOps().GetVdpaDeviceByPci(pr.CNIConf.DeviceID) + if err == nil && vdpaDevice.GetDriver() == kvdpa.VirtioVdpaDriver { + return vdpaDevice.GetNetDev(), nil + } + vfNetdevices, err := util.GetSriovnetOps().GetNetDevicesFromPci(pr.CNIConf.DeviceID) if err != nil { return "", err diff --git a/go-controller/pkg/util/mocks/VdpaDevice.go b/go-controller/pkg/util/mocks/VdpaDevice.go new file mode 100644 index 00000000000..65f9a97bf08 --- /dev/null +++ b/go-controller/pkg/util/mocks/VdpaDevice.go @@ -0,0 +1,66 @@ +// Code generated by mockery v2.2.1. DO NOT EDIT. + +package mocks + +import mock "github.com/stretchr/testify/mock" + +// VdpaDevice is an autogenerated mock type for the VdpaDevice type +type VdpaDevice struct { + mock.Mock +} + +// GetDriver provides a mock function with given fields: +func (_m *VdpaDevice) GetDriver() string { + ret := _m.Called() + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// GetNetDev provides a mock function with given fields: +func (_m *VdpaDevice) GetNetDev() string { + ret := _m.Called() + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// GetParent provides a mock function with given fields: +func (_m *VdpaDevice) GetParent() string { + ret := _m.Called() + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// GetPath provides a mock function with given fields: +func (_m *VdpaDevice) GetPath() string { + ret := _m.Called() + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} diff --git a/go-controller/pkg/util/mocks/VdpaOps.go b/go-controller/pkg/util/mocks/VdpaOps.go new file mode 100644 index 00000000000..028c6bcce5b --- /dev/null +++ b/go-controller/pkg/util/mocks/VdpaOps.go @@ -0,0 +1,36 @@ +// Code generated by mockery v2.2.1. DO NOT EDIT. + +package mocks + +import ( + kvdpa "github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa" + mock "github.com/stretchr/testify/mock" +) + +// VdpaOps is an autogenerated mock type for the VdpaOps type +type VdpaOps struct { + mock.Mock +} + +// GetVdpaDeviceByPci provides a mock function with given fields: pciAddress +func (_m *VdpaOps) GetVdpaDeviceByPci(pciAddress string) (kvdpa.VdpaDevice, error) { + ret := _m.Called(pciAddress) + + var r0 kvdpa.VdpaDevice + if rf, ok := ret.Get(0).(func(string) kvdpa.VdpaDevice); ok { + r0 = rf(pciAddress) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(kvdpa.VdpaDevice) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(pciAddress) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/go-controller/pkg/util/vdpa_linux.go b/go-controller/pkg/util/vdpa_linux.go new file mode 100644 index 00000000000..5e32a8848c4 --- /dev/null +++ b/go-controller/pkg/util/vdpa_linux.go @@ -0,0 +1,32 @@ +package util + +import ( + "github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa" +) + +type VdpaDevice interface { + kvdpa.VdpaDevice +} + +type VdpaOps interface { + GetVdpaDeviceByPci(pciAddress string) (kvdpa.VdpaDevice, error) +} + +type defaultVdpaOps struct { +} + +var vdpaOps VdpaOps = &defaultVdpaOps{} + +// SetVdpaOpsInst method should be used by unit tests in +func SetVdpaOpsInst(mockInst VdpaOps) { + vdpaOps = mockInst +} + +// GetVdpaOps will be invoked by functions in other packages that would need access to the govdpa library methods. +func GetVdpaOps() VdpaOps { + return vdpaOps +} + +func (v *defaultVdpaOps) GetVdpaDeviceByPci(pciAddress string) (kvdpa.VdpaDevice, error) { + return kvdpa.GetVdpaDeviceByPci(pciAddress) +} diff --git a/go-controller/vendor/github.com/k8snetworkplumbingwg/govdpa/LICENSE b/go-controller/vendor/github.com/k8snetworkplumbingwg/govdpa/LICENSE new file mode 100644 index 00000000000..261eeb9e9f8 --- /dev/null +++ b/go-controller/vendor/github.com/k8snetworkplumbingwg/govdpa/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/go-controller/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/kvdpa.go b/go-controller/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/kvdpa.go new file mode 100644 index 00000000000..24471adc262 --- /dev/null +++ b/go-controller/vendor/github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa/kvdpa.go @@ -0,0 +1,242 @@ +package kvdpa + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +/*Exported constants */ +const ( + VhostVdpaDriver = "vhost_vdpa" + VirtioVdpaDriver = "virtio_vdpa" +) + +/*Private constants */ +const ( + vdpaBusDevDir = "/sys/bus/vdpa/devices" + pciBusDevDir = "/sys/bus/pci/devices" + vdpaVhostDevDir = "/dev" + virtioDevDir = "/sys/bus/virtio/devices" + rootDevDir = "/sys/devices" +) + +/*VdpaDevice contains information about a Vdpa Device*/ +type VdpaDevice interface { + GetDriver() string + GetParent() string + GetPath() string + GetNetDev() string +} + +/*vdpaDevimplements VdpaDevice interface */ +type vdpaDev struct { + name string + driver string + path string // Path of the vhost or virtio device + netdev string // VirtioNet netdev (only for virtio-vdpa devices) +} + +func (vd *vdpaDev) GetDriver() string { + return vd.driver +} + +func (vd *vdpaDev) GetParent() string { + return vd.name +} + +func (vd *vdpaDev) GetPath() string { + return vd.path +} + +func (vd *vdpaDev) GetNetDev() string { + return vd.netdev +} + +/*GetVdpaDeviceList returns a list of all available vdpa devices */ +func GetVdpaDeviceList() ([]VdpaDevice, error) { + vdpaDevList := make([]VdpaDevice, 0) + fd, err := os.Open(vdpaBusDevDir) + if err != nil { + return nil, err + } + defer fd.Close() + + fileInfos, err := fd.Readdir(-1) + if err != nil { + return nil, err + } + var errors []string + for _, file := range fileInfos { + if vdpaDev, err := GetVdpaDeviceByName(file.Name()); err != nil { + errors = append(errors, err.Error()) + } else { + vdpaDevList = append(vdpaDevList, vdpaDev) + } + } + + if len(errors) > 0 { + return vdpaDevList, fmt.Errorf(strings.Join(errors, ";")) + } + return vdpaDevList, nil +} + +/*GetVdpaDeviceByName returns the vdpa device information by a vdpa device name */ +func GetVdpaDeviceByName(name string) (VdpaDevice, error) { + var err error + var path string + var netdev string + + driverLink, err := os.Readlink(filepath.Join(vdpaBusDevDir, name, "driver")) + if err != nil { + return nil, err + } + + driver := filepath.Base(driverLink) + switch driver { + case VhostVdpaDriver: + path, err = getVhostVdpaDev(name) + if err != nil { + return nil, err + } + case VirtioVdpaDriver: + path, err = getVirtioVdpaDev(name) + if err != nil { + return nil, err + } + virtioNetDir := filepath.Join(path, "net") + netDeviceFiles, err := ioutil.ReadDir(virtioNetDir) + if err != nil || len(netDeviceFiles) != 1 { + return nil, fmt.Errorf("failed to get network device name from vdpa device in %v %v", name, err) + } + netdev = strings.TrimSpace(netDeviceFiles[0].Name()) + default: + return nil, fmt.Errorf("Unknown vdpa bus driver %s", driver) + } + + return &vdpaDev{ + name: name, + driver: driver, + path: path, + netdev: netdev, + }, nil +} + +/* Finds the vhost vdpa device of a vdpa device and returns it's path */ +func getVhostVdpaDev(name string) (string, error) { + file := filepath.Join(vdpaBusDevDir, name) + fd, err := os.Open(file) + if err != nil { + return "", err + } + defer fd.Close() + + fileInfos, err := fd.Readdir(-1) + for _, file := range fileInfos { + if strings.Contains(file.Name(), "vhost-vdpa") && + file.IsDir() { + devicePath := filepath.Join(vdpaVhostDevDir, file.Name()) + info, err := os.Stat(devicePath) + if err != nil { + return "", err + } + if info.Mode()&os.ModeDevice == 0 { + return "", fmt.Errorf("vhost device %s is not a valid device", devicePath) + } + return devicePath, nil + } + } + return "", fmt.Errorf("vhost device not found for vdpa device %s", name) +} + +/*GetVdpaDeviceByPci returns the vdpa device information corresponding to a PCI device*/ +/* Based on the following directory hiearchy: +/sys/bus/pci/devices/{PCIDev}/ + /vdpa{N}/ + +/sys/bus/vdpa/devices/vdpa{N} -> ../../../devices/pci.../{PCIDev}/vdpa{N} +*/ +func GetVdpaDeviceByPci(pciAddr string) (VdpaDevice, error) { + path, err := filepath.EvalSymlinks(filepath.Join(pciBusDevDir, pciAddr)) + if err != nil { + return nil, err + } + fd, err := os.Open(path) + if err != nil { + return nil, err + } + defer fd.Close() + + fileInfos, err := fd.Readdir(-1) + for _, file := range fileInfos { + if strings.Contains(file.Name(), "vdpa") { + parent, err := getParentDevice(filepath.Join(vdpaBusDevDir, file.Name())) + if err != nil { + return nil, err + } + if parent != path { + return nil, fmt.Errorf("vdpa device %s parent (%s) does not match containing dir (%s)", + file.Name(), parent, path) + } + return GetVdpaDeviceByName(file.Name()) + } + } + return nil, fmt.Errorf("PCI address %s does not contain a vdpa device", pciAddr) +} + +/* Finds the virtio vdpa device of a vdpa device and returns it's path +Currently, PCI-based devices have the following sysfs structure: +/sys/bus/vdpa/devices/ + vdpa1 -> ../../../devices/pci0000:00/0000:00:03.2/0000:05:00.2/vdpa1 + +In order to find the virtio device we look for virtio* devices inside the parent device: + sys/devices/pci0000:00/0000:00:03.2/0000:05:00.2/virtio{N} + +We also check the virtio device exists in the virtio bus: +/sys/bus/virtio/devices + virtio{N} -> ../../../devices/pci0000:00/0000:00:03.2/0000:05:00.2/virtio{N} +*/ +func getVirtioVdpaDev(name string) (string, error) { + vdpaDevicePath := filepath.Join(vdpaBusDevDir, name) + parentPath, err := getParentDevice(vdpaDevicePath) + if err != nil { + return "", err + } + + fd, err := os.Open(parentPath) + if err != nil { + return "", err + } + defer fd.Close() + + fileInfos, err := fd.Readdir(-1) + for _, file := range fileInfos { + if strings.Contains(file.Name(), "virtio") && + file.IsDir() { + virtioDevPath := filepath.Join(virtioDevDir, file.Name()) + if _, err := os.Stat(virtioDevPath); os.IsNotExist(err) { + return "", fmt.Errorf("virtio device %s does not exist", virtioDevPath) + } + return virtioDevPath, nil + } + } + + return "", fmt.Errorf("virtio device not found for vdpa device %s", name) +} + +/* getParentDevice returns the parent's path of a vdpa device path */ +func getParentDevice(path string) (string, error) { + devicePath, err := filepath.EvalSymlinks(path) + if err != nil { + return "", err + } + + parent := filepath.Dir(devicePath) + // if the "parent" is sys/devices, we have reached the "root" device + if parent == rootDevDir { + return devicePath, nil + } + return parent, nil +} diff --git a/go-controller/vendor/modules.txt b/go-controller/vendor/modules.txt index fad9685089f..39a3b35268a 100644 --- a/go-controller/vendor/modules.txt +++ b/go-controller/vendor/modules.txt @@ -108,6 +108,8 @@ github.com/imdario/mergo github.com/json-iterator/go # github.com/juju/errors v0.0.0-20200330140219-3fe23663418f github.com/juju/errors +# github.com/k8snetworkplumbingwg/govdpa v0.1.3 +github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa # github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200626054723-37f83d1996bc github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1