From 349a48c2070fc67c7e8a78350913f862eb115e5c Mon Sep 17 00:00:00 2001 From: "eoghanrussell1212@gmail.com" Date: Mon, 27 Sep 2021 12:01:45 -0400 Subject: [PATCH] Implementation of exclusing pooling, with unit tests. --- cmd/sriovdp/manager.go | 3 +- cmd/sriovdp/manager_test.go | 3 + pkg/factory/factory.go | 25 ++++-- pkg/factory/factory_test.go | 110 ++++++++++++++++++++++++++ pkg/types/mocks/AccelDevice.go | 2 +- pkg/types/mocks/DeviceInfoProvider.go | 2 +- pkg/types/mocks/DeviceProvider.go | 2 +- pkg/types/mocks/DeviceSelector.go | 5 +- pkg/types/mocks/LinkWatcher.go | 2 +- pkg/types/mocks/NadUtils.go | 2 +- pkg/types/mocks/PciDevice.go | 2 +- pkg/types/mocks/PciNetDevice.go | 5 +- pkg/types/mocks/RdmaSpec.go | 2 +- pkg/types/mocks/ResourceFactory.go | 28 ++++++- pkg/types/mocks/ResourcePool.go | 2 +- pkg/types/mocks/ResourceServer.go | 2 +- pkg/types/types.go | 1 + 17 files changed, 172 insertions(+), 26 deletions(-) diff --git a/cmd/sriovdp/manager.go b/cmd/sriovdp/manager.go index 455f2c466..14303b97c 100644 --- a/cmd/sriovdp/manager.go +++ b/cmd/sriovdp/manager.go @@ -103,6 +103,7 @@ func (rm *resourceManager) readConfig() error { func (rm *resourceManager) initServers() error { rf := rm.rFactory glog.Infof("number of config: %d\n", len(rm.configList)) + deviceAllocated := make(map[string]bool) for _, rc := range rm.configList { // Create new ResourcePool glog.Infof("") @@ -123,7 +124,7 @@ func (rm *resourceManager) initServers() error { glog.Infof("no devices in device pool, skipping creating resource server for %s", rc.ResourceName) continue } - rPool, err := rm.rFactory.GetResourcePool(rc, filteredDevices) + rPool, err := rm.rFactory.GetResourcePoolExclusive(rc, filteredDevices, deviceAllocated) if err != nil { glog.Errorf("initServers(): error creating ResourcePool with config %+v: %q", rc, err) return err diff --git a/cmd/sriovdp/manager_test.go b/cmd/sriovdp/manager_test.go index bbdc73846..03ba38381 100644 --- a/cmd/sriovdp/manager_test.go +++ b/cmd/sriovdp/manager_test.go @@ -255,6 +255,9 @@ var _ = Describe("Resource manager", func() { mockedRf := &mocks.ResourceFactory{} mockedRf.On("GetResourcePool", rc, devs).Return(rp, nil). On("GetResourceServer", rp).Return(mockedServer, nil) + deviceAllocated := make(map[string]bool) + mockedRf.On("GetResourcePoolExclusive", rc, devs, deviceAllocated).Return(rp, nil). + On("GetResourceServer", rp).Return(mockedServer, nil) dp.On("GetDevices", rc).Return(devs) rm := &resourceManager{ diff --git a/pkg/factory/factory.go b/pkg/factory/factory.go index 05b3e15ab..5c6d55aa6 100644 --- a/pkg/factory/factory.go +++ b/pkg/factory/factory.go @@ -97,17 +97,28 @@ func (rf *resourceFactory) GetSelector(attr string, values []string) (types.Devi // GetResourcePool returns an instance of resourcePool func (rf *resourceFactory) GetResourcePool(rc *types.ResourceConfig, filteredDevice []types.PciDevice) (types.ResourcePool, error) { + return rf.GetResourcePoolExclusive(rc, filteredDevice, make(map[string]bool)) +} + +// GetResourcePoolExclusive returns an exclusive resourcePool +func (rf *resourceFactory) GetResourcePoolExclusive(rc *types.ResourceConfig, filteredDevice []types.PciDevice, + deviceAllocated map[string]bool) (types.ResourcePool, error) { devicePool := make(map[string]types.PciDevice) apiDevices := make(map[string]*pluginapi.Device) for _, dev := range filteredDevice { pciAddr := dev.GetPciAddr() - devicePool[pciAddr] = dev - apiDevices[pciAddr] = dev.GetAPIDevice() - glog.Infof("device added: [pciAddr: %s, vendor: %s, device: %s, driver: %s]", - dev.GetPciAddr(), - dev.GetVendor(), - dev.GetDeviceCode(), - dev.GetDriver()) + if !deviceAllocated[pciAddr] { + deviceAllocated[pciAddr] = true + devicePool[pciAddr] = dev + apiDevices[pciAddr] = dev.GetAPIDevice() + glog.Infof("device added: [pciAddr: %s, vendor: %s, device: %s, driver: %s]", + dev.GetPciAddr(), + dev.GetVendor(), + dev.GetDeviceCode(), + dev.GetDriver()) + } else { + glog.Infof("Cannot add PCI Address [%s]. Already allocated.", pciAddr) + } } var rPool types.ResourcePool diff --git a/pkg/factory/factory_test.go b/pkg/factory/factory_test.go index abc38f91a..848f6b2e7 100644 --- a/pkg/factory/factory_test.go +++ b/pkg/factory/factory_test.go @@ -160,6 +160,116 @@ var _ = Describe("Factory", func() { }) }) }) + Describe("getting exclusive resource pool for netdevice", func() { + Context("with all types of selectors used and matching devices found", func() { + defer utils.UseFakeLinks()() + var ( + rp types.ResourcePool + rp2 types.ResourcePool + err error + devs []types.PciDevice + ) + BeforeEach(func() { + f := factory.NewResourceFactory("fake", "fake", true) + deviceAllocated := make(map[string]bool) + devs = make([]types.PciDevice, 4) + vendors := []string{"8086", "8086", "8086", "8086"} + codes := []string{"1111", "1111", "1111", "1111"} + drivers := []string{"iavf", "iavf", "vfio-pci", "vfio-pci"} + pciAddr := []string{"0000:03:02.0", "0000:03:02.0", "0000:03:02.0", "0000:03:02.0"} + pfNames := []string{"enp2s0f2", "ens0", "eth0", "net2"} + rootDevices := []string{"0000:86:00.0", "0000:86:00.1", "0000:86:00.2", "0000:86:00.3"} + linkTypes := []string{"ether", "infiniband", "other", "other2"} + ddpProfiles := []string{"GTP", "PPPoE", "GTP", "PPPoE"} + for i := range devs { + d := &mocks.PciNetDevice{} + d.On("GetVendor").Return(vendors[i]). + On("GetDeviceCode").Return(codes[i]). + On("GetDriver").Return(drivers[i]). + On("GetPciAddr").Return(pciAddr[i]). + On("GetPFName").Return(pfNames[i]). + On("GetPfPciAddr").Return(rootDevices[i]). + On("GetAPIDevice").Return(&pluginapi.Device{}). + On("GetLinkType").Return(linkTypes[i]). + On("GetDDPProfiles").Return(ddpProfiles[i]) + devs[i] = d + } + + var selectors json.RawMessage + err = selectors.UnmarshalJSON([]byte(` + { + "vendors": ["8086"], + "devices": ["1111"], + "drivers": ["iavf","vfio-pci"], + "pciAddresses": ["0000:03:02.0"], + "pfNames": ["enp2s0f2"], + "rootDevices": ["0000:86:00.0"], + "linkTypes": ["ether"], + "ddpProfiles": ["GTP"] + } + `), + ) + Expect(err).NotTo(HaveOccurred()) + + var selectors2 json.RawMessage + err = selectors2.UnmarshalJSON([]byte(` + { + "vendors": ["8086"], + "devices": ["1111"], + "drivers": ["iavf","vfio-pci"], + "pciAddresses": ["0000:03:02.0"], + "pfNames": ["enp2s0f2"], + "rootDevices": ["0000:86:00.0"], + "linkTypes": ["ether"], + "ddpProfiles": ["GTP"] + } + `), + ) + Expect(err).NotTo(HaveOccurred()) + + c := &types.ResourceConfig{ + ResourceName: "fake", + Selectors: &selectors, + DeviceType: types.NetDeviceType, + } + + dp := f.GetDeviceProvider(c.DeviceType) + c.SelectorObj, err = f.GetDeviceFilter(c) + Expect(err).NotTo(HaveOccurred()) + filteredDevices, err := dp.GetFilteredDevices(devs, c) + Expect(err).NotTo(HaveOccurred()) + rp, err = f.GetResourcePoolExclusive(c, filteredDevices, deviceAllocated) + Expect(err).NotTo(HaveOccurred()) + + // Second config definition + c2 := &types.ResourceConfig{ + ResourceName: "fake", + Selectors: &selectors2, + DeviceType: types.NetDeviceType, + } + + dp2 := f.GetDeviceProvider(c2.DeviceType) + c2.SelectorObj, err = f.GetDeviceFilter(c2) + Expect(err).NotTo(HaveOccurred()) + filteredDevices, err = dp2.GetFilteredDevices(devs, c2) + Expect(err).NotTo(HaveOccurred()) + rp2, err = f.GetResourcePoolExclusive(c2, filteredDevices, deviceAllocated) + Expect(err).NotTo(HaveOccurred()) + + }) + It("should return valid exclusive resource pool", func() { + Expect(rp).NotTo(BeNil()) + Expect(rp.GetDevices()).To(HaveLen(1)) + Expect(rp.GetDevices()).To(HaveKey("0000:03:02.0")) + // Check second resource pool to make sure nothing got added to it. + Expect(rp2).NotTo(BeNil()) + Expect(rp2.GetDevices()).To(HaveLen(0)) + }) + It("should not fail", func() { + Expect(err).NotTo(HaveOccurred()) + }) + }) + }) Describe("getting resource pool for accelerator", func() { Context("with all types of selectors used and matching devices found", func() { defer utils.UseFakeLinks()() diff --git a/pkg/types/mocks/AccelDevice.go b/pkg/types/mocks/AccelDevice.go index c755e5074..e8f0ec5d4 100644 --- a/pkg/types/mocks/AccelDevice.go +++ b/pkg/types/mocks/AccelDevice.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.2.1. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package mocks diff --git a/pkg/types/mocks/DeviceInfoProvider.go b/pkg/types/mocks/DeviceInfoProvider.go index 2a901bcc2..70f56c824 100644 --- a/pkg/types/mocks/DeviceInfoProvider.go +++ b/pkg/types/mocks/DeviceInfoProvider.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.2.1. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package mocks diff --git a/pkg/types/mocks/DeviceProvider.go b/pkg/types/mocks/DeviceProvider.go index 97114aa46..e96ffd63e 100644 --- a/pkg/types/mocks/DeviceProvider.go +++ b/pkg/types/mocks/DeviceProvider.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.2.1. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package mocks diff --git a/pkg/types/mocks/DeviceSelector.go b/pkg/types/mocks/DeviceSelector.go index 2534a0cda..8d1b8efaf 100644 --- a/pkg/types/mocks/DeviceSelector.go +++ b/pkg/types/mocks/DeviceSelector.go @@ -1,11 +1,10 @@ -// Code generated by mockery v2.2.1. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package mocks import ( - mock "github.com/stretchr/testify/mock" - types "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/types" + mock "github.com/stretchr/testify/mock" ) // DeviceSelector is an autogenerated mock type for the DeviceSelector type diff --git a/pkg/types/mocks/LinkWatcher.go b/pkg/types/mocks/LinkWatcher.go index 548b938aa..fd290a4fc 100644 --- a/pkg/types/mocks/LinkWatcher.go +++ b/pkg/types/mocks/LinkWatcher.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.2.1. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package mocks diff --git a/pkg/types/mocks/NadUtils.go b/pkg/types/mocks/NadUtils.go index 41577fcf9..b09d5ff4d 100644 --- a/pkg/types/mocks/NadUtils.go +++ b/pkg/types/mocks/NadUtils.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.2.1. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package mocks diff --git a/pkg/types/mocks/PciDevice.go b/pkg/types/mocks/PciDevice.go index 4dead2439..f8f0c4070 100644 --- a/pkg/types/mocks/PciDevice.go +++ b/pkg/types/mocks/PciDevice.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.2.1. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package mocks diff --git a/pkg/types/mocks/PciNetDevice.go b/pkg/types/mocks/PciNetDevice.go index bee1cf41e..b669b4c42 100644 --- a/pkg/types/mocks/PciNetDevice.go +++ b/pkg/types/mocks/PciNetDevice.go @@ -1,12 +1,11 @@ -// Code generated by mockery v2.2.1. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package mocks import ( + types "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/types" mock "github.com/stretchr/testify/mock" v1beta1 "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" - - types "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/types" ) // PciNetDevice is an autogenerated mock type for the PciNetDevice type diff --git a/pkg/types/mocks/RdmaSpec.go b/pkg/types/mocks/RdmaSpec.go index 95ed3dba3..ed470bfb4 100644 --- a/pkg/types/mocks/RdmaSpec.go +++ b/pkg/types/mocks/RdmaSpec.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.2.1. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package mocks diff --git a/pkg/types/mocks/ResourceFactory.go b/pkg/types/mocks/ResourceFactory.go index 141f8065b..cef308eec 100644 --- a/pkg/types/mocks/ResourceFactory.go +++ b/pkg/types/mocks/ResourceFactory.go @@ -1,11 +1,10 @@ -// Code generated by mockery v2.2.1. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package mocks import ( - mock "github.com/stretchr/testify/mock" - types "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/types" + mock "github.com/stretchr/testify/mock" ) // ResourceFactory is an autogenerated mock type for the ResourceFactory type @@ -123,6 +122,29 @@ func (_m *ResourceFactory) GetResourcePool(rc *types.ResourceConfig, deviceList return r0, r1 } +// GetResourcePoolExclusive provides a mock function with given fields: rc, deviceList, deviceAllocated +func (_m *ResourceFactory) GetResourcePoolExclusive(rc *types.ResourceConfig, deviceList []types.PciDevice, deviceAllocated map[string]bool) (types.ResourcePool, error) { + ret := _m.Called(rc, deviceList, deviceAllocated) + + var r0 types.ResourcePool + if rf, ok := ret.Get(0).(func(*types.ResourceConfig, []types.PciDevice, map[string]bool) types.ResourcePool); ok { + r0 = rf(rc, deviceList, deviceAllocated) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(types.ResourcePool) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*types.ResourceConfig, []types.PciDevice, map[string]bool) error); ok { + r1 = rf(rc, deviceList, deviceAllocated) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetResourceServer provides a mock function with given fields: _a0 func (_m *ResourceFactory) GetResourceServer(_a0 types.ResourcePool) (types.ResourceServer, error) { ret := _m.Called(_a0) diff --git a/pkg/types/mocks/ResourcePool.go b/pkg/types/mocks/ResourcePool.go index 47721f1a5..ad08fd784 100644 --- a/pkg/types/mocks/ResourcePool.go +++ b/pkg/types/mocks/ResourcePool.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.2.1. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package mocks diff --git a/pkg/types/mocks/ResourceServer.go b/pkg/types/mocks/ResourceServer.go index c0d1200ad..09b077b28 100644 --- a/pkg/types/mocks/ResourceServer.go +++ b/pkg/types/mocks/ResourceServer.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.2.1. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package mocks diff --git a/pkg/types/types.go b/pkg/types/types.go index 8d7d051ef..f6a4a96f0 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -129,6 +129,7 @@ type ResourceFactory interface { GetDefaultInfoProvider(string, string) DeviceInfoProvider GetSelector(string, []string) (DeviceSelector, error) GetResourcePool(rc *ResourceConfig, deviceList []PciDevice) (ResourcePool, error) + GetResourcePoolExclusive(rc *ResourceConfig, deviceList []PciDevice, deviceAllocated map[string]bool) (ResourcePool, error) GetRdmaSpec(string) RdmaSpec GetDeviceProvider(DeviceType) DeviceProvider GetDeviceFilter(*ResourceConfig) (interface{}, error)