Skip to content

Commit

Permalink
CDI implementation
Browse files Browse the repository at this point in the history
This commit implements Container Device Interface [1] support.

[1] https://github.com/container-orchestrated-devices/container-device-interface
  • Loading branch information
e0ne committed Sep 8, 2023
1 parent 70efb55 commit 8c38780
Show file tree
Hide file tree
Showing 105 changed files with 10,870 additions and 207 deletions.
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ mockery: | $(BASE) $(GOMOCKERY) ; $(info Running mockery...) @ ## Run golint on
# $Q cd $(BASE)/pkg/types && rm -rf mocks && $(GOMOCKERY) --all 2>/dev/null
$Q $(GOMOCKERY) --name=".*" --dir=pkg/types --output=pkg/types/mocks --recursive=false --log-level=debug
$Q $(GOMOCKERY) --name=".*" --dir=pkg/utils --output=pkg/utils/mocks --recursive=false --log-level=debug
$Q $(GOMOCKERY) --name=".*" --dir=pkg/cdi --output=pkg/cdi/mocks --recursive=false --log-level=debug

.PHONY: help
help: ; @ ## Display this help message
Expand Down
9 changes: 6 additions & 3 deletions cmd/sriovdp/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,15 @@ const (
defaultConfig = "/etc/pcidp/config.json"
)

// Parse Command line flags
// flagInit parse command line flags
func flagInit(cp *cliParams) {
flag.StringVar(&cp.configFile, "config-file", defaultConfig,
"JSON device pool config file location")
flag.StringVar(&cp.resourcePrefix, "resource-prefix", "intel.com",
"resource name prefix used for K8s extended resource")
"resource name prefix used for K8s extended re"+
"source")
flag.BoolVar(&cp.useCdi, "use-cdi", false,
"Use Container Device Interface to expose devices in containers")
}

func main() {
Expand All @@ -43,7 +46,7 @@ func main() {

glog.Infof("resource manager reading configs")
if err := rm.readConfig(); err != nil {
glog.Errorf("error getting resources from file %v", err)
glog.Error("error getting resources from file", err)
return
}

Expand Down
7 changes: 5 additions & 2 deletions cmd/sriovdp/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,14 @@ const (
socketSuffix = "sock"
)

// cliParams presents CLI parameters for SR-IOV Network Device Plugin
type cliParams struct {
configFile string
resourcePrefix string
useCdi bool
}

// resourceManager manages resources for SR-IOV Network Device Plugin binaries
type resourceManager struct {
cliParams
pluginWatchMode bool
Expand All @@ -45,6 +48,7 @@ type resourceManager struct {
deviceProviders map[types.DeviceType]types.DeviceProvider
}

// newResourceManager initiates a new instance of resourceManager
func newResourceManager(cp *cliParams) *resourceManager {
pluginWatchMode := utils.DetectPluginWatchMode(types.SockDir)
if pluginWatchMode {
Expand All @@ -53,7 +57,7 @@ func newResourceManager(cp *cliParams) *resourceManager {
glog.Infof("Using Deprecated Device Plugin Registry Path")
}

rf := factory.NewResourceFactory(cp.resourcePrefix, socketSuffix, pluginWatchMode)
rf := factory.NewResourceFactory(cp.resourcePrefix, socketSuffix, pluginWatchMode, cp.useCdi)
dp := make(map[types.DeviceType]types.DeviceProvider)
for k := range types.SupportedDevices {
dp[k] = rf.GetDeviceProvider(k)
Expand Down Expand Up @@ -106,7 +110,6 @@ func (rm *resourceManager) initServers() error {
deviceAllocated := make(map[string]bool)
for _, rc := range rm.configList {
// Create new ResourcePool
glog.Infof("")
glog.Infof("Creating new ResourcePool: %s", rc.ResourceName)
glog.Infof("DeviceType: %+v", rc.DeviceType)
dp, ok := rm.deviceProviders[rc.DeviceType]
Expand Down
2 changes: 1 addition & 1 deletion cmd/sriovdp/manager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -441,7 +441,7 @@ var _ = Describe("Resource manager", func() {
_ = os.Unsetenv("GHW_CHROOT")
}()

rf := factory.NewResourceFactory("fake", "fake", true)
rf := factory.NewResourceFactory("fake", "fake", true, false)

rm := &resourceManager{
rFactory: rf,
Expand Down
13 changes: 13 additions & 0 deletions deployments/cdi/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# SR-IOV Network Device Plugin in CDI mode deployment

SR-IOV Network Device Plugin supports [Container Device Interface (CDI)](https://github.com/container-orchestrated-devices/container-device-interface).

To enable CDI mode, SR-IOV Network Device Plugin should be started with `--use-cdi` CLI argument.
This mode has different deployment requirements: `sriovdp-daemonset.yaml`

```yaml
- mountPath: /etc/cdi/
name: default-cdi
- mountPath: /var/run/cdi
name: dynamic-cdi
```
246 changes: 246 additions & 0 deletions deployments/cdi/sriovdp-daemonset.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,246 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: sriov-device-plugin
namespace: kube-system

---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-sriov-device-plugin-amd64
namespace: kube-system
labels:
tier: node
app: sriovdp
spec:
selector:
matchLabels:
name: sriov-device-plugin
template:
metadata:
labels:
name: sriov-device-plugin
tier: node
app: sriovdp
spec:
hostNetwork: true
nodeSelector:
kubernetes.io/arch: amd64
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
serviceAccountName: sriov-device-plugin
containers:
- name: kube-sriovdp
image: ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin:latest-amd64
imagePullPolicy: IfNotPresent
args:
- --log-dir=sriovdp
- --log-level=10
- --use-cdi
securityContext:
privileged: true
resources:
requests:
cpu: "250m"
memory: "40Mi"
limits:
cpu: 1
memory: "200Mi"
volumeMounts:
- name: devicesock
mountPath: /var/lib/kubelet/
readOnly: false
- name: log
mountPath: /var/log
- name: config-volume
mountPath: /etc/pcidp
- name: device-info
mountPath: /var/run/k8s.cni.cncf.io/devinfo/dp
- name: default-cdi
mountPath: /etc/cdi/
- name: dynamic-cdi
mountPath: /var/run/cdi
volumes:
- name: devicesock
hostPath:
path: /var/lib/kubelet/
- name: log
hostPath:
path: /var/log
- name: device-info
hostPath:
path: /var/run/k8s.cni.cncf.io/devinfo/dp
type: DirectoryOrCreate
- name: config-volume
configMap:
name: sriovdp-config
items:
- key: config.json
path: config.json
- name: default-cdi
hostPath:
path: /etc/cdi
type: DirectoryOrCreate
- name: dynamic-cdi
hostPath:
path: /var/run/cdi
type: DirectoryOrCreate


---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-sriov-device-plugin-ppc64le
namespace: kube-system
labels:
tier: node
app: sriovdp
spec:
selector:
matchLabels:
name: sriov-device-plugin
template:
metadata:
labels:
name: sriov-device-plugin
tier: node
app: sriovdp
spec:
hostNetwork: true
nodeSelector:
kubernetes.io/arch: ppc64le
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
serviceAccountName: sriov-device-plugin
containers:
- name: kube-sriovdp
image: ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin:latest-ppc64le
imagePullPolicy: IfNotPresent
args:
- --log-dir=sriovdp
- --log-level=10
- --use-cdi
securityContext:
privileged: true
resources:
requests:
cpu: "250m"
memory: "40Mi"
limits:
cpu: 1
memory: "200Mi"
volumeMounts:
- name: devicesock
mountPath: /var/lib/kubelet/
readOnly: false
- name: log
mountPath: /var/log
- name: config-volume
mountPath: /etc/pcidp
- name: device-info
mountPath: /var/run/k8s.cni.cncf.io/devinfo/dp
volumes:
- name: devicesock
hostPath:
path: /var/lib/kubelet/
- name: log
hostPath:
path: /var/log
- name: device-info
hostPath:
path: /var/run/k8s.cni.cncf.io/devinfo/dp
type: DirectoryOrCreate
- name: config-volume
configMap:
name: sriovdp-config
items:
- key: config.json
path: config.json
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-sriov-device-plugin-arm64
namespace: kube-system
labels:
tier: node
app: sriovdp
spec:
selector:
matchLabels:
name: sriov-device-plugin
template:
metadata:
labels:
name: sriov-device-plugin
tier: node
app: sriovdp
spec:
hostNetwork: true
nodeSelector:
kubernetes.io/arch: arm64
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
serviceAccountName: sriov-device-plugin
containers:
- name: kube-sriovdp
image: ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin:latest-arm64
imagePullPolicy: IfNotPresent
args:
- --log-dir=sriovdp
- --log-level=10
- --use-cdi
securityContext:
privileged: true
resources:
requests:
cpu: "250m"
memory: "40Mi"
limits:
cpu: 1
memory: "200Mi"
volumeMounts:
- name: devicesock
mountPath: /var/lib/kubelet/
readOnly: false
- name: log
mountPath: /var/log
- name: config-volume
mountPath: /etc/pcidp
- name: device-info
mountPath: /var/run/k8s.cni.cncf.io/devinfo/dp
volumes:
- name: devicesock
hostPath:
path: /var/lib/kubelet/
- name: log
hostPath:
path: /var/log
- name: device-info
hostPath:
path: /var/run/k8s.cni.cncf.io/devinfo/dp
type: DirectoryOrCreate
- name: config-volume
configMap:
name: sriovdp-config
items:
- key: config.json
path: config.json
6 changes: 6 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.20

require (
github.com/Mellanox/rdmamap v1.1.0
github.com/container-orchestrated-devices/container-device-interface v0.5.4
github.com/golang/glog v1.1.0
github.com/jaypipes/ghw v0.9.0
github.com/jaypipes/pcidb v1.0.0
Expand Down Expand Up @@ -45,10 +46,15 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/opencontainers/runc v1.1.2 // indirect
github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb // indirect
github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/spf13/afero v1.9.4 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
github.com/vishvananda/netns v0.0.4 // indirect
golang.org/x/mod v0.8.0 // indirect
golang.org/x/net v0.13.0 // indirect
golang.org/x/oauth2 v0.8.0 // indirect
golang.org/x/sys v0.10.0 // indirect
Expand Down
Loading

0 comments on commit 8c38780

Please sign in to comment.