From 2a580811f58bca17a6a6120bff3ac10f5f2167b9 Mon Sep 17 00:00:00 2001 From: Amarnath Valluri Date: Fri, 24 Jul 2020 17:52:28 +0300 Subject: [PATCH] operator: API to configure kubelet directory path Not all clusters use /var/lib/kubelet as state directory for kubelet (kubernetes-csi/csi-driver-host-path#71). There is no way for the operator to auto-detect this path. Hence, added a new field to deployment spec that could be used to provide the kubelet path to driver deployment. FIXES: #668 --- docs/install.md | 1 + pkg/apis/pmemcsi/v1alpha1/deployment_types.go | 20 ++++++++++++++++++- pkg/deployments/load.go | 5 +++++ .../deployment/controller_driver.go | 12 ++++++----- .../deployment/deployment_controller_test.go | 5 +++++ .../deployment/testcases/testcases.go | 3 +++ 6 files changed, 40 insertions(+), 6 deletions(-) diff --git a/docs/install.md b/docs/install.md index 8846c3460d..d2dfbf9868 100644 --- a/docs/install.md +++ b/docs/install.md @@ -844,6 +844,7 @@ The current API for PMEM-CSI `Deployment` resources is: | nodeSelector | string map | [Labels to use for selecting Nodes](../docs/install.md#run-pmem-csi-on-kubernetes) on which PMEM-CSI driver should run. | `{ "storage": "pmem" }`| | pmemPercentage | integer | Percentage of PMEM space to be used by the driver on each node. This is only valid for a driver deployed in `lvm` mode. This field can be modified, but by that time the old value may have been used already. Reducing the percentage is not supported. | 100 | | labels | string map | Additional labels for all objects created by the operator. Can be modified after the initial creation, but removed labels will not be removed from existing objects because the operator cannot know which labels it needs to remove and which it has to leave in place. | +| kubeletDir | string | Kubelet's root directory path | /var/lib/kubelet | 1 To use the same container image as default driver image the operator pod must set with below environment variables with diff --git a/pkg/apis/pmemcsi/v1alpha1/deployment_types.go b/pkg/apis/pmemcsi/v1alpha1/deployment_types.go index 864f64230e..664b2901c4 100644 --- a/pkg/apis/pmemcsi/v1alpha1/deployment_types.go +++ b/pkg/apis/pmemcsi/v1alpha1/deployment_types.go @@ -91,6 +91,8 @@ type DeploymentSpec struct { PMEMPercentage uint16 `json:"pmemPercentage,omitempty"` // Labels contains additional labels for all objects created by the operator. Labels map[string]string `json:"labels,omitempty"` + // KubeletDir kubelet's root directory path + KubeletDir string `json:"kubeletDir,omitempty"` } // DeploymentStatus defines the observed state of Deployment @@ -163,6 +165,8 @@ const ( DefaultDeviceMode = DeviceModeLVM // DefaultPMEMPercentage PMEM space to reserve for the driver DefaultPMEMPercentage = 100 + // DefaultKubeletDir default kubelet's path + DefaultKubeletDir = "/var/lib/kubelet" ) var ( @@ -204,6 +208,7 @@ const ( RegistryKey NodeControllerCertificate NodeControllerKey + KubeletDir ) func (c DeploymentChange) String() string { @@ -224,6 +229,7 @@ func (c DeploymentChange) String() string { RegistryKey: "registryKey", NodeControllerCertificate: "nodeControllerCert", NodeControllerKey: "nodeControllerKey", + KubeletDir: "kubeletDir", }[c] } @@ -293,6 +299,10 @@ func (d *Deployment) EnsureDefaults(operatorImage string) error { d.Spec.PMEMPercentage = DefaultPMEMPercentage } + if d.Spec.KubeletDir == "" { + d.Spec.KubeletDir = DefaultKubeletDir + } + return nil } @@ -357,6 +367,9 @@ func (d *Deployment) Compare(other *Deployment) map[DeploymentChange]struct{} { if bytes.Compare(d.Spec.NodeControllerPrivateKey, other.Spec.NodeControllerPrivateKey) != 0 { changes[NodeControllerKey] = struct{}{} } + if d.Spec.KubeletDir != other.Spec.KubeletDir { + changes[KubeletDir] = struct{}{} + } return changes } @@ -460,7 +473,12 @@ func GetDeploymentCRDSchema() *apiextensions.JSONSchemaProps { Type: "string", }, }, - }}, + }, + "kubeletDir": apiextensions.JSONSchemaProps{ + Type: "string", + Description: "Kubelet root directory path", + }, + }, }, "status": apiextensions.JSONSchemaProps{ Type: "object", diff --git a/pkg/deployments/load.go b/pkg/deployments/load.go index 76d806eec6..0f3f16dbd1 100644 --- a/pkg/deployments/load.go +++ b/pkg/deployments/load.go @@ -47,6 +47,11 @@ func LoadAndCustomizeObjects(kubernetes version.Version, deviceMode api.DeviceMo *yaml = bytes.ReplaceAll(*yaml, []byte("path: /var/lib/pmem-csi.intel.com"), []byte("path: /var/lib/"+deployment.Name)) *yaml = bytes.ReplaceAll(*yaml, []byte("mountPath: /var/lib/pmem-csi.intel.com"), []byte("mountPath: /var/lib/"+deployment.Name)) + // Update kubelet path + if deployment.Spec.KubeletDir != api.DefaultKubeletDir { + *yaml = bytes.ReplaceAll(*yaml, []byte("/var/lib/kubelet"), []byte(deployment.Spec.KubeletDir)) + } + // This assumes that all namespaced objects actually have "namespace: default". *yaml = bytes.ReplaceAll(*yaml, []byte("namespace: default"), []byte("namespace: "+namespace)) diff --git a/pkg/pmem-csi-operator/controller/deployment/controller_driver.go b/pkg/pmem-csi-operator/controller/deployment/controller_driver.go index e67203925b..50197a55d0 100644 --- a/pkg/pmem-csi-operator/controller/deployment/controller_driver.go +++ b/pkg/pmem-csi-operator/controller/deployment/controller_driver.go @@ -105,6 +105,8 @@ func (d *PmemCSIDriver) reconcileDeploymentChanges(r *ReconcileDeployment, chang updateAll = true case api.CACertificate, api.RegistryCertificate, api.NodeControllerCertificate: updateSecrets = true + case api.KubeletDir: + updateNodeDriver = true } if err != nil { @@ -717,7 +719,7 @@ func (d *PmemCSIDriver) getNodeDaemonSet() *appsv1.DaemonSet { Name: "registration-dir", VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/lib/kubelet/plugins_registry/", + Path: d.Spec.KubeletDir + "/plugins_registry/", Type: &directoryOrCreate, }, }, @@ -726,7 +728,7 @@ func (d *PmemCSIDriver) getNodeDaemonSet() *appsv1.DaemonSet { Name: "mountpoint-dir", VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/lib/kubelet/plugins/kubernetes.io/csi", + Path: d.Spec.KubeletDir + "/plugins/kubernetes.io/csi", Type: &directoryOrCreate, }, }, @@ -735,7 +737,7 @@ func (d *PmemCSIDriver) getNodeDaemonSet() *appsv1.DaemonSet { Name: "pods-dir", VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/lib/kubelet/pods", + Path: d.Spec.KubeletDir + "/pods", Type: &directoryOrCreate, }, }, @@ -908,12 +910,12 @@ func (d *PmemCSIDriver) getNodeDriverContainer() corev1.Container { VolumeMounts: []corev1.VolumeMount{ { Name: "mountpoint-dir", - MountPath: "/var/lib/kubelet/plugins/kubernetes.io/csi", + MountPath: d.Spec.KubeletDir + "/plugins/kubernetes.io/csi", MountPropagation: &bidirectional, }, { Name: "pods-dir", - MountPath: "/var/lib/kubelet/pods", + MountPath: d.Spec.KubeletDir + "/pods", MountPropagation: &bidirectional, }, { diff --git a/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go b/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go index aab88a33a0..59f811c6a0 100644 --- a/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go +++ b/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go @@ -45,6 +45,7 @@ type pmemDeployment struct { controllerCPU, controllerMemory string nodeCPU, nodeMemory string caCert, regCert, regKey, ncCert, ncKey []byte + kubeletDir string } func getDeployment(d *pmemDeployment) *api.Deployment { @@ -91,6 +92,9 @@ func getDeployment(d *pmemDeployment) *api.Deployment { spec.RegistryPrivateKey = d.regKey spec.NodeControllerCert = d.ncCert spec.NodeControllerPrivateKey = d.ncKey + if d.kubeletDir != "" { + spec.KubeletDir = d.kubeletDir + } return dep } @@ -236,6 +240,7 @@ func TestDeploymentController(t *testing.T) { controllerMemory: "300Mi", nodeCPU: "1000m", nodeMemory: "500Mi", + kubeletDir: "/some/directory", } dep := getDeployment(d) diff --git a/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go b/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go index ce946c58f7..7831daf9ca 100644 --- a/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go +++ b/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go @@ -77,6 +77,9 @@ func UpdateTests() []UpdateTest { } d.Spec.Labels["foo"] = "bar" }, + "kubeletDir": func(d *api.Deployment) { + d.Spec.KubeletDir = "/foo/bar" + }, } full := api.Deployment{