Fix leader election in deployment

deploy/cephfs/helm/templates/provisioner-deployment.yaml

@@ -30,6 +30,7 @@ spec:
           image: "{{ .Values.provisioner.image.repository }}:{{ .Values.provisioner.image.tag }}"
           args:
             - "--csi-address=$(ADDRESS)"
+            - "--enable-leader-election=true"
             - "--leader-election-type=leases"
             - "--v=5"
           env:

deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml

@@ -20,6 +20,7 @@ spec:
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=5"
+            - "--enable-leader-election=true"
             - "--leader-election-type=leases"
           env:
             - name: ADDRESS

deploy/rbd/helm/templates/provisioner-deployment.yaml

@@ -31,6 +31,7 @@ spec:
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=5"
+            - "--enable-leader-election=true"
             - "--leader-election-type=leases"
           env:
             - name: ADDRESS

deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml

@@ -20,7 +20,8 @@ spec:
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=5"
-            - --leader-election-type=leases"
+            - "--enable-leader-election=true"
+            - "--leader-election-type=leases"
           env:
             - name: ADDRESS
               value: unix:///csi/csi-provisioner.sock

deploy/rbd/kubernetes/csi-rbdplugin.yaml

@@ -20,7 +20,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
         - name: driver-registrar
-          image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2
+          image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0
           args:
             - "--v=5"
             - "--csi-address=/csi/csi.sock"

e2e/cephfs.go

@@ -1,6 +1,8 @@
 package e2e
 
 import (
+	"fmt"
+
 	. "github.com/onsi/ginkgo" // nolint
 
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -41,7 +43,9 @@ var _ = Describe("cephfs", func() {
 		cephfsFiles := getFilesinDirectory(cephfsDirPath)
 		for _, file := range cephfsFiles {
 			res, err := framework.RunKubectl("delete", "-f", cephfsDirPath+file.Name())
-			framework.Logf("failed to delete resource in %s with err %v", res, err)
+			if err != nil {
+				framework.Logf("failed to delete resource in %s with err %v", res, err)
+			}
 		}
 		deleteSecret(cephfsExamplePath + "secret.yaml")
 		deleteStorageClass(cephfsExamplePath + "storageclass.yaml")
@@ -50,6 +54,8 @@ var _ = Describe("cephfs", func() {
 
 	Context("Test cephfs CSI", func() {
 		It("Test cephfs CSI", func() {
+			pvcPath := cephfsExamplePath + "pvc.yaml"
+			appPath := cephfsExamplePath + "pod.yaml"
 			By("checking provisioner deployment is completed")
 			err := waitForDeploymentComplete(cephfsDeploymentName, namespace, f.ClientSet, deployTimeout)
 			if err != nil {
@@ -64,16 +70,42 @@ var _ = Describe("cephfs", func() {
 
 			By("create and delete a PVC", func() {
 				By("create a PVC and Bind it to an app", func() {
-					pvcPath := cephfsExamplePath + "pvc.yaml"
-					appPath := cephfsExamplePath + "pod.yaml"
+
 					validatePVCAndAppBinding(pvcPath, appPath, f)
 
 				})
 
 				By("create a PVC and Bind it to an app with normal user", func() {
-					pvcPath := cephfsExamplePath + "pvc.yaml"
 					validateNormalUserPVCAccess(pvcPath, f)
 				})
+
+				By("create/delete multiple PVC and App", func() {
+					totalCount := 2
+					pvc := loadPVC(pvcPath)
+					pvc.Namespace = f.UniqueName
+					app := loadApp(appPath)
+					app.Namespace = f.UniqueName
+					//create pvc and app
+					for i := 0; i < totalCount; i++ {
+						name := fmt.Sprintf("%s%d", f.UniqueName, i)
+						err := createPVCAndApp(name, f, pvc, app)
+						if err != nil {
+							Fail(err.Error())
+						}
+
+					}
+					//TODO add cephfs backend validation
+
+					//delete pvc and app
+					for i := 0; i < totalCount; i++ {
+						name := fmt.Sprintf("%s%d", f.UniqueName, i)
+						err := deletePVCAndApp(name, f, pvc, app)
+						if err != nil {
+							Fail(err.Error())
+						}
+
+					}
+				})
 			})
 		})
 	})

e2e/deploy-rook.go

@@ -49,12 +49,12 @@ func createRBDPool() {
 }
 func deleteFileSystem() {
 	commonPath := fmt.Sprintf("%s/%s", rookURL, "filesystem-test.yaml")
-	framework.RunKubectlOrDie("delete", "-f", commonPath)
+	framework.RunKubectl("delete", "-f", commonPath)
 }
 
 func deleteRBDPool() {
 	commonPath := fmt.Sprintf("%s/%s", rookURL, "pool-test.yaml")
-	framework.RunKubectlOrDie("delete", "-f", commonPath)
+	framework.RunKubectl("delete", "-f", commonPath)
 }
 
 func deployOperator(c kubernetes.Interface) {
@@ -109,5 +109,5 @@ func tearDownRook() {
 	// TODO need to add selector for cleanup validation
 	framework.Cleanup(opPath, rookNS)
 	commonPath := fmt.Sprintf("%s/%s", rookURL, "common.yaml")
-	framework.RunKubectlOrDie("delete", "-f", commonPath)
+	framework.RunKubectl("delete", "-f", commonPath)
 }

e2e/rbd.go

@@ -1,6 +1,8 @@
 package e2e
 
 import (
+	"fmt"
+
 	. "github.com/onsi/ginkgo" // nolint
 
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -43,7 +45,9 @@ var _ = Describe("RBD", func() {
 		rbdFiles := getFilesinDirectory(rbdDirPath)
 		for _, file := range rbdFiles {
 			res, err := framework.RunKubectl("delete", "-f", rbdDirPath+file.Name())
-			framework.Logf("failed to delete resource in %s with err %v", res, err)
+			if err != nil {
+				framework.Logf("failed to delete resource in %s with err %v", res, err)
+			}
 		}
 		deleteRBDPool()
 		deleteSecret(rbdExamplePath + "secret.yaml")
@@ -52,6 +56,9 @@ var _ = Describe("RBD", func() {
 
 	Context("Test RBD CSI", func() {
 		It("Test RBD CSI", func() {
+
+			pvcPath := rbdExamplePath + "pvc.yaml"
+			appPath := rbdExamplePath + "pod.yaml"
 			By("checking provisioner deployment is completed")
 			err := waitForDeploymentComplete(rbdDeploymentName, namespace, f.ClientSet, deployTimeout)
 			if err != nil {
@@ -65,15 +72,52 @@ var _ = Describe("RBD", func() {
 			}
 
 			By("create a PVC and Bind it to an app", func() {
-				pvcPath := rbdExamplePath + "pvc.yaml"
-				appPath := rbdExamplePath + "pod.yaml"
 				validatePVCAndAppBinding(pvcPath, appPath, f)
 			})
 
 			By("create a PVC and Bind it to an app with normal user", func() {
-				pvcPath := rbdExamplePath + "pvc.yaml"
 				validateNormalUserPVCAccess(pvcPath, f)
 			})
+
+			By("create/delete multiple PVC and App", func() {
+				totalCount := 2
+				pvc := loadPVC(pvcPath)
+				pvc.Namespace = f.UniqueName
+				app := loadApp(appPath)
+				app.Namespace = f.UniqueName
+				//create pvc and app
+				for i := 0; i < totalCount; i++ {
+					name := fmt.Sprintf("%s%d", f.UniqueName, i)
+					err := createPVCAndApp(name, f, pvc, app)
+					if err != nil {
+						Fail(err.Error())
+					}
+
+				}
+				// validate created backend rbd images
+				images := listRBDImages(f)
+				if len(images) != totalCount {
+					framework.Logf("backend image creation not matching pvc count, image count = % pvc count %d", len(images), totalCount)
+					Fail("validate multiple pvc failed")
+				}
+
+				//delete pvc and app
+				for i := 0; i < totalCount; i++ {
+					name := fmt.Sprintf("%s%d", f.UniqueName, i)
+					err := deletePVCAndApp(name, f, pvc, app)
+					if err != nil {
+						Fail(err.Error())
+					}
+
+				}
+
+				// validate created backend rbd images
+				images = listRBDImages(f)
+				if len(images) > 0 {
+					framework.Logf("left out rbd backend images count %d", len(images))
+					Fail("validate multiple pvc failed")
+				}
+			})
 		})
 	})
 

e2e/utils.go

@@ -470,28 +470,55 @@ func checkCephPods(ns string, c kubernetes.Interface, count, t int, opt *metav1.
 
 }
 
+// createPVCAndApp creates pvc and pod
+// if name is not empty same will be set as pvc and app name
+func createPVCAndApp(name string, f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *v1.Pod) error {
+
+	if name != "" {
+		pvc.Name = name
+		app.Name = name
+		app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = name
+	}
+	err := createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
+	if err != nil {
+		return err
+	}
+	err = createApp(f.ClientSet, app, deployTimeout)
+	return err
+}
+
+// deletePVCAndApp delete pvc and pod
+// if name is not empty same will be set as pvc and app name
+func deletePVCAndApp(name string, f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *v1.Pod) error {
+
+	if name != "" {
+		pvc.Name = name
+		app.Name = name
+		app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = name
+	}
+
+	err := deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
+	if err != nil {
+		return err
+	}
+	err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
+	return err
+}
+
 func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) {
 	pvc := loadPVC(pvcPath)
 	pvc.Namespace = f.UniqueName
 	framework.Logf("The PVC  template %+v", pvc)
-	err := createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
-	if err != nil {
-		Fail(err.Error())
-	}
 
 	app := loadApp(appPath)
 	app.Namespace = f.UniqueName
-	err = createApp(f.ClientSet, app, deployTimeout)
-	if err != nil {
-		Fail(err.Error())
-	}
 
-	err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
+	err := createPVCAndApp("", f, pvc, app)
 	if err != nil {
 		Fail(err.Error())
 	}
 
-	err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
+	err = deletePVCAndApp("", f, pvc, app)
 	if err != nil {
 		Fail(err.Error())
 	}
@@ -506,6 +533,7 @@ func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) {
 	if err != nil {
 		Fail(err.Error())
 	}
+
 	var user int64 = 2000
 	app := &v1.Pod{
 		TypeMeta: metav1.TypeMeta{
@@ -568,4 +596,21 @@ func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) {
 	if err != nil {
 		Fail(err.Error())
 	}
+
+}
+
+func listRBDImages(f *framework.Framework) []string {
+
+	opt := metav1.ListOptions{
+		LabelSelector: "app=rook-ceph-tools",
+	}
+	stdout := execCommandInPod(f, "rbd ls --pool=replicapool --format=json", rookNS, &opt)
+
+	var imgInfos []string
+
+	err := json.Unmarshal([]byte(stdout), &imgInfos)
+	if err != nil {
+		Fail(err.Error())
+	}
+	return imgInfos
 }