diff --git a/deploy/cephfs/helm/templates/provisioner-deployment.yaml b/deploy/cephfs/helm/templates/provisioner-deployment.yaml index 1a933232ac7..d9cd1cd2055 100644 --- a/deploy/cephfs/helm/templates/provisioner-deployment.yaml +++ b/deploy/cephfs/helm/templates/provisioner-deployment.yaml @@ -30,6 +30,7 @@ spec: image: "{{ .Values.provisioner.image.repository }}:{{ .Values.provisioner.image.tag }}" args: - "--csi-address=$(ADDRESS)" + - "--enable-leader-election=true" - "--leader-election-type=leases" - "--v=5" env: diff --git a/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml b/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml index eedae2bc076..723650bd549 100644 --- a/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml +++ b/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml @@ -20,6 +20,7 @@ spec: args: - "--csi-address=$(ADDRESS)" - "--v=5" + - "--enable-leader-election=true" - "--leader-election-type=leases" env: - name: ADDRESS diff --git a/deploy/rbd/helm/templates/provisioner-deployment.yaml b/deploy/rbd/helm/templates/provisioner-deployment.yaml index e9034f13185..d57b55576fe 100644 --- a/deploy/rbd/helm/templates/provisioner-deployment.yaml +++ b/deploy/rbd/helm/templates/provisioner-deployment.yaml @@ -31,6 +31,7 @@ spec: args: - "--csi-address=$(ADDRESS)" - "--v=5" + - "--enable-leader-election=true" - "--leader-election-type=leases" env: - name: ADDRESS diff --git a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml b/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml index 9470f1b7219..1102071d39e 100644 --- a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml +++ b/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml @@ -20,7 +20,8 @@ spec: args: - "--csi-address=$(ADDRESS)" - "--v=5" - - --leader-election-type=leases" + - "--enable-leader-election=true" + - "--leader-election-type=leases" env: - name: ADDRESS value: unix:///csi/csi-provisioner.sock diff --git a/deploy/rbd/kubernetes/csi-rbdplugin.yaml b/deploy/rbd/kubernetes/csi-rbdplugin.yaml index 07431e7ad0c..1ec8ab1786c 100644 --- a/deploy/rbd/kubernetes/csi-rbdplugin.yaml +++ b/deploy/rbd/kubernetes/csi-rbdplugin.yaml @@ -20,7 +20,7 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: driver-registrar - image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2 + image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0 args: - "--v=5" - "--csi-address=/csi/csi.sock" diff --git a/e2e/cephfs.go b/e2e/cephfs.go index 22e9e17a865..28b1c1875ab 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -1,6 +1,8 @@ package e2e import ( + "fmt" + . "github.com/onsi/ginkgo" // nolint "k8s.io/kubernetes/test/e2e/framework" @@ -41,7 +43,9 @@ var _ = Describe("cephfs", func() { cephfsFiles := getFilesinDirectory(cephfsDirPath) for _, file := range cephfsFiles { res, err := framework.RunKubectl("delete", "-f", cephfsDirPath+file.Name()) - framework.Logf("failed to delete resource in %s with err %v", res, err) + if err != nil { + framework.Logf("failed to delete resource in %s with err %v", res, err) + } } deleteSecret(cephfsExamplePath + "secret.yaml") deleteStorageClass(cephfsExamplePath + "storageclass.yaml") @@ -50,6 +54,8 @@ var _ = Describe("cephfs", func() { Context("Test cephfs CSI", func() { It("Test cephfs CSI", func() { + pvcPath := cephfsExamplePath + "pvc.yaml" + appPath := cephfsExamplePath + "pod.yaml" By("checking provisioner deployment is completed") err := waitForDeploymentComplete(cephfsDeploymentName, namespace, f.ClientSet, deployTimeout) if err != nil { @@ -64,16 +70,42 @@ var _ = Describe("cephfs", func() { By("create and delete a PVC", func() { By("create a PVC and Bind it to an app", func() { - pvcPath := cephfsExamplePath + "pvc.yaml" - appPath := cephfsExamplePath + "pod.yaml" + validatePVCAndAppBinding(pvcPath, appPath, f) }) By("create a PVC and Bind it to an app with normal user", func() { - pvcPath := cephfsExamplePath + "pvc.yaml" validateNormalUserPVCAccess(pvcPath, f) }) + + By("create/delete multiple PVC and App", func() { + totalCount := 2 + pvc := loadPVC(pvcPath) + pvc.Namespace = f.UniqueName + app := loadApp(appPath) + app.Namespace = f.UniqueName + //create pvc and app + for i := 0; i < totalCount; i++ { + name := fmt.Sprintf("%s%d", f.UniqueName, i) + err := createPVCAndApp(name, f, pvc, app) + if err != nil { + Fail(err.Error()) + } + + } + //TODO add cephfs backend validation + + //delete pvc and app + for i := 0; i < totalCount; i++ { + name := fmt.Sprintf("%s%d", f.UniqueName, i) + err := deletePVCAndApp(name, f, pvc, app) + if err != nil { + Fail(err.Error()) + } + + } + }) }) }) }) diff --git a/e2e/deploy-rook.go b/e2e/deploy-rook.go index 3741aaa6187..fb1dcfce999 100644 --- a/e2e/deploy-rook.go +++ b/e2e/deploy-rook.go @@ -49,12 +49,12 @@ func createRBDPool() { } func deleteFileSystem() { commonPath := fmt.Sprintf("%s/%s", rookURL, "filesystem-test.yaml") - framework.RunKubectlOrDie("delete", "-f", commonPath) + framework.RunKubectl("delete", "-f", commonPath) } func deleteRBDPool() { commonPath := fmt.Sprintf("%s/%s", rookURL, "pool-test.yaml") - framework.RunKubectlOrDie("delete", "-f", commonPath) + framework.RunKubectl("delete", "-f", commonPath) } func deployOperator(c kubernetes.Interface) { @@ -109,5 +109,5 @@ func tearDownRook() { // TODO need to add selector for cleanup validation framework.Cleanup(opPath, rookNS) commonPath := fmt.Sprintf("%s/%s", rookURL, "common.yaml") - framework.RunKubectlOrDie("delete", "-f", commonPath) + framework.RunKubectl("delete", "-f", commonPath) } diff --git a/e2e/rbd.go b/e2e/rbd.go index b85924dff9b..51364222908 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -1,6 +1,8 @@ package e2e import ( + "fmt" + . "github.com/onsi/ginkgo" // nolint "k8s.io/kubernetes/test/e2e/framework" @@ -43,7 +45,9 @@ var _ = Describe("RBD", func() { rbdFiles := getFilesinDirectory(rbdDirPath) for _, file := range rbdFiles { res, err := framework.RunKubectl("delete", "-f", rbdDirPath+file.Name()) - framework.Logf("failed to delete resource in %s with err %v", res, err) + if err != nil { + framework.Logf("failed to delete resource in %s with err %v", res, err) + } } deleteRBDPool() deleteSecret(rbdExamplePath + "secret.yaml") @@ -52,6 +56,9 @@ var _ = Describe("RBD", func() { Context("Test RBD CSI", func() { It("Test RBD CSI", func() { + + pvcPath := rbdExamplePath + "pvc.yaml" + appPath := rbdExamplePath + "pod.yaml" By("checking provisioner deployment is completed") err := waitForDeploymentComplete(rbdDeploymentName, namespace, f.ClientSet, deployTimeout) if err != nil { @@ -65,15 +72,52 @@ var _ = Describe("RBD", func() { } By("create a PVC and Bind it to an app", func() { - pvcPath := rbdExamplePath + "pvc.yaml" - appPath := rbdExamplePath + "pod.yaml" validatePVCAndAppBinding(pvcPath, appPath, f) }) By("create a PVC and Bind it to an app with normal user", func() { - pvcPath := rbdExamplePath + "pvc.yaml" validateNormalUserPVCAccess(pvcPath, f) }) + + By("create/delete multiple PVC and App", func() { + totalCount := 2 + pvc := loadPVC(pvcPath) + pvc.Namespace = f.UniqueName + app := loadApp(appPath) + app.Namespace = f.UniqueName + //create pvc and app + for i := 0; i < totalCount; i++ { + name := fmt.Sprintf("%s%d", f.UniqueName, i) + err := createPVCAndApp(name, f, pvc, app) + if err != nil { + Fail(err.Error()) + } + + } + // validate created backend rbd images + images := listRBDImages(f) + if len(images) != totalCount { + framework.Logf("backend image creation not matching pvc count, image count = % pvc count %d", len(images), totalCount) + Fail("validate multiple pvc failed") + } + + //delete pvc and app + for i := 0; i < totalCount; i++ { + name := fmt.Sprintf("%s%d", f.UniqueName, i) + err := deletePVCAndApp(name, f, pvc, app) + if err != nil { + Fail(err.Error()) + } + + } + + // validate created backend rbd images + images = listRBDImages(f) + if len(images) > 0 { + framework.Logf("left out rbd backend images count %d", len(images)) + Fail("validate multiple pvc failed") + } + }) }) }) diff --git a/e2e/utils.go b/e2e/utils.go index 6c39317f292..6f24ee2c701 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -470,28 +470,55 @@ func checkCephPods(ns string, c kubernetes.Interface, count, t int, opt *metav1. } +// createPVCAndApp creates pvc and pod +// if name is not empty same will be set as pvc and app name +func createPVCAndApp(name string, f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *v1.Pod) error { + + if name != "" { + pvc.Name = name + app.Name = name + app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = name + } + err := createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + return err + } + err = createApp(f.ClientSet, app, deployTimeout) + return err +} + +// deletePVCAndApp delete pvc and pod +// if name is not empty same will be set as pvc and app name +func deletePVCAndApp(name string, f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *v1.Pod) error { + + if name != "" { + pvc.Name = name + app.Name = name + app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = name + } + + err := deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout) + if err != nil { + return err + } + err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) + return err +} + func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) { pvc := loadPVC(pvcPath) pvc.Namespace = f.UniqueName framework.Logf("The PVC template %+v", pvc) - err := createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) - if err != nil { - Fail(err.Error()) - } app := loadApp(appPath) app.Namespace = f.UniqueName - err = createApp(f.ClientSet, app, deployTimeout) - if err != nil { - Fail(err.Error()) - } - err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout) + err := createPVCAndApp("", f, pvc, app) if err != nil { Fail(err.Error()) } - err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) + err = deletePVCAndApp("", f, pvc, app) if err != nil { Fail(err.Error()) } @@ -506,6 +533,7 @@ func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) { if err != nil { Fail(err.Error()) } + var user int64 = 2000 app := &v1.Pod{ TypeMeta: metav1.TypeMeta{ @@ -568,4 +596,21 @@ func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) { if err != nil { Fail(err.Error()) } + +} + +func listRBDImages(f *framework.Framework) []string { + + opt := metav1.ListOptions{ + LabelSelector: "app=rook-ceph-tools", + } + stdout := execCommandInPod(f, "rbd ls --pool=replicapool --format=json", rookNS, &opt) + + var imgInfos []string + + err := json.Unmarshal([]byte(stdout), &imgInfos) + if err != nil { + Fail(err.Error()) + } + return imgInfos }