diff --git a/e2e/cephfs.go b/e2e/cephfs.go index fad6ad881e33..ed5545ecd855 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -1,6 +1,7 @@ package e2e import ( + "fmt" "time" . "github.com/onsi/ginkgo" // nolint @@ -43,7 +44,9 @@ var _ = Describe("cephfs", func() { cephfsFiles := getFilesinDirectory(cephfsDirPath) for _, file := range cephfsFiles { res, err := framework.RunKubectl("delete", "-f", cephfsDirPath+file.Name()) - framework.Logf("failed to delete resource in %s with err %v", res, err) + if err != nil { + framework.Logf("failed to delete resource in %s with err %v", res, err) + } } deleteResource(cephfsExamplePath + "secret.yaml") deleteResource(cephfsExamplePath + "storageclass.yaml") @@ -52,6 +55,8 @@ var _ = Describe("cephfs", func() { Context("Test cephfs CSI", func() { It("Test cephfs CSI", func() { + pvcPath := cephfsExamplePath + "pvc.yaml" + appPath := cephfsExamplePath + "pod.yaml" By("checking provisioner statefulset is running") timeout := time.Duration(deployTimeout) * time.Minute err := framework.WaitForStatefulSetReplicasReady(cephfsDeploymentName, namespace, f.ClientSet, 1*time.Second, timeout) @@ -67,16 +72,49 @@ var _ = Describe("cephfs", func() { By("create and delete a PVC", func() { By("create a PVC and Bind it to an app", func() { - pvcPath := cephfsExamplePath + "pvc.yaml" - appPath := cephfsExamplePath + "pod.yaml" validatePVCAndAppBinding(pvcPath, appPath, f) }) By("create a PVC and Bind it to an app with normal user", func() { - pvcPath := cephfsExamplePath + "pvc.yaml" validateNormalUserPVCAccess(pvcPath, f) }) + + By("create/delete multiple PVC and App", func() { + totalCount := 2 + pvc, err := loadPVC(pvcPath) + if err != nil { + Fail(err.Error()) + } + pvc.Namespace = f.UniqueName + + app, err := loadApp(appPath) + if err != nil { + Fail(err.Error()) + } + app.Namespace = f.UniqueName + // create pvc and app + for i := 0; i < totalCount; i++ { + name := fmt.Sprintf("%s%d", f.UniqueName, i) + err := createPVCAndApp(name, f, pvc, app) + if err != nil { + Fail(err.Error()) + } + + } + // TODO add cephfs backend validation + + // delete pvc and app + for i := 0; i < totalCount; i++ { + name := fmt.Sprintf("%s%d", f.UniqueName, i) + err := deletePVCAndApp(name, f, pvc, app) + if err != nil { + Fail(err.Error()) + } + + } + }) + }) }) diff --git a/e2e/deploy-rook.go b/e2e/deploy-rook.go index 3741aaa6187e..496f052af942 100644 --- a/e2e/deploy-rook.go +++ b/e2e/deploy-rook.go @@ -49,19 +49,26 @@ func createRBDPool() { } func deleteFileSystem() { commonPath := fmt.Sprintf("%s/%s", rookURL, "filesystem-test.yaml") - framework.RunKubectlOrDie("delete", "-f", commonPath) + _, err := framework.RunKubectl("delete", "-f", commonPath) + if err != nil { + framework.Logf("failed to delete file-system %v", err) + } } func deleteRBDPool() { commonPath := fmt.Sprintf("%s/%s", rookURL, "pool-test.yaml") - framework.RunKubectlOrDie("delete", "-f", commonPath) + _, err := framework.RunKubectl("delete", "-f", commonPath) + if err != nil { + framework.Logf("failed to delete pool %v", err) + } } func deployOperator(c kubernetes.Interface) { opPath := fmt.Sprintf("%s/%s", rookURL, "operator.yaml") - framework.RunKubectlOrDie("create", "-f", opPath) - err := waitForDaemonSets("rook-ceph-agent", rookNS, c, deployTimeout) + _, err := framework.RunKubectl("create", "-f", opPath) + Expect(err).Should(BeNil()) + err = waitForDaemonSets("rook-ceph-agent", rookNS, c, deployTimeout) Expect(err).Should(BeNil()) err = waitForDaemonSets("rook-discover", rookNS, c, deployTimeout) Expect(err).Should(BeNil()) @@ -109,5 +116,8 @@ func tearDownRook() { // TODO need to add selector for cleanup validation framework.Cleanup(opPath, rookNS) commonPath := fmt.Sprintf("%s/%s", rookURL, "common.yaml") - framework.RunKubectlOrDie("delete", "-f", commonPath) + _, err := framework.RunKubectl("delete", "-f", commonPath) + if err != nil { + framework.Logf("failed to delete rook common %v", err) + } } diff --git a/e2e/rbd.go b/e2e/rbd.go index 744a2783b88e..4251c52899c5 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -1,6 +1,7 @@ package e2e import ( + "fmt" "time" . "github.com/onsi/ginkgo" // nolint @@ -46,7 +47,9 @@ var _ = Describe("RBD", func() { rbdFiles := getFilesinDirectory(rbdDirPath) for _, file := range rbdFiles { res, err := framework.RunKubectl("delete", "-f", rbdDirPath+file.Name()) - framework.Logf("failed to delete resource in %s with err %v", res, err) + if err != nil { + framework.Logf("failed to delete resource in %s with err %v", res, err) + } } deleteRBDPool() deleteResource(rbdExamplePath + "secret.yaml") @@ -56,6 +59,12 @@ var _ = Describe("RBD", func() { Context("Test RBD CSI", func() { It("Test RBD CSI", func() { + pvcPath := rbdExamplePath + "raw-block-pvc.yaml" + appPath := rbdExamplePath + "raw-block-pod.yaml" + pvcClonePath := rbdExamplePath + "pvc-restore.yaml" + appClonePath := rbdExamplePath + "pod-restore.yaml" + snapshotPath := rbdExamplePath + "snapshot.yaml" + By("checking provisioner statefulset is running") timeout := time.Duration(deployTimeout) * time.Minute err := framework.WaitForStatefulSetReplicasReady(rbdDeploymentName, namespace, f.ClientSet, 1*time.Second, timeout) @@ -70,22 +79,15 @@ var _ = Describe("RBD", func() { } By("create a PVC and Bind it to an app", func() { - pvcPath := rbdExamplePath + "pvc.yaml" - appPath := rbdExamplePath + "pod.yaml" validatePVCAndAppBinding(pvcPath, appPath, f) }) By("create a PVC and Bind it to an app with normal user", func() { - pvcPath := rbdExamplePath + "pvc.yaml" validateNormalUserPVCAccess(pvcPath, f) }) By("create a PVC clone and Bind it to an app", func() { createRBDSnapshotClass(f) - pvcPath := rbdExamplePath + "pvc.yaml" - pvcClonePath := rbdExamplePath + "pvc-restore.yaml" - appClonePath := rbdExamplePath + "pod-restore.yaml" - snapshotPath := rbdExamplePath + "snapshot.yaml" pvc, err := loadPVC(pvcPath) if err != nil { Fail(err.Error()) @@ -106,6 +108,15 @@ var _ = Describe("RBD", func() { Fail(err.Error()) } + snapList, err := listSnapshots(f, "replicapool", pvc.Name) + if err != nil { + Fail(err.Error()) + } + if len(snapList) != 1 { + framework.Logf("backend snapshot not matching kube snap count,snap count = % kube snap count %d", len(snapList), 1) + Fail("validate backend snapshot failed") + } + validatePVCAndAppBinding(pvcClonePath, appClonePath, f) err = deleteSnapshot(&snap, deployTimeout) @@ -119,10 +130,55 @@ var _ = Describe("RBD", func() { }) By("create a block type PVC and Bind it to an app", func() { - pvcPath := rbdExamplePath + "raw-block-pvc.yaml" - appPath := rbdExamplePath + "raw-block-pod.yaml" validatePVCAndAppBinding(pvcPath, appPath, f) }) + + By("create/delete multiple PVC and App", func() { + totalCount := 2 + pvc, err := loadPVC(pvcPath) + if err != nil { + Fail(err.Error()) + } + pvc.Namespace = f.UniqueName + + app, err := loadApp(appPath) + if err != nil { + Fail(err.Error()) + } + app.Namespace = f.UniqueName + // create pvc and app + for i := 0; i < totalCount; i++ { + name := fmt.Sprintf("%s%d", f.UniqueName, i) + err := createPVCAndApp(name, f, pvc, app) + if err != nil { + Fail(err.Error()) + } + + } + // validate created backend rbd images + images := listRBDImages(f) + if len(images) != totalCount { + framework.Logf("backend image creation not matching pvc count, image count = % pvc count %d", len(images), totalCount) + Fail("validate multiple pvc failed") + } + + // delete pvc and app + for i := 0; i < totalCount; i++ { + name := fmt.Sprintf("%s%d", f.UniqueName, i) + err := deletePVCAndApp(name, f, pvc, app) + if err != nil { + Fail(err.Error()) + } + + } + + // validate created backend rbd images + images = listRBDImages(f) + if len(images) > 0 { + framework.Logf("left out rbd backend images count %d", len(images)) + Fail("validate multiple pvc failed") + } + }) }) }) diff --git a/e2e/utils.go b/e2e/utils.go index 551f1b0ee3ae..2f6d686e9b42 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -9,6 +9,8 @@ import ( "strings" "time" + "github.com/ceph/ceph-csi/pkg/rbd" + "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1alpha1" snapClient "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/typed/volumesnapshot/v1alpha1" . "github.com/onsi/ginkgo" // nolint @@ -507,6 +509,41 @@ func checkCephPods(ns string, c kubernetes.Interface, count, t int, opt *metav1. } +// createPVCAndApp creates pvc and pod +// if name is not empty same will be set as pvc and app name +func createPVCAndApp(name string, f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *v1.Pod) error { + + if name != "" { + pvc.Name = name + app.Name = name + app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = name + } + err := createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + return err + } + err = createApp(f.ClientSet, app, deployTimeout) + return err +} + +// deletePVCAndApp delete pvc and pod +// if name is not empty same will be set as pvc and app name +func deletePVCAndApp(name string, f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *v1.Pod) error { + + if name != "" { + pvc.Name = name + app.Name = name + app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = name + } + + err := deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout) + if err != nil { + return err + } + err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) + return err +} + func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) { pvc, err := loadPVC(pvcPath) if pvc == nil { @@ -514,27 +551,19 @@ func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) { } pvc.Namespace = f.UniqueName framework.Logf("The PVC template %+v", pvc) - err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) - if err != nil { - Fail(err.Error()) - } app, err := loadApp(appPath) if err != nil { Fail(err.Error()) } app.Namespace = f.UniqueName - err = createApp(f.ClientSet, app, deployTimeout) - if err != nil { - Fail(err.Error()) - } - err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout) + err = createPVCAndApp("", f, pvc, app) if err != nil { Fail(err.Error()) } - err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) + err = deletePVCAndApp("", f, pvc, app) if err != nil { Fail(err.Error()) } @@ -682,3 +711,33 @@ func deleteSnapshot(snap *v1alpha1.VolumeSnapshot, t int) error { return false, nil }) } + +func listRBDImages(f *framework.Framework) []string { + + opt := metav1.ListOptions{ + LabelSelector: "app=rook-ceph-tools", + } + stdout := execCommandInPod(f, "rbd ls --pool=replicapool --format=json", rookNS, &opt) + + var imgInfos []string + + err := json.Unmarshal([]byte(stdout), &imgInfos) + if err != nil { + Fail(err.Error()) + } + return imgInfos +} + +func listSnapshots(f *framework.Framework, pool, imageName string) ([]rbd.SnapInfo, error) { + opt := metav1.ListOptions{ + LabelSelector: "app=rook-ceph-tools", + } + adminKey := execCommandInPod(f, "ceph auth get-key client.admin", rookNS, &opt) + id := string([]byte("admin")) + key := string([]byte(adminKey)) + mons := getMons(rookNS, f.ClientSet) + + snapList, err := rbd.ListSnapshots(mons[0], id, key, pool, imageName) + + return snapList, err +} diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go index cbbbd85c7625..ab60a28be842 100644 --- a/pkg/rbd/rbd_util.go +++ b/pkg/rbd/rbd_util.go @@ -699,27 +699,17 @@ func getImageInfo(monitors, adminID, key, poolName, imageName string) (imageInfo return imgInfo, nil } -// snapInfo strongly typed JSON spec for snap ls rbd output -type snapInfo struct { +// SnapInfo strongly typed JSON spec for snap ls rbd output +type SnapInfo struct { ID int64 `json:"id"` Name string `json:"name"` Size int64 `json:"size"` Timestamp string `json:"timestamp"` } -/* -getSnapInfo queries rbd about the snapshots of the given image and returns its metadata, and -returns ErrImageNotFound if provided image is not found, and ErrSnapNotFound if provided snap -is not found in the images snapshot list -*/ -func getSnapInfo(monitors, adminID, key, poolName, imageName, snapName string) (snapInfo, error) { - // rbd --format=json snap ls [image-spec] - - var ( - snpInfo snapInfo - snaps []snapInfo - ) - +// ListSnapshots lists snapshot in ceph backend +func ListSnapshots(monitors, adminID, key, poolName, imageName string) ([]SnapInfo, error) { + var snaps []SnapInfo stdout, _, err := util.ExecCommand( "rbd", "-m", monitors, @@ -729,23 +719,42 @@ func getSnapInfo(monitors, adminID, key, poolName, imageName, snapName string) ( "--format="+"json", "snap", "ls", poolName+"/"+imageName) if err != nil { - klog.Errorf("failed getting snap (%s) information from image (%s): (%s)", - snapName, poolName+"/"+imageName, err) + klog.Errorf("failed getting snapshot information from image (%s): (%s)", + poolName+"/"+imageName, err) if strings.Contains(string(stdout), "rbd: error opening image "+imageName+ ": (2) No such file or directory") { - return snpInfo, ErrImageNotFound{imageName, err} + return snaps, ErrImageNotFound{imageName, err} } - return snpInfo, err + return snaps, err } err = json.Unmarshal(stdout, &snaps) if err != nil { klog.Errorf("failed to parse JSON output of image snap list (%s): (%s)", poolName+"/"+imageName, err) - return snpInfo, fmt.Errorf("unmarshal failed: %+v. raw buffer response: %s", + return snaps, fmt.Errorf("unmarshal failed: %+v. raw buffer response: %s", err, string(stdout)) } + return snaps, nil +} + +/* +getSnapInfo queries rbd about the snapshots of the given image and returns its metadata, and +returns ErrImageNotFound if provided image is not found, and ErrSnapNotFound if provided snap +is not found in the images snapshot list +*/ +func getSnapInfo(monitors, adminID, key, poolName, imageName, snapName string) (SnapInfo, error) { + // rbd --format=json snap ls [image-spec] + + var ( + snpInfo SnapInfo + ) + + snaps, err := ListSnapshots(monitors, adminID, key, poolName, imageName) + if err != nil { + return snpInfo, err + } for _, snap := range snaps { if snap.Name == snapName { return snap, nil