Skip to content

Commit

Permalink
Add validation of backend rbd image and snapshots
Browse files Browse the repository at this point in the history 
Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
  • Loading branch information
@Madhu-1
Madhu-1 committed Jun 18, 2019
1 parent 096d7fa commit 78f6228
Show file tree
Hide file tree
Showing 5 changed files with 221 additions and 49 deletions.
46 changes: 42 additions & 4 deletions e2e/cephfs.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package e2e

import (
"fmt"
"time"

. "github.com/onsi/ginkgo" // nolint
Expand Down Expand Up @@ -43,7 +44,9 @@ var _ = Describe("cephfs", func() {
cephfsFiles := getFilesinDirectory(cephfsDirPath)
for _, file := range cephfsFiles {
res, err := framework.RunKubectl("delete", "-f", cephfsDirPath+file.Name())
framework.Logf("failed to delete resource in %s with err %v", res, err)
if err != nil {
framework.Logf("failed to delete resource in %s with err %v", res, err)
}
}
deleteResource(cephfsExamplePath + "secret.yaml")
deleteResource(cephfsExamplePath + "storageclass.yaml")
Expand All @@ -52,6 +55,8 @@ var _ = Describe("cephfs", func() {

Context("Test cephfs CSI", func() {
It("Test cephfs CSI", func() {
pvcPath := cephfsExamplePath + "pvc.yaml"
appPath := cephfsExamplePath + "pod.yaml"
By("checking provisioner statefulset is running")
timeout := time.Duration(deployTimeout) * time.Minute
err := framework.WaitForStatefulSetReplicasReady(cephfsDeploymentName, namespace, f.ClientSet, 1*time.Second, timeout)
Expand All @@ -67,16 +72,49 @@ var _ = Describe("cephfs", func() {

By("create and delete a PVC", func() {
By("create a PVC and Bind it to an app", func() {
pvcPath := cephfsExamplePath + "pvc.yaml"
appPath := cephfsExamplePath + "pod.yaml"
validatePVCAndAppBinding(pvcPath, appPath, f)

})

By("create a PVC and Bind it to an app with normal user", func() {
pvcPath := cephfsExamplePath + "pvc.yaml"
validateNormalUserPVCAccess(pvcPath, f)
})

By("create/delete multiple PVC and App", func() {
totalCount := 2
pvc, err := loadPVC(pvcPath)
if err != nil {
Fail(err.Error())
}
pvc.Namespace = f.UniqueName

app, err := loadApp(appPath)
if err != nil {
Fail(err.Error())
}
app.Namespace = f.UniqueName
// create pvc and app
for i := 0; i < totalCount; i++ {
name := fmt.Sprintf("%s%d", f.UniqueName, i)
err := createPVCAndApp(name, f, pvc, app)
if err != nil {
Fail(err.Error())
}

}
// TODO add cephfs backend validation

// delete pvc and app
for i := 0; i < totalCount; i++ {
name := fmt.Sprintf("%s%d", f.UniqueName, i)
err := deletePVCAndApp(name, f, pvc, app)
if err != nil {
Fail(err.Error())
}

}
})

})

})
Expand Down
20 changes: 15 additions & 5 deletions e2e/deploy-rook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,19 +49,26 @@ func createRBDPool() {
}
func deleteFileSystem() {
commonPath := fmt.Sprintf("%s/%s", rookURL, "filesystem-test.yaml")
framework.RunKubectlOrDie("delete", "-f", commonPath)
_, err := framework.RunKubectl("delete", "-f", commonPath)
if err != nil {
framework.Logf("failed to delete file-system %v", err)
}
}

func deleteRBDPool() {
commonPath := fmt.Sprintf("%s/%s", rookURL, "pool-test.yaml")
framework.RunKubectlOrDie("delete", "-f", commonPath)
_, err := framework.RunKubectl("delete", "-f", commonPath)
if err != nil {
framework.Logf("failed to delete pool %v", err)
}
}

func deployOperator(c kubernetes.Interface) {
opPath := fmt.Sprintf("%s/%s", rookURL, "operator.yaml")

framework.RunKubectlOrDie("create", "-f", opPath)
err := waitForDaemonSets("rook-ceph-agent", rookNS, c, deployTimeout)
_, err := framework.RunKubectl("create", "-f", opPath)
Expect(err).Should(BeNil())
err = waitForDaemonSets("rook-ceph-agent", rookNS, c, deployTimeout)
Expect(err).Should(BeNil())
err = waitForDaemonSets("rook-discover", rookNS, c, deployTimeout)
Expect(err).Should(BeNil())
Expand Down Expand Up @@ -109,5 +116,8 @@ func tearDownRook() {
// TODO need to add selector for cleanup validation
framework.Cleanup(opPath, rookNS)
commonPath := fmt.Sprintf("%s/%s", rookURL, "common.yaml")
framework.RunKubectlOrDie("delete", "-f", commonPath)
_, err := framework.RunKubectl("delete", "-f", commonPath)
if err != nil {
framework.Logf("failed to delete rook common %v", err)
}
}
76 changes: 66 additions & 10 deletions e2e/rbd.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package e2e

import (
"fmt"
"time"

. "github.com/onsi/ginkgo" // nolint
Expand Down Expand Up @@ -46,7 +47,9 @@ var _ = Describe("RBD", func() {
rbdFiles := getFilesinDirectory(rbdDirPath)
for _, file := range rbdFiles {
res, err := framework.RunKubectl("delete", "-f", rbdDirPath+file.Name())
framework.Logf("failed to delete resource in %s with err %v", res, err)
if err != nil {
framework.Logf("failed to delete resource in %s with err %v", res, err)
}
}
deleteRBDPool()
deleteResource(rbdExamplePath + "secret.yaml")
Expand All @@ -56,6 +59,12 @@ var _ = Describe("RBD", func() {

Context("Test RBD CSI", func() {
It("Test RBD CSI", func() {
pvcPath := rbdExamplePath + "raw-block-pvc.yaml"
appPath := rbdExamplePath + "raw-block-pod.yaml"
pvcClonePath := rbdExamplePath + "pvc-restore.yaml"
appClonePath := rbdExamplePath + "pod-restore.yaml"
snapshotPath := rbdExamplePath + "snapshot.yaml"

By("checking provisioner statefulset is running")
timeout := time.Duration(deployTimeout) * time.Minute
err := framework.WaitForStatefulSetReplicasReady(rbdDeploymentName, namespace, f.ClientSet, 1*time.Second, timeout)
Expand All @@ -70,22 +79,15 @@ var _ = Describe("RBD", func() {
}

By("create a PVC and Bind it to an app", func() {
pvcPath := rbdExamplePath + "pvc.yaml"
appPath := rbdExamplePath + "pod.yaml"
validatePVCAndAppBinding(pvcPath, appPath, f)
})

By("create a PVC and Bind it to an app with normal user", func() {
pvcPath := rbdExamplePath + "pvc.yaml"
validateNormalUserPVCAccess(pvcPath, f)
})

By("create a PVC clone and Bind it to an app", func() {
createRBDSnapshotClass(f)
pvcPath := rbdExamplePath + "pvc.yaml"
pvcClonePath := rbdExamplePath + "pvc-restore.yaml"
appClonePath := rbdExamplePath + "pod-restore.yaml"
snapshotPath := rbdExamplePath + "snapshot.yaml"
pvc, err := loadPVC(pvcPath)
if err != nil {
Fail(err.Error())
Expand All @@ -106,6 +108,15 @@ var _ = Describe("RBD", func() {
Fail(err.Error())
}

snapList, err := listSnapshots(f, "replicapool", pvc.Name)
if err != nil {
Fail(err.Error())
}
if len(snapList) != 1 {
framework.Logf("backend snapshot not matching kube snap count,snap count = % kube snap count %d", len(snapList), 1)
Fail("validate backend snapshot failed")
}

validatePVCAndAppBinding(pvcClonePath, appClonePath, f)

err = deleteSnapshot(&snap, deployTimeout)
Expand All @@ -119,10 +130,55 @@ var _ = Describe("RBD", func() {
})

By("create a block type PVC and Bind it to an app", func() {
pvcPath := rbdExamplePath + "raw-block-pvc.yaml"
appPath := rbdExamplePath + "raw-block-pod.yaml"
validatePVCAndAppBinding(pvcPath, appPath, f)
})

By("create/delete multiple PVC and App", func() {
totalCount := 2
pvc, err := loadPVC(pvcPath)
if err != nil {
Fail(err.Error())
}
pvc.Namespace = f.UniqueName

app, err := loadApp(appPath)
if err != nil {
Fail(err.Error())
}
app.Namespace = f.UniqueName
// create pvc and app
for i := 0; i < totalCount; i++ {
name := fmt.Sprintf("%s%d", f.UniqueName, i)
err := createPVCAndApp(name, f, pvc, app)
if err != nil {
Fail(err.Error())
}

}
// validate created backend rbd images
images := listRBDImages(f)
if len(images) != totalCount {
framework.Logf("backend image creation not matching pvc count, image count = % pvc count %d", len(images), totalCount)
Fail("validate multiple pvc failed")
}

// delete pvc and app
for i := 0; i < totalCount; i++ {
name := fmt.Sprintf("%s%d", f.UniqueName, i)
err := deletePVCAndApp(name, f, pvc, app)
if err != nil {
Fail(err.Error())
}

}

// validate created backend rbd images
images = listRBDImages(f)
if len(images) > 0 {
framework.Logf("left out rbd backend images count %d", len(images))
Fail("validate multiple pvc failed")
}
})
})
})

Expand Down
79 changes: 69 additions & 10 deletions e2e/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (
"strings"
"time"

"github.com/ceph/ceph-csi/pkg/rbd"

"github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1alpha1"
snapClient "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/typed/volumesnapshot/v1alpha1"
. "github.com/onsi/ginkgo" // nolint
Expand Down Expand Up @@ -507,34 +509,61 @@ func checkCephPods(ns string, c kubernetes.Interface, count, t int, opt *metav1.

}

// createPVCAndApp creates pvc and pod
// if name is not empty same will be set as pvc and app name
func createPVCAndApp(name string, f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *v1.Pod) error {

if name != "" {
pvc.Name = name
app.Name = name
app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = name
}
err := createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
return err
}
err = createApp(f.ClientSet, app, deployTimeout)
return err
}

// deletePVCAndApp delete pvc and pod
// if name is not empty same will be set as pvc and app name
func deletePVCAndApp(name string, f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *v1.Pod) error {

if name != "" {
pvc.Name = name
app.Name = name
app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = name
}

err := deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
if err != nil {
return err
}
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
return err
}

func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) {
pvc, err := loadPVC(pvcPath)
if pvc == nil {
Fail(err.Error())
}
pvc.Namespace = f.UniqueName
framework.Logf("The PVC template %+v", pvc)
err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
Fail(err.Error())
}

app, err := loadApp(appPath)
if err != nil {
Fail(err.Error())
}
app.Namespace = f.UniqueName
err = createApp(f.ClientSet, app, deployTimeout)
if err != nil {
Fail(err.Error())
}

err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
err = createPVCAndApp("", f, pvc, app)
if err != nil {
Fail(err.Error())
}

err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
err = deletePVCAndApp("", f, pvc, app)
if err != nil {
Fail(err.Error())
}
Expand Down Expand Up @@ -682,3 +711,33 @@ func deleteSnapshot(snap *v1alpha1.VolumeSnapshot, t int) error {
return false, nil
})
}

func listRBDImages(f *framework.Framework) []string {

opt := metav1.ListOptions{
LabelSelector: "app=rook-ceph-tools",
}
stdout := execCommandInPod(f, "rbd ls --pool=replicapool --format=json", rookNS, &opt)

var imgInfos []string

err := json.Unmarshal([]byte(stdout), &imgInfos)
if err != nil {
Fail(err.Error())
}
return imgInfos
}

func listSnapshots(f *framework.Framework, pool, imageName string) ([]rbd.SnapInfo, error) {
opt := metav1.ListOptions{
LabelSelector: "app=rook-ceph-tools",
}
adminKey := execCommandInPod(f, "ceph auth get-key client.admin", rookNS, &opt)
id := string([]byte("admin"))
key := string([]byte(adminKey))
mons := getMons(rookNS, f.ClientSet)

snapList, err := rbd.ListSnapshots(mons[0], id, key, pool, imageName)

return snapList, err
}
Loading

0 comments on commit 78f6228

Please sign in to comment.