diff --git a/e2e/cephfs.go b/e2e/cephfs.go index 22e9e17a8659..a20290eb3845 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -43,8 +43,8 @@ var _ = Describe("cephfs", func() { res, err := framework.RunKubectl("delete", "-f", cephfsDirPath+file.Name()) framework.Logf("failed to delete resource in %s with err %v", res, err) } - deleteSecret(cephfsExamplePath + "secret.yaml") - deleteStorageClass(cephfsExamplePath + "storageclass.yaml") + deleteResource(cephfsExamplePath + "secret.yaml") + deleteResource(cephfsExamplePath + "storageclass.yaml") deleteFileSystem() }) @@ -75,6 +75,7 @@ var _ = Describe("cephfs", func() { validateNormalUserPVCAccess(pvcPath, f) }) }) + }) }) diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 5e74830a17a4..641bb977d41e 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -23,7 +23,6 @@ func init() { flag.BoolVar(&rookRequired, "deploy-rook", true, "deploy rook on kubernetes") flag.IntVar(&deployTimeout, "deploy-timeout", 10, "timeout to wait for created kubernetes resources") - // Register framework flags, then handle flags framework.HandleFlags() framework.AfterReadingAllFlags(&framework.TestContext) diff --git a/e2e/rbd.go b/e2e/rbd.go index b85924dff9bb..91d5ce1a4dc2 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -37,6 +37,7 @@ var _ = Describe("RBD", func() { deployRBDPlugin() createRBDStorageClass(f.ClientSet, f) createRBDSecret(f.ClientSet, f) + }) AfterEach(func() { @@ -46,8 +47,9 @@ var _ = Describe("RBD", func() { framework.Logf("failed to delete resource in %s with err %v", res, err) } deleteRBDPool() - deleteSecret(rbdExamplePath + "secret.yaml") - deleteStorageClass(rbdExamplePath + "storageclass.yaml") + deleteResource(rbdExamplePath + "secret.yaml") + deleteResource(rbdExamplePath + "storageclass.yaml") + deleteResource(rbdExamplePath + "snapshotclass.yaml") }) Context("Test RBD CSI", func() { @@ -74,6 +76,50 @@ var _ = Describe("RBD", func() { pvcPath := rbdExamplePath + "pvc.yaml" validateNormalUserPVCAccess(pvcPath, f) }) + + By("create a PVC clone and Bind it to an app", func() { + createRBDSnapshotClass(f) + pvcPath := rbdExamplePath + "pvc.yaml" + pvcClonePath := rbdExamplePath + "pvc-restore.yaml" + appClonePath := rbdExamplePath + "pod-restore.yaml" + snapshotPath := rbdExamplePath + "snapshot.yaml" + pvc, err := loadPVC(pvcPath) + if err != nil { + Fail(err.Error()) + } + + pvc.Namespace = f.UniqueName + framework.Logf("The PVC template %+v", pvc) + err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + Fail(err.Error()) + } + snap := getSnapshot(snapshotPath) + snap.Namespace = f.UniqueName + snap.Spec.Source.Name = pvc.Name + snap.Spec.Source.Kind = "PersistentVolumeClaim" + err = createSnapshot(&snap, deployTimeout) + if err != nil { + Fail(err.Error()) + } + + validatePVCAndAppBinding(pvcClonePath, appClonePath, f) + + err = deleteSnapshot(&snap, deployTimeout) + if err != nil { + Fail(err.Error()) + } + err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + Fail(err.Error()) + } + }) + + By("create a block type PVC and Bind it to an app", func() { + pvcPath := rbdExamplePath + "raw-block-pvc.yaml" + appPath := rbdExamplePath + "raw-block-pod.yaml" + validatePVCAndAppBinding(pvcPath, appPath, f) + }) }) }) diff --git a/e2e/utils.go b/e2e/utils.go index 6c39317f292a..551f1b0ee3ae 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -9,13 +9,14 @@ import ( "strings" "time" - apierrs "k8s.io/apimachinery/pkg/api/errors" - + "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1alpha1" + snapClient "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/typed/volumesnapshot/v1alpha1" . "github.com/onsi/ginkgo" // nolint . "github.com/onsi/gomega" // nolint apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" scv1 "k8s.io/api/storage/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" utilyaml "k8s.io/apimachinery/pkg/util/yaml" @@ -141,19 +142,32 @@ func getMons(ns string, c kubernetes.Interface) []string { return services } -func getStorageClass(c kubernetes.Interface, path string) scv1.StorageClass { +func getStorageClass(path string) scv1.StorageClass { sc := scv1.StorageClass{} err := unmarshal(path, &sc) Expect(err).Should(BeNil()) + return sc +} - mons := getMons(rookNS, c) - sc.Parameters["monitors"] = strings.Join(mons, ",") +func getSnapshotClass(path string) v1alpha1.VolumeSnapshotClass { + sc := v1alpha1.VolumeSnapshotClass{} + sc.Kind = "VolumeSnapshotClass" + sc.APIVersion = "snapshot.storage.k8s.io/v1alpha1" + err := unmarshal(path, &sc) + Expect(err).Should(BeNil()) + return sc +} + +func getSnapshot(path string) v1alpha1.VolumeSnapshot { + sc := v1alpha1.VolumeSnapshot{} + err := unmarshal(path, &sc) + Expect(err).Should(BeNil()) return sc } func createCephfsStorageClass(c kubernetes.Interface, f *framework.Framework) { scPath := fmt.Sprintf("%s/%s", cephfsExamplePath, "storageclass.yaml") - sc := getStorageClass(c, scPath) + sc := getStorageClass(scPath) sc.Parameters["pool"] = "myfs-data0" sc.Parameters["fsName"] = "myfs" opt := metav1.ListOptions{ @@ -170,7 +184,7 @@ func createCephfsStorageClass(c kubernetes.Interface, f *framework.Framework) { func createRBDStorageClass(c kubernetes.Interface, f *framework.Framework) { scPath := fmt.Sprintf("%s/%s", rbdExamplePath, "storageclass.yaml") - sc := getStorageClass(c, scPath) + sc := getStorageClass(scPath) delete(sc.Parameters, "userid") sc.Parameters["pool"] = "replicapool" opt := metav1.ListOptions{ @@ -185,6 +199,34 @@ func createRBDStorageClass(c kubernetes.Interface, f *framework.Framework) { Expect(err).Should(BeNil()) } +func newSnapshotClient() (*snapClient.VolumesnapshotV1alpha1Client, error) { + config, err := framework.LoadConfig() + if err != nil { + return nil, fmt.Errorf("error creating client: %v", err.Error()) + } + c, err := snapClient.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("error creating snapshot client: %v", err.Error()) + } + return c, err +} +func createRBDSnapshotClass(f *framework.Framework) { + scPath := fmt.Sprintf("%s/%s", rbdExamplePath, "snapshotclass.yaml") + sc := getSnapshotClass(scPath) + + opt := metav1.ListOptions{ + LabelSelector: "app=rook-ceph-tools", + } + fsID := execCommandInPod(f, "ceph fsid", rookNS, &opt) + // remove new line present in fsID + fsID = strings.Trim(fsID, "\n") + sc.Parameters["clusterID"] = fsID + sclient, err := newSnapshotClient() + Expect(err).Should(BeNil()) + _, err = sclient.VolumeSnapshotClasses().Create(&sc) + Expect(err).Should(BeNil()) +} + func createConfigMap(c kubernetes.Interface, f *framework.Framework) { path := rbdDirPath + rbdConfigMap cm := v1.ConfigMap{} @@ -254,23 +296,18 @@ func createRBDSecret(c kubernetes.Interface, f *framework.Framework) { Expect(err).Should(BeNil()) } -func deleteSecret(scPath string) { - _, err := framework.RunKubectl("delete", "-f", scPath) - Expect(err).Should(BeNil()) -} - -func deleteStorageClass(scPath string) { +func deleteResource(scPath string) { _, err := framework.RunKubectl("delete", "-f", scPath) Expect(err).Should(BeNil()) } -func loadPVC(path string) *v1.PersistentVolumeClaim { +func loadPVC(path string) (*v1.PersistentVolumeClaim, error) { pvc := &v1.PersistentVolumeClaim{} err := unmarshal(path, &pvc) if err != nil { - return nil + return nil, err } - return pvc + return pvc, err } func createPVCAndvalidatePV(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, t int) error { @@ -360,13 +397,13 @@ func deletePVCAndValidatePV(c kubernetes.Interface, pvc *v1.PersistentVolumeClai }) } -func loadApp(path string) *v1.Pod { +func loadApp(path string) (*v1.Pod, error) { app := v1.Pod{} err := unmarshal(path, &app) if err != nil { - return nil + return nil, err } - return &app + return &app, nil } func createApp(c kubernetes.Interface, app *v1.Pod, timeout int) error { @@ -471,15 +508,21 @@ func checkCephPods(ns string, c kubernetes.Interface, count, t int, opt *metav1. } func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) { - pvc := loadPVC(pvcPath) + pvc, err := loadPVC(pvcPath) + if pvc == nil { + Fail(err.Error()) + } pvc.Namespace = f.UniqueName framework.Logf("The PVC template %+v", pvc) - err := createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) + err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) if err != nil { Fail(err.Error()) } - app := loadApp(appPath) + app, err := loadApp(appPath) + if err != nil { + Fail(err.Error()) + } app.Namespace = f.UniqueName err = createApp(f.ClientSet, app, deployTimeout) if err != nil { @@ -498,11 +541,14 @@ func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) { } func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) { - pvc := loadPVC(pvcPath) + pvc, err := loadPVC(pvcPath) + if err != nil { + Fail(err.Error()) + } pvc.Namespace = f.UniqueName pvc.Name = f.UniqueName framework.Logf("The PVC template %+v", pvc) - err := createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) + err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) if err != nil { Fail(err.Error()) } @@ -569,3 +615,70 @@ func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) { Fail(err.Error()) } } + +func createSnapshot(snap *v1alpha1.VolumeSnapshot, t int) error { + + sclient, err := newSnapshotClient() + if err != nil { + return err + } + _, err = sclient.VolumeSnapshots(snap.Namespace).Create(snap) + if err != nil { + return err + } + + timeout := time.Duration(t) * time.Minute + name := snap.Name + start := time.Now() + framework.Logf("Waiting up to %v to be in Ready state", snap) + + return wait.PollImmediate(poll, timeout, func() (bool, error) { + framework.Logf("waiting for snapshot %s (%d seconds elapsed)", snap.Name, int(time.Since(start).Seconds())) + snaps, err := sclient.VolumeSnapshots(snap.Namespace).Get(name, metav1.GetOptions{}) + if err != nil { + framework.Logf("Error getting snapshot in namespace: '%s': %v", snap.Namespace, err) + if testutils.IsRetryableAPIError(err) { + return false, nil + } + if apierrs.IsNotFound(err) { + return false, nil + } + return false, err + } + if snaps.Status.ReadyToUse { + return true, nil + } + return false, nil + }) +} + +func deleteSnapshot(snap *v1alpha1.VolumeSnapshot, t int) error { + sclient, err := newSnapshotClient() + if err != nil { + return err + } + err = sclient.VolumeSnapshots(snap.Namespace).Delete(snap.Name, &metav1.DeleteOptions{}) + if err != nil { + return err + } + + timeout := time.Duration(t) * time.Minute + name := snap.Name + start := time.Now() + framework.Logf("Waiting up to %v to be in Ready state", snap) + + return wait.PollImmediate(poll, timeout, func() (bool, error) { + framework.Logf("waiting for snapshot %s (%d seconds elapsed)", name, int(time.Since(start).Seconds())) + _, err := sclient.VolumeSnapshots(snap.Namespace).Get(name, metav1.GetOptions{}) + + if err == nil { + return false, nil + } + + if !apierrs.IsNotFound(err) { + return false, fmt.Errorf("get on deleted snapshot %v failed with error other than \"not found\": %v", name, err) + } + + return false, nil + }) +}