rbd: add support for rbd striping

RBD supports creating rbd images with object size, strip unit and strip count to support striping. This PR adds the support for the same. More details about strip at https://docs.ceph.com/en/quincy/man/8/rbd/#striping fixes: #3124 Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
ceph · May 24, 2022 · 3a289cd · 3a289cd
1 parent 952105e
commit 3a289cd
Show file tree

Hide file tree

Showing 5 changed files with 314 additions and 7 deletions.
diff --git a/e2e/rbd.go b/e2e/rbd.go
@@ -4041,6 +4041,148 @@ var _ = Describe("RBD", func() {
 				})
 			})
 
+			By("validate rbd image strip", func() {
+				stripUnit := 4096
+				stripCount := 8
+				objectSize := 16
+				err := deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass: %v", err)
+				}
+
+				err = createRBDStorageClass(
+					f.ClientSet,
+					f,
+					defaultSCName,
+					nil,
+					map[string]string{
+						"stripUnit":  fmt.Sprintf("%d", stripUnit),
+						"stripCount": fmt.Sprintf("%d", stripCount),
+						"objectSize": fmt.Sprintf("%d", objectSize),
+					},
+					deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass: %v", err)
+				}
+				defer func() {
+					err = deleteResource(rbdExamplePath + "storageclass.yaml")
+					if err != nil {
+						e2elog.Failf("failed to delete storageclass: %v", err)
+					}
+					err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
+					if err != nil {
+						e2elog.Failf("failed to create storageclass: %v", err)
+					}
+				}()
+
+				err = createRBDSnapshotClass(f)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass: %v", err)
+				}
+				defer func() {
+					err = deleteRBDSnapshotClass()
+					if err != nil {
+						e2elog.Failf("failed to delete VolumeSnapshotClass: %v", err)
+					}
+				}()
+
+				// create PVC and bind it to an app
+				pvc, err := loadPVC(pvcPath)
+				if err != nil {
+					e2elog.Failf("failed to load PVC: %v", err)
+				}
+
+				pvc.Namespace = f.UniqueName
+
+				err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to create PVC and application: %v", err)
+				}
+				// validate created backend rbd images
+				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
+				err = validateStrip(f, pvc, stripUnit, stripCount, objectSize)
+				if err != nil {
+					e2elog.Failf("failed to validate strip %v", err)
+				}
+
+				snap := getSnapshot(snapshotPath)
+				snap.Namespace = f.UniqueName
+				snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name
+
+				err = createSnapshot(&snap, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to create snapshot: %v", err)
+				}
+				// validate created backend rbd images
+				// parent PVC + snapshot
+				totalImages := 2
+				validateRBDImageCount(f, totalImages, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)
+				pvcClone, err := loadPVC(pvcClonePath)
+				if err != nil {
+					e2elog.Failf("failed to load PVC: %v", err)
+				}
+
+				// create clone PVC as ROX
+				pvcClone.Namespace = f.UniqueName
+				pvcClone.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany}
+				err = createPVCAndvalidatePV(f.ClientSet, pvcClone, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to create PVC: %v", err)
+				}
+				// validate created backend rbd images
+				// parent pvc+ snapshot + clone
+				totalImages = 3
+				validateRBDImageCount(f, totalImages, defaultRBDPool)
+				validateOmapCount(f, 2, rbdType, defaultRBDPool, volumesType)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)
+				err = validateStrip(f, pvcClone, stripUnit, stripCount, objectSize)
+				if err != nil {
+					e2elog.Failf("failed to validate strip for clone %v", err)
+				}
+				// delete snapshot
+				err = deleteSnapshot(&snap, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to delete snapshot: %v", err)
+				}
+
+				pvcSmartClone, err := loadPVC(pvcSmartClonePath)
+				if err != nil {
+					e2elog.Failf("failed to load pvcSmartClone: %v", err)
+				}
+				pvcSmartClone.Namespace = f.UniqueName
+
+				err = createPVCAndvalidatePV(f.ClientSet, pvcSmartClone, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to create pvc: %v", err)
+				}
+				// validate created backend rbd images
+				// parent pvc + temp clone + clone
+				totalImages = 3
+				validateRBDImageCount(f, totalImages, defaultRBDPool)
+				validateOmapCount(f, 2, rbdType, defaultRBDPool, volumesType)
+				err = validateStrip(f, pvcClone, stripUnit, stripCount, objectSize)
+				if err != nil {
+					e2elog.Failf("failed to validate strip for clone %v", err)
+				}
+				// delete parent pvc
+				err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to delete PVC: %v", err)
+				}
+
+				// delete clone pvc
+				err = deletePVCAndValidatePV(f.ClientSet, pvcSmartClone, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to delete PVC: %v", err)
+				}
+				// validate created backend rbd images
+				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
+			})
+
 			// Make sure this should be last testcase in this file, because
 			// it deletes pool
 			By("Create a PVC and delete PVC when backend pool deleted", func() {

diff --git a/e2e/rbd_helper.go b/e2e/rbd_helper.go
@@ -940,3 +940,64 @@ func waitToRemoveImagesFromTrash(f *framework.Framework, poolName string, t int)
 
 	return err
 }
+
+// imageInfo strongly typed JSON spec for image info.
+type imageInfo struct {
+	ObjectUUID  string `json:"name"`
+	Size        int64  `json:"size"`
+	Format      int64  `json:"format"`
+	StripeUnit  int    `json:"stripe_unit"`
+	StripeCount int    `json:"stripe_count"`
+	Order       int    `json:"order"`
+}
+
+// getImageInfo queries rbd about the given image and returns its metadata, and returns
+// error if provided image is not found.
+func getImageInfo(f *framework.Framework, imageName, poolName string) (imageInfo, error) {
+	// rbd --format=json info [image-spec | snap-spec]
+	var imgInfo imageInfo
+
+	stdOut, stdErr, err := execCommandInToolBoxPod(
+		f,
+		fmt.Sprintf("rbd info %s %s --format json", rbdOptions(poolName), imageName),
+		rookNamespace)
+	if err != nil {
+		return imgInfo, fmt.Errorf("failed to get rbd info %w", err)
+	}
+	if stdErr != "" {
+		return imgInfo, fmt.Errorf("failed to get rbd info %v", stdErr)
+	}
+	err = json.Unmarshal([]byte(stdOut), &imgInfo)
+	if err != nil {
+		return imgInfo, fmt.Errorf("unmarshal failed: %w. raw buffer response: %s",
+			err, stdOut)
+	}
+
+	return imgInfo, nil
+}
+
+func validateStrip(f *framework.Framework, pvc *v1.PersistentVolumeClaim, stripUnit, stripCount, objectSize int) error {
+	imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
+	if err != nil {
+		return err
+	}
+
+	imgInfo, err := getImageInfo(f, imageData.imageName, defaultRBDPool)
+	if err != nil {
+		return err
+	}
+
+	if imgInfo.Order != objectSize {
+		return fmt.Errorf("objectSize %d does not match expected %d", imgInfo.Order, objectSize)
+	}
+
+	if imgInfo.StripeUnit != stripUnit {
+		return fmt.Errorf("stripUnit %d does not match expected %d", imgInfo.StripeUnit, stripUnit)
+	}
+
+	if imgInfo.StripeCount != stripCount {
+		return fmt.Errorf("stripCount %d does not match expected %d", imgInfo.StripeCount, stripCount)
+	}
+
+	return nil
+}
diff --git a/examples/rbd/storageclass.yaml b/examples/rbd/storageclass.yaml
@@ -134,6 +134,14 @@ parameters:
    #       {"domainLabel":"zone","value":"zone1"}]}
    #   ]
 
+   # Image stripping, Refer https://docs.ceph.com/en/latest/man/8/rbd/#striping
+   # For more details
+   # (optional) stripe unit in bytes.
+   # stripUnit: <>
+   # (optional) objects to stripe over before looping.
+   # stripCount: <>
+   # (optional) The object size in bytes.
+   # objectSize: <>
 reclaimPolicy: Delete
 allowVolumeExpansion: true
 mountOptions:

diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go
@@ -91,6 +91,25 @@ func (cs *ControllerServer) validateVolumeReq(ctx context.Context, req *csi.Crea
 		return err
 	}
 
+	err = validateStriping(req.Parameters)
+	if err != nil {
+		return status.Error(codes.InvalidArgument, err.Error())
+	}
+
+	return nil
+}
+
+func validateStriping(parameters map[string]string) error {
+	stripUnit := parameters["stripUnit"]
+	stripCount := parameters["stripCount"]
+	if stripUnit != "" && stripCount == "" {
+		return errors.New("stripCount must be specified when stripUnit is specified")
+	}
+
+	if stripUnit == "" && stripCount != "" {
+		return errors.New("stripUnit must be specified when stripCount is specified")
+	}
+
 	return nil
 }