rbd: add support for rbd striping

docs/deploy-rbd.md

@@ -66,6 +66,9 @@ make image-cephcsi
 | `mounter`                                                                                           | no                   | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images                                                                                                                                                                                                                                                                                                                                                                                                                        |
 | `encrypted`                                                                                         | no                   | disabled by default, use `"true"` to enable LUKS encryption on PVC and `"false"` to disable it. **Do not change for existing storageclasses**                                                                                                                                                                                                                                                                                                                                                                                     |
 | `encryptionKMSID`                                                                                   | no                   | required if encryption is enabled and a kms is used to store passphrases                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| `stripeUnit`                                                                                   | no                   | stripe unit in bytes                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
+| `stripeCount`                                                                                   | no                   | objects to stripe over before looping                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| `objectSize`                                                                                   | no                   | object size in bytes                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
 
 **NOTE:** An accompanying CSI configuration file, needs to be provided to the
 running pods. Refer to [Creating CSI configuration](../examples/README.md#creating-csi-configuration)

e2e/rbd.go

@@ -4080,6 +4080,153 @@ var _ = Describe("RBD", func() {
 				})
 			})
 
+			By("validate rbd image stripe", func() {
+				stripeUnit := 4096
+				stripeCount := 8
+				objectSize := 131072
+				err := deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass: %v", err)
+				}
+
+				err = createRBDStorageClass(
+					f.ClientSet,
+					f,
+					defaultSCName,
+					nil,
+					map[string]string{
+						"stripeUnit":  fmt.Sprintf("%d", stripeUnit),
+						"stripeCount": fmt.Sprintf("%d", stripeCount),
+						"objectSize":  fmt.Sprintf("%d", objectSize),
+					},
+					deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass: %v", err)
+				}
+				defer func() {
+					err = deleteResource(rbdExamplePath + "storageclass.yaml")
+					if err != nil {
+						e2elog.Failf("failed to delete storageclass: %v", err)
+					}
+					err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
+					if err != nil {
+						e2elog.Failf("failed to create storageclass: %v", err)
+					}
+				}()
+
+				err = createRBDSnapshotClass(f)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass: %v", err)
+				}
+				defer func() {
+					err = deleteRBDSnapshotClass()
+					if err != nil {
+						e2elog.Failf("failed to delete VolumeSnapshotClass: %v", err)
+					}
+				}()
+
+				// create PVC and bind it to an app
+				pvc, err := loadPVC(pvcPath)
+				if err != nil {
+					e2elog.Failf("failed to load PVC: %v", err)
+				}
+
+				pvc.Namespace = f.UniqueName
+
+				err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to create PVC and application: %v", err)
+				}
+				// validate created backend rbd images
+				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
+				err = validateStripe(f, pvc, stripeUnit, stripeCount, objectSize)
+				if err != nil {
+					e2elog.Failf("failed to validate stripe: %v", err)
+				}
+
+				snap := getSnapshot(snapshotPath)
+				snap.Namespace = f.UniqueName
+				snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name
+
+				err = createSnapshot(&snap, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to create snapshot: %v", err)
+				}
+				// validate created backend rbd images
+				// parent PVC + snapshot
+				totalImages := 2
+				validateRBDImageCount(f, totalImages, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)
+				pvcClone, err := loadPVC(pvcClonePath)
+				if err != nil {
+					e2elog.Failf("failed to load PVC: %v", err)
+				}
+
+				// create clone PVC as ROX
+				pvcClone.Namespace = f.UniqueName
+				pvcClone.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany}
+				err = createPVCAndvalidatePV(f.ClientSet, pvcClone, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to create PVC: %v", err)
+				}
+				// validate created backend rbd images
+				// parent pvc + snapshot + clone
+				totalImages = 3
+				validateRBDImageCount(f, totalImages, defaultRBDPool)
+				validateOmapCount(f, 2, rbdType, defaultRBDPool, volumesType)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)
+				err = validateStripe(f, pvcClone, stripeUnit, stripeCount, objectSize)
+				if err != nil {
+					e2elog.Failf("failed to validate stripe for clone: %v", err)
+				}
+				// delete snapshot
+				err = deleteSnapshot(&snap, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to delete snapshot: %v", err)
+				}
+				// delete clone pvc
+				err = deletePVCAndValidatePV(f.ClientSet, pvcClone, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to delete PVC: %v", err)
+				}
+
+				pvcSmartClone, err := loadPVC(pvcSmartClonePath)
+				if err != nil {
+					e2elog.Failf("failed to load pvcSmartClone: %v", err)
+				}
+				pvcSmartClone.Namespace = f.UniqueName
+
+				err = createPVCAndvalidatePV(f.ClientSet, pvcSmartClone, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to create pvc: %v", err)
+				}
+				// validate created backend rbd images
+				// parent pvc + temp clone + clone
+				totalImages = 3
+				validateRBDImageCount(f, totalImages, defaultRBDPool)
+				validateOmapCount(f, 2, rbdType, defaultRBDPool, volumesType)
+				err = validateStripe(f, pvcSmartClone, stripeUnit, stripeCount, objectSize)
+				if err != nil {
+					e2elog.Failf("failed to validate stripe for clone: %v", err)
+				}
+				// delete parent pvc
+				err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to delete PVC: %v", err)
+				}
+
+				// delete clone pvc
+				err = deletePVCAndValidatePV(f.ClientSet, pvcSmartClone, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to delete PVC: %v", err)
+				}
+				// validate created backend rbd images
+				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
+			})
+
 			// Make sure this should be last testcase in this file, because
 			// it deletes pool
 			By("Create a PVC and delete PVC when backend pool deleted", func() {

e2e/rbd_helper.go

@@ -942,3 +942,69 @@ func waitToRemoveImagesFromTrash(f *framework.Framework, poolName string, t int)
 
 	return err
 }
+
+// imageInfo strongly typed JSON spec for image info.
+type imageInfo struct {
+	Name        string `json:"name"`
+	StripeUnit  int    `json:"stripe_unit"`
+	StripeCount int    `json:"stripe_count"`
+	ObjectSize  int    `json:"object_size"`
+}
+
+// getImageInfo queries rbd about the given image and returns its metadata, and returns
+// error if provided image is not found.
+func getImageInfo(f *framework.Framework, imageName, poolName string) (imageInfo, error) {
+	// rbd --format=json info [image-spec | snap-spec]
+	var imgInfo imageInfo
+
+	stdOut, stdErr, err := execCommandInToolBoxPod(
+		f,
+		fmt.Sprintf("rbd info %s %s --format json", rbdOptions(poolName), imageName),
+		rookNamespace)
+	if err != nil {
+		return imgInfo, fmt.Errorf("failed to get rbd info: %w", err)
+	}
+	if stdErr != "" {
+		return imgInfo, fmt.Errorf("failed to get rbd info: %v", stdErr)
+	}
+	err = json.Unmarshal([]byte(stdOut), &imgInfo)
+	if err != nil {
+		return imgInfo, fmt.Errorf("unmarshal failed: %w. raw buffer response: %s",
+			err, stdOut)
+	}
+
+	return imgInfo, nil
+}
+
+// validateStripe validate the stripe count, stripe unit and object size of the
+// image.
+func validateStripe(f *framework.Framework,
+	pvc *v1.PersistentVolumeClaim,
+	stripeUnit,
+	stripeCount,
+	objectSize int,
+) error {
+	imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
+	if err != nil {
+		return err
+	}
+
+	imgInfo, err := getImageInfo(f, imageData.imageName, defaultRBDPool)
+	if err != nil {
+		return err
+	}
+
+	if imgInfo.ObjectSize != objectSize {
+		return fmt.Errorf("objectSize %d does not match expected %d", imgInfo.ObjectSize, objectSize)
+	}
+
+	if imgInfo.StripeUnit != stripeUnit {
+		return fmt.Errorf("stripeUnit %d does not match expected %d", imgInfo.StripeUnit, stripeUnit)
+	}
+
+	if imgInfo.StripeCount != stripeCount {
+		return fmt.Errorf("stripeCount %d does not match expected %d", imgInfo.StripeCount, stripeCount)
+	}
+
+	return nil
+}

examples/rbd/storageclass.yaml

@@ -134,6 +134,14 @@ parameters:
    #       {"domainLabel":"zone","value":"zone1"}]}
    #   ]
 
+   # Image striping, Refer https://docs.ceph.com/en/latest/man/8/rbd/#striping
+   # For more details
+   # (optional) stripe unit in bytes.
+   # stripeUnit: <>
+   # (optional) objects to stripe over before looping.
+   # stripeCount: <>
+   # (optional) The object size in bytes.
+   # objectSize: <>
 reclaimPolicy: Delete
 allowVolumeExpansion: true
 mountOptions:

internal/rbd/controllerserver.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strconv"
 
 	csicommon "github.com/ceph/ceph-csi/internal/csi-common"
 	"github.com/ceph/ceph-csi/internal/util"
@@ -94,6 +95,43 @@ func (cs *ControllerServer) validateVolumeReq(ctx context.Context, req *csi.Crea
 		return err
 	}
 
+	err = validateStriping(req.Parameters)
+	if err != nil {
+		return status.Error(codes.InvalidArgument, err.Error())
+	}
+
+	return nil
+}
+
+func validateStriping(parameters map[string]string) error {
+	stripeUnit := parameters["stripeUnit"]
+	stripeCount := parameters["stripeCount"]
+	if stripeUnit != "" && stripeCount == "" {
+		return errors.New("stripeCount must be specified when stripeUnit is specified")
+	}
+
+	if stripeUnit == "" && stripeCount != "" {
+		return errors.New("stripeUnit must be specified when stripeCount is specified")
+	}
+
+	objectSize := parameters["objectSize"]
+	if objectSize != "" {
+		objSize, err := strconv.ParseUint(objectSize, 10, 64)
+		if err != nil {
+			return fmt.Errorf("failed to parse objectSize %s: %w", objectSize, err)
+		}
+		// check objectSize is power of 2
+		/*
+			Take 2^3=8 for example.
+			x & (x-1)
+			8 & 7
+			1000 & 0111 = 0000
+		*/
+		if objSize == 0 || (objSize&(objSize-1)) != 0 {
+			return fmt.Errorf("objectSize %s is not power of 2", objectSize)
+		}
+	}
+
 	return nil
 }
 

internal/rbd/controllerserver_test.go

@@ -0,0 +1,88 @@
+/*
+Copyright 2022 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import "testing"
+
+func TestValidateStriping(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name       string
+		parameters map[string]string
+		wantErr    bool
+	}{
+		{
+			name: "when stripeUnit is not specified",
+			parameters: map[string]string{
+				"stripeUnit":  "",
+				"stripeCount": "10",
+				"objectSize":  "2",
+			},
+			wantErr: true,
+		},
+		{
+			name: "when stripeCount is not specified",
+			parameters: map[string]string{
+				"stripeUnit":  "4096",
+				"stripeCount": "",
+				"objectSize":  "2",
+			},
+			wantErr: true,
+		},
+		{
+			name: "when objectSize is not power of 2",
+			parameters: map[string]string{
+				"stripeUnit":  "4096",
+				"stripeCount": "8",
+				"objectSize":  "3",
+			},
+			wantErr: true,
+		},
+		{
+			name: "when objectSize is 0",
+			parameters: map[string]string{
+				"stripeUnit":  "4096",
+				"stripeCount": "8",
+				"objectSize":  "0",
+			},
+			wantErr: true,
+		},
+		{
+			name: "when valid stripe parameters are specified",
+			parameters: map[string]string{
+				"stripeUnit":  "4096",
+				"stripeCount": "8",
+				"objectSize":  "131072",
+			},
+			wantErr: false,
+		},
+		{
+			name:       "when no stripe parameters are specified",
+			parameters: map[string]string{},
+			wantErr:    false,
+		},
+	}
+	for _, tt := range tests {
+		ts := tt
+		t.Run(ts.name, func(t *testing.T) {
+			t.Parallel()
+			if err := validateStriping(ts.parameters); (err != nil) != ts.wantErr {
+				t.Errorf("validateStriping() error = %v, wantErr %v", err, ts.wantErr)
+			}
+		})
+	}
+}