diff --git a/docs/rbd/deploy.md b/docs/rbd/deploy.md index f27f32df022..fd5bb3715c6 100644 --- a/docs/rbd/deploy.md +++ b/docs/rbd/deploy.md @@ -73,6 +73,15 @@ make image-cephcsi | `stripeUnit` | no | stripe unit in bytes | | `stripeCount` | no | objects to stripe over before looping | | `objectSize` | no | object size in bytes | +| `BaseReadIops` | no | the base limit of read operations per second | +| `BaseWriteIops` | no | the base limit of write operations per second | +| `BaseReadBytesPerSecond` | no | the base limit of read bytes per second | +| `BaseWriteBytesPerSecond` | no | the base limit of write bytes per second | +| `ReadIopsPerGB` | no | the limit of read operations per GiB | +| `WriteIopsPerGB` | no | the limit of write operations per GiB | +| `ReadBpsPerGB` | no | the limit of read bytes per GiB | +| `WriteBpsPerGB` | no | the limit of write bytes per GiB | +| `BaseVolSizeBytes` | no | the min size of volume what use to calc qos beased on capacity | | `extraDeploy` | no | array of extra objects to deploy with the release | **NOTE:** An accompanying CSI configuration file, needs to be provided to the diff --git a/e2e/rbd.go b/e2e/rbd.go index d47469a8b7b..6d7ac3480ad 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -4643,6 +4643,65 @@ var _ = Describe("RBD", func() { validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType) }) + By("validate rbd image qos", func() { + qosParameters := map[string]string{ + "BaseReadIops": "2000", + "BaseWriteIops": "1000", + "BaseReadBytesPerSecond": "209715200", + "baseWriteBytesPerSecond": "104857600", + } + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + framework.Failf("failed to delete storageclass: %v", err) + } + + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + qosParameters, + deletePolicy) + if err != nil { + framework.Failf("failed to create storageclass: %v", err) + } + defer func() { + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + framework.Failf("failed to delete storageclass: %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy) + if err != nil { + framework.Failf("failed to create storageclass: %v", err) + } + }() + + // create PVC + pvc, err := loadPVC(pvcPath) + if err != nil { + framework.Failf("failed to load PVC: %v", err) + } + + pvc.Namespace = f.UniqueName + + err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + framework.Failf("failed to create PVC and application: %v", err) + } + + // validate rbd image qos + err = validateQOS(f, pvc, qosParameters) + if err != nil { + framework.Failf("failed to validate qos: %v", err) + } + + // delete pvc + err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + framework.Failf("failed to delete PVC: %v", err) + } + }) + By("create a PVC and check PVC/PV metadata on RBD image after setmetadata is set to false", func() { err := createRBDSnapshotClass(f) if err != nil { diff --git a/e2e/rbd_helper.go b/e2e/rbd_helper.go index 786660722df..f593fee525d 100644 --- a/e2e/rbd_helper.go +++ b/e2e/rbd_helper.go @@ -1166,3 +1166,36 @@ func validateStripe(f *framework.Framework, return nil } + +func validateQOS(f *framework.Framework, + pvc *v1.PersistentVolumeClaim, + qosParameters map[string]string, +) error { + imageQosPair := map[string]string{ + "BaseReadIops": "rbd_qos_read_iops_limit", + "BaseWriteIops": "rbd_qos_write_iops_limit", + "BaseReadBytesPerSecond": "rbd_qos_read_bps_limit", + "BaseWriteBytesPerSecond": "rbd_qos_write_bps_limit", + } + metadataConfPrefix := "conf_" + + imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f) + if err != nil { + return err + } + + rbdImageSpec := imageSpec(defaultRBDPool, imageData.imageName) + for k, v := range qosParameters { + if r, ok := imageQosPair[k]; ok { + qosVal, err := getImageMeta(rbdImageSpec, metadataConfPrefix+r, f) + if err != nil { + return err + } + if qosVal != v { + return fmt.Errorf("%s: %s does not match expected %s", r, qosVal, v) + } + } + } + + return nil +} diff --git a/examples/rbd/storageclass.yaml b/examples/rbd/storageclass.yaml index 9d87b03ef05..8e01a3b3b2a 100644 --- a/examples/rbd/storageclass.yaml +++ b/examples/rbd/storageclass.yaml @@ -162,6 +162,38 @@ parameters: # stripeCount: <> # (optional) The object size in bytes. # objectSize: <> + + # rbd volume QoS. + # QoS provides settings for rbd volume read/write iops + # and read/write bandwidth. There are 4 base qos parameters + # among them, when users apply for a volume capacity equal + # to or less than BaseVolSizebytes, use base qos limit. + # For the portion of capacity exceeding BaseVolSizebytes, + # QoS will be increased in steps set per GiB. If the step + # size parameter per GiB is not provided, only base QoS limit + # will be used and not associated with capacity size. + # + # note: currently supports rbd-nbd mounter. + # + # For more details + # (optional) the base limit of read operations per second. + # BaseReadIops: <> + # (optional) the base limit of write operations per second. + # BaseWriteIops: <> + # (optional) the base limit of read bytes per second. + # BaseReadBytesPerSecond: <> + # (optional) the base limit of write bytes per second. + # BaseWriteBytesPerSecond: <> + # (optional) the limit of read operations per GiB. + # ReadIopsPerGB: <> + # (optional) the limit of write operations per GiB. + # WriteIopsPerGB: <> + # (optional) the limit of read bytes per GiB. + # ReadBpsPerGB: <> + # (optional) the limit of write bytes per GiB. + # WriteBpsPerGB: <> + # (optional) min size of volume what use to calc qos beased on capacity. + # BaseVolSizeBytes:<> reclaimPolicy: Delete allowVolumeExpansion: true diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go index 3ec067c3ecf..24de665fdf2 100644 --- a/internal/rbd/controllerserver.go +++ b/internal/rbd/controllerserver.go @@ -231,6 +231,12 @@ func (cs *ControllerServer) parseVolCreateRequest( return nil, status.Error(codes.InvalidArgument, err.Error()) } + // Get QosParameters from SC if qos configuration existing in SC + err = rbdVol.SetQOS(ctx, req.GetParameters()) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + err = rbdVol.Connect(cr) if err != nil { log.ErrorLog(ctx, "failed to connect to volume %v: %v", rbdVol.RbdImageName, err) @@ -424,6 +430,19 @@ func (cs *ControllerServer) CreateVolume( return nil, err } + // Apply Qos parameters to rbd image. + err = rbdVol.ApplyQOS(ctx) + if err != nil { + log.DebugLog(ctx, "failed apply QOS for rbd image: %v", err) + + return nil, status.Error(codes.Internal, err.Error()) + } + // Save Qos parameters from SC in Image medatate, we will use it while resize volume. + err = rbdVol.SaveQOS(ctx, req.GetParameters()) + if err != nil { + return nil, err + } + // Set Metadata on PV Create metadata := k8s.GetVolumeMetadata(req.GetParameters()) err = rbdVol.setAllMetadata(metadata) @@ -1605,6 +1624,13 @@ func (cs *ControllerServer) ControllerExpandVolume( if err != nil { log.ErrorLog(ctx, "failed to resize rbd image: %s with error: %v", rbdVol, err) + return nil, status.Error(codes.Internal, err.Error()) + } + // adjust rbd qos after resize volume. + err = rbdVol.AdjustQOS(ctx) + if err != nil { + log.DebugLog(ctx, "failed adjust QOS for rbd image") + return nil, status.Error(codes.Internal, err.Error()) } } diff --git a/internal/rbd/qos.go b/internal/rbd/qos.go new file mode 100644 index 00000000000..7ba69884825 --- /dev/null +++ b/internal/rbd/qos.go @@ -0,0 +1,299 @@ +/* +Copyright 2024 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rbd + +import ( + "context" + "errors" + "strconv" + + "github.com/ceph/ceph-csi/internal/util/log" + + librbd "github.com/ceph/go-ceph/rbd" +) + +const ( + // Qos parameters name of StorageClass. + baseReadIops = "BaseReadIops" + baseWriteIops = "BaseWriteIops" + baseReadBytesPerSecond = "BaseReadBytesPerSecond" + baseWriteBytesPerSecond = "BaseWriteBytesPerSecond" + readIopsPerGB = "ReadIopsPerGB" + writeIopsPerGB = "WriteIopsPerGB" + readBpsPerGB = "ReadBpsPerGB" + writeBpsPerGB = "WriteBpsPerGB" + baseVolSizeBytes = "BaseVolSizeBytes" + + // Qos type name of rbd image. + readIopsLimit = "rbd_qos_read_iops_limit" + writeIopsLimit = "rbd_qos_write_iops_limit" + readBpsLimit = "rbd_qos_read_bps_limit" + writeBpsLimit = "rbd_qos_write_bps_limit" + metadataConfPrefix = "conf_" + + // The params use to calc qos based on capacity. + baseQosReadIopsLimit = "rbd_base_qos_read_iops_limit" + baseQosWriteIopsLimit = "rbd_base_qos_write_iops_limit" + baseQosReadBpsLimit = "rbd_base_qos_read_bps_limit" + baseQosWriteBpsLimit = "rbd_base_qos_write_bps_limit" + readIopsPerGBLimit = "rbd_read_iops_per_gb_limit" + writeIopsPerGBLimit = "rbd_write_iops_per_gb_limit" + readBpsPerGBLimit = "rbd_read_bps_per_gb_limit" + writeBpsPerGBLimit = "rbd_write_bps_per_gb_limit" + baseQosVolSize = "rbd_base_qos_vol_size" +) + +type qosSpec struct { + baseLimitType string + baseLimit string + perGBLimitType string + perGBLimit string + provide bool +} + +func parseQosParams( + params map[string]string, +) map[string]*qosSpec { + rbdQosParameters := map[string]*qosSpec{ + baseReadIops: {readIopsLimit, "", readIopsPerGB, "", false}, + baseWriteIops: {writeIopsLimit, "", writeIopsPerGB, "", false}, + baseReadBytesPerSecond: {readBpsLimit, "", readBpsPerGB, "", false}, + baseWriteBytesPerSecond: {writeBpsLimit, "", writeBpsPerGB, "", false}, + } + for k, v := range params { + if qos, ok := rbdQosParameters[k]; ok && v != "" { + qos.baseLimit = v + qos.provide = true + for p, q := range params { + if p == qos.perGBLimitType { + if q != "" { + qos.perGBLimit = q + } + } + } + } + } + + return rbdQosParameters +} + +func (rv *rbdVolume) SetQOS( + ctx context.Context, + params map[string]string, +) error { + baseVolSize := "" + if v, ok := params[baseVolSizeBytes]; ok && v != "" { + baseVolSize = v + } + + rbdQosParameters := parseQosParams(params) + for _, qos := range rbdQosParameters { + if qos.provide { + err := calcQosBasedOnCapacity(ctx, rv, *qos, baseVolSize) + if err != nil { + return err + } + } + } + + return nil +} + +func (rv *rbdVolume) ApplyQOS( + ctx context.Context, +) error { + if len(rv.QosParameters) != 0 { + err := setRbdImageQos(ctx, rv) + if err != nil { + return err + } + } + + return nil +} + +func calcQosBasedOnCapacity( + ctx context.Context, + rbdVol *rbdVolume, + qos qosSpec, + baseVolSize string, +) error { + if rbdVol.QosParameters == nil { + rbdVol.QosParameters = make(map[string]string) + } + + // Don't set qos if base qos limit empty. + if qos.baseLimit == "" { + return nil + } + baseLimit, err := strconv.ParseInt(qos.baseLimit, 10, 64) + if err != nil { + log.ErrorLog(ctx, "%v", err) + + return err + } + + // if provide qosPerGB and baseVolSize, we will set qos based on capacity, + // otherwise, we only set base qos limit. + if qos.perGBLimit != "" && baseVolSize != "" { + perGBLimit, err := strconv.ParseInt(qos.perGBLimit, 10, 64) + if err != nil { + log.ErrorLog(ctx, "%v", err) + + return err + } + + baseVolSizeInt, err := strconv.ParseInt(baseVolSize, 10, 64) + if err != nil { + log.ErrorLog(ctx, "%v", err) + + return err + } + + if rbdVol.VolSize <= baseVolSizeInt { + rbdVol.QosParameters[qos.baseLimitType] = qos.baseLimit + } else { + capacityQos := (rbdVol.VolSize - baseVolSizeInt) / int64(oneGB) * perGBLimit + finalQosLimit := baseLimit + capacityQos + rbdVol.QosParameters[qos.baseLimitType] = strconv.FormatInt(finalQosLimit, 10) + } + } else { + rbdVol.QosParameters[qos.baseLimitType] = qos.baseLimit + } + + return nil +} + +func setRbdImageQos( + ctx context.Context, + rbdVol *rbdVolume, +) error { + for k, v := range rbdVol.QosParameters { + err := rbdVol.SetMetadata(metadataConfPrefix+k, v) + if err != nil { + log.ErrorLog(ctx, "failed to set rbd qos, %s: %s, %v", k, v, err) + + return err + } + } + + return nil +} + +func (rv *rbdVolume) SaveQOS( + ctx context.Context, + params map[string]string, +) error { + needSaveQosParameters := map[string]string{ + baseReadIops: baseQosReadIopsLimit, + baseWriteIops: baseQosWriteIopsLimit, + baseReadBytesPerSecond: baseQosReadBpsLimit, + baseWriteBytesPerSecond: baseQosWriteBpsLimit, + readIopsPerGB: readIopsPerGBLimit, + writeIopsPerGB: writeIopsPerGBLimit, + readBpsPerGB: readBpsPerGBLimit, + writeBpsPerGB: writeBpsPerGBLimit, + baseVolSizeBytes: baseQosVolSize, + } + for k, v := range params { + if param, ok := needSaveQosParameters[k]; ok { + if v != "" { + err := rv.SetMetadata(param, v) + if err != nil { + log.ErrorLog(ctx, "failed to save metadata, %s: %s, %v", k, v, err) + + return err + } + } + } + } + + return nil +} + +func getRbdImageQos( + ctx context.Context, + rbdVol *rbdVolume, +) (map[string]qosSpec, string, error) { + QosParams := map[string]struct { + rbdQosType string + rbdQosPerGBType string + }{ + baseQosReadIopsLimit: {readIopsLimit, readIopsPerGBLimit}, + baseQosWriteIopsLimit: {writeIopsLimit, writeIopsPerGBLimit}, + baseQosReadBpsLimit: {readBpsLimit, readBpsPerGBLimit}, + baseQosWriteBpsLimit: {writeBpsLimit, writeBpsPerGBLimit}, + } + rbdQosParameters := make(map[string]qosSpec) + for k, param := range QosParams { + baseLimit, err := rbdVol.GetMetadata(k) + if errors.Is(err, librbd.ErrNotFound) { + // if base qos dose not exist, skipping. + continue + } else if err != nil { + log.ErrorLog(ctx, "failed to get metadata: %v", err) + + return nil, "", err + } + perGBLimit, err := rbdVol.GetMetadata(param.rbdQosPerGBType) + if errors.Is(err, librbd.ErrNotFound) { + // rbdQosPerGBType does not exist, set it empty. + perGBLimit = "" + } else if err != nil { + log.ErrorLog(ctx, "failed to get metadata: %v", err) + + return nil, "", err + } + rbdQosParameters[k] = qosSpec{param.rbdQosType, baseLimit, param.rbdQosPerGBType, perGBLimit, true} + } + baseVolSize, err := rbdVol.GetMetadata(baseQosVolSize) + if errors.Is(err, librbd.ErrNotFound) { + // rbdBaseQosVolSize does not exist, set it empty. + baseVolSize = "" + } else if err != nil { + log.ErrorLog(ctx, "failed to get metadata: %v", err) + + return nil, "", err + } + + return rbdQosParameters, baseVolSize, nil +} + +func (rv *rbdVolume) AdjustQOS( + ctx context.Context, +) error { + rbdQosParameters, baseQosVolSize, err := getRbdImageQos(ctx, rv) + if err != nil { + log.ErrorLog(ctx, "get image metadata failed: %v", err) + + return err + } + for _, param := range rbdQosParameters { + err = calcQosBasedOnCapacity(ctx, rv, param, baseQosVolSize) + if err != nil { + return err + } + } + err = setRbdImageQos(ctx, rv) + if err != nil { + log.ErrorLog(ctx, "set image metadata failed: %v", err) + + return err + } + + return nil +} diff --git a/internal/rbd/qos_test.go b/internal/rbd/qos_test.go new file mode 100644 index 00000000000..3aad0870302 --- /dev/null +++ b/internal/rbd/qos_test.go @@ -0,0 +1,64 @@ +/* +Copyright 2024 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rbd + +import ( + "context" + "testing" +) + +func TestSetQOS(t *testing.T) { + t.Parallel() + ctx := context.TODO() + + tests := map[string]string{ + baseReadIops: "2000", + baseWriteIops: "1000", + baseReadBytesPerSecond: "209715200", + baseWriteBytesPerSecond: "104857600", + } + + rv := rbdVolume{} + + err := rv.SetQOS(ctx, tests) + if err != nil { + t.Errorf("SetQOS failed: %v", err) + } + if len(rv.QosParameters) != len(tests) { + t.Errorf("SetQOS: rbdVolume.QosParameters size = %d, want %d", len(rv.QosParameters), len(tests)) + } + for k, v := range rv.QosParameters { + switch k { + case readIopsLimit: + if v != tests[baseReadIops] { + t.Errorf("SetQOS: %s: %s, want %s", k, v, tests[baseReadIops]) + } + case writeIopsLimit: + if v != tests[baseReadIops] { + t.Errorf("SetQOS: %s: %s, want %s", k, v, tests[baseReadIops]) + } + case readBpsLimit: + if v != tests[baseReadBytesPerSecond] { + t.Errorf("SetQOS: %s: %s, want %s", k, v, tests[baseReadBytesPerSecond]) + } + case writeBpsLimit: + if v != tests[baseWriteBytesPerSecond] { + t.Errorf("SetQOS: %s: %s, want %s", k, v, tests[baseWriteBytesPerSecond]) + } + } + } +} diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index fbae7f4181d..ad1ca9623ba 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -148,6 +148,9 @@ type rbdImage struct { EnableMetadata bool // ParentInTrash indicates the parent image is in trash. ParentInTrash bool + + // RBD QoS configuration + QosParameters map[string]string } // check that rbdVolume implements the types.Volume interface.