diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go index 3ec067c3ecf..384b839826d 100644 --- a/internal/rbd/controllerserver.go +++ b/internal/rbd/controllerserver.go @@ -231,6 +231,22 @@ func (cs *ControllerServer) parseVolCreateRequest( return nil, status.Error(codes.InvalidArgument, err.Error()) } + // Get QosParameters from SC if qos configuration existing in SC + baseQosVolSize := "" + params := req.GetParameters() + if v, ok := params[baseVolSizeBytes]; ok && v != "" { + baseQosVolSize = v + } + rbdQosParameters := parseQosParams(params) + for _, param := range rbdQosParameters { + if param.provide { + err := calcQosBasedOnCapacity(ctx, rbdVol, param.rbdQosType, param.rbdBaseQosLimit, param.rbdQosPerGB, baseQosVolSize) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + } + } + err = rbdVol.Connect(cr) if err != nil { log.ErrorLog(ctx, "failed to connect to volume %v: %v", rbdVol.RbdImageName, err) @@ -243,6 +259,32 @@ func (cs *ControllerServer) parseVolCreateRequest( return rbdVol, nil } +func parseQosParams( + params map[string]string, +) (map[string]*rbdQos) { + rbdQosParameters := map[string]*rbdQos{ + baseReadIOPS: {rbdQosReadIopsLimit, "", readIopsPerGB, "", false}, + baseWriteIOPS: {rbdQosWriteIopsLimit, "", writeIopsPerGB, "", false}, + baseReadBytesPerSecond: {rbdQosReadBpsLimit, "", readBpsPerGB, "", false}, + baseWriteBytesPerSecond: {rbdQosWriteBpsLimit, "", writeBpsPerGB, "", false}, + } + for k, v := range params { + if param, ok := rbdQosParameters[k]; ok && v != "" { + param.rbdBaseQosLimit = v + param.provide = true + for p, q := range params { + if p == param.rbdQosPerGBType { + if q != "" { + param.rbdQosPerGB = q + } + } + } + } + } + + return rbdQosParameters +} + func (rbdVol *rbdVolume) ToCSI(ctx context.Context) (*csi.Volume, error) { vol := &csi.Volume{ VolumeId: rbdVol.VolID, @@ -424,6 +466,19 @@ func (cs *ControllerServer) CreateVolume( return nil, err } + // Set rbd qos for image + if len(rbdVol.QosParameters) != 0 && rbdVol.Mounter == rbdNbdMounter { + err = setRbdImageQos(ctx, rbdVol) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + } + // Save qos parameters from SC in image medatate, we will use it while resize volume + err = saveQosToRbdImage(ctx, rbdVol, req.GetParameters()) + if err != nil { + return nil, err + } + // Set Metadata on PV Create metadata := k8s.GetVolumeMetadata(req.GetParameters()) err = rbdVol.setAllMetadata(metadata) @@ -1607,6 +1662,11 @@ func (cs *ControllerServer) ControllerExpandVolume( return nil, status.Error(codes.Internal, err.Error()) } + // we need adjust rbd qos after resize volume + err = cs.adjustRbdImageQos(ctx, rbdVol) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } } return &csi.ControllerExpandVolumeResponse{ @@ -1615,6 +1675,34 @@ func (cs *ControllerServer) ControllerExpandVolume( }, nil } +func (cs *ControllerServer) adjustRbdImageQos( + ctx context.Context, + rbdVol *rbdVolume, +) error { + rbdQosParameters, baseQosVolSize, err := getRbdImageQos(ctx, rbdVol) + if err != nil { + log.ErrorLog(ctx, "get image metadata failed: %v", err) + + return err + } + for _, param := range rbdQosParameters { + err = calcQosBasedOnCapacity(ctx, rbdVol, param.rbdQosType, param.rbdBaseQosLimit, param.rbdQosPerGB, baseQosVolSize) + if err != nil { + + return err + } + } + err = setRbdImageQos(ctx, rbdVol) + if err != nil { + log.ErrorLog(ctx, "set image metadata failed: %v", err) + + return err + } + log.DebugLog(ctx, "adjust rbd image qos successfully") + + return nil +} + // ControllerPublishVolume is a dummy publish implementation to mimic a successful attach operation being a NOOP. func (cs *ControllerServer) ControllerPublishVolume( ctx context.Context, diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index fbae7f4181d..224e9016e32 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -81,6 +81,35 @@ const ( // clusterNameKey cluster Key, set on RBD image. clusterNameKey = "csi.ceph.com/cluster/name" + + // Qos parameters name of StorageClass. + baseReadIOPS = "BaseReadIOPS" + baseWriteIOPS = "BaseWriteIOPS" + baseReadBytesPerSecond = "BaseReadBytesPerSecond" + baseWriteBytesPerSecond = "BaseWriteBytesPerSecond" + readIopsPerGB = "ReadIopsPerGB" + writeIopsPerGB = "WriteIopsPerGB" + readBpsPerGB = "ReadBpsPerGB" + writeBpsPerGB = "WriteBpsPerGB" + baseVolSizeBytes = "BaseVolSizeBytes" + + // Qos type name of rbd image. + rbdQosReadIopsLimit = "rbd_qos_read_iops_limit" + rbdQosWriteIopsLimit = "rbd_qos_write_iops_limit" + rbdQosReadBpsLimit = "rbd_qos_read_bps_limit" + rbdQosWriteBpsLimit = "rbd_qos_write_bps_limit" + metadataConfPrefix = "conf_" + + // The params use to calc qos based on capacity. + rbdBaseQosReadIopsLimit = "rbd_base_qos_read_iops_limit" + rbdBaseQosWriteIopsLimit = "rbd_base_qos_write_iops_limit" + rbdBaseQosReadBpsLimit = "rbd_base_qos_read_bps_limit" + rbdBaseQosWriteBpsLimit = "rbd_base_qos_write_bps_limit" + rbdReadIopsPerGB = "rbd_read_iops_per_gb" + rbdWriteIopsPerGB = "rbd_write_iops_per_gb" + rbdReadBpsPerGB = "rbd_read_bps_per_gb" + rbdWriteBpsPerGB = "rbd_write_bps_per_gb" + rbdBaseQosVolSize = "rbd_base_qos_vol_size" ) // rbdImage contains common attributes and methods for the rbdVolume and @@ -148,6 +177,9 @@ type rbdImage struct { EnableMetadata bool // ParentInTrash indicates the parent image is in trash. ParentInTrash bool + + // RBD QoS configuration + QosParameters map[string]string } // check that rbdVolume implements the types.Volume interface. @@ -212,6 +244,14 @@ type migrationVolID struct { clusterID string } +type rbdQos struct { + rbdQosType string + rbdBaseQosLimit string + rbdQosPerGBType string + rbdQosPerGB string + provide bool +} + var ( supportedFeatures = map[string]imageFeature{ librbd.FeatureNameLayering: { @@ -2273,3 +2313,147 @@ func (ri *rbdImage) GetClusterID(ctx context.Context) (string, error) { return ri.ClusterID, nil } + +func calcQosBasedOnCapacity( + ctx context.Context, + rbdVol *rbdVolume, + qosType string, + baseQosLimit string, + qosPerGB string, + baseQosVolSize string, +) error { + if rbdVol.QosParameters == nil { + rbdVol.QosParameters = make(map[string]string) + } + + // Don't set qos if base limit empty + if baseQosLimit == "" { + return nil + } + baseQosLimitInt, err := strconv.ParseInt(baseQosLimit, 10, 64) + if err != nil { + log.ErrorLog(ctx, "%v", err) + + return err + } + + // if provide qosPerGB and baseQosVolSize, we will set qos based on capacity, + // otherwise, we only set base qos limit. + if qosPerGB != "" && baseQosVolSize != "" { + qosPerGBInt, err := strconv.ParseInt(qosPerGB, 10, 64) + if err != nil { + log.ErrorLog(ctx, "%v", err) + return err + } + + baseQosVolSizeInt, err := strconv.ParseInt(baseQosVolSize, 10, 64) + if err != nil { + log.ErrorLog(ctx, "%v", err) + return err + } + + if rbdVol.VolSize <= baseQosVolSizeInt { + rbdVol.QosParameters[qosType] = baseQosLimit + } else { + capacityQos := (rbdVol.VolSize - baseQosVolSizeInt) / int64(oneGB) * qosPerGBInt + v := baseQosLimitInt + capacityQos + rbdVol.QosParameters[qosType] = strconv.FormatInt(v, 10) + } + } else { + rbdVol.QosParameters[qosType] = baseQosLimit + } + + return nil +} + +func setRbdImageQos( + ctx context.Context, + rbdVol *rbdVolume, +) error { + for k, v := range rbdVol.QosParameters { + err := rbdVol.SetMetadata(metadataConfPrefix+k, v) + if err != nil { + log.ErrorLog(ctx, "failed to set rbd qos, %s: %s, %v", k, v, err) + return err + } + } + + return nil +} + +func getRbdImageQos( + ctx context.Context, + rbdVol *rbdVolume, +) (map[string]rbdQos, string, error) { + QosParams := map[string]struct { + rbdQosType string + rbdQosPerGBType string + }{ + rbdBaseQosReadIopsLimit: {rbdQosReadIopsLimit, rbdReadIopsPerGB}, + rbdBaseQosWriteIopsLimit: {rbdQosWriteIopsLimit, rbdWriteIopsPerGB}, + rbdBaseQosReadBpsLimit: {rbdQosReadBpsLimit, rbdReadBpsPerGB}, + rbdBaseQosWriteBpsLimit: {rbdQosWriteBpsLimit, rbdWriteBpsPerGB}, + } + rbdQosParameters := make(map[string]rbdQos) + var err error + for k, param := range QosParams { + baseLimit, err := rbdVol.GetMetadata(k) + if errors.Is(err, librbd.ErrNotFound) { + // if base qos dose not exist, skipping. + continue + } else if err != nil { + log.ErrorLog(ctx, "failed to get metadata: %v", err) + return nil, "", err + } + perGBLimit, err := rbdVol.GetMetadata(param.rbdQosPerGBType) + if errors.Is(err, librbd.ErrNotFound) { + // rbdQosPerGBType does not exist, set it empty. + perGBLimit = "" + } else if err != nil { + log.ErrorLog(ctx, "failed to get metadata: %v", err) + return nil, "", err + } + rbdQosParameters[k] = rbdQos{param.rbdQosType, baseLimit, param.rbdQosPerGBType, perGBLimit, true} + } + baseQosVolSize, err := rbdVol.GetMetadata(rbdBaseQosVolSize) + if errors.Is(err, librbd.ErrNotFound) { + // rbdBaseQosVolSize does not exist, set it empty. + baseQosVolSize = "" + } else if err != nil { + log.ErrorLog(ctx, "failed to get metadata: %v", err) + return nil, "", err + } + + return rbdQosParameters, baseQosVolSize, nil +} + +func saveQosToRbdImage( + ctx context.Context, + rbdVol *rbdVolume, + params map[string]string, +) error { + needSaveQosParameters := map[string]string{ + baseReadIOPS: rbdBaseQosReadIopsLimit, + baseWriteIOPS: rbdBaseQosWriteIopsLimit, + baseReadBytesPerSecond: rbdBaseQosReadBpsLimit, + baseWriteBytesPerSecond: rbdBaseQosWriteBpsLimit, + readIopsPerGB: rbdReadIopsPerGB, + writeIopsPerGB: rbdWriteIopsPerGB, + readBpsPerGB: rbdReadBpsPerGB, + writeBpsPerGB: rbdWriteBpsPerGB, + baseVolSizeBytes: rbdBaseQosVolSize, + } + for k, v := range params { + if param, ok := needSaveQosParameters[k]; ok { + if v != "" { + err := rbdVol.SetMetadata(param, v) + if err != nil { + log.ErrorLog(ctx, "failed to save metadata, %s: %s, %v", k, v, err) + return err + } + } + } + } + + return nil +}