Skip to content

Commit

Permalink
rbd: support QoS based on capacity for rbd volume
Browse files Browse the repository at this point in the history 
1. QoS provides settings for rbd volume read/write iops
   and read/write bandwidth.
2. All QoS parameters are placed in the SC,
   send QoS parameters from SC to Cephcsi through PVC create request.
3. We need provide QoS parameters in the SC as below:
   - BaseReadIOPS
   - BaseWriteIOPS
   - BaseReadBytesPerSecond
   - BaseWriteBytesPerSecond
   - ReadIopsPerGB
   - WriteIopsPerGB
   - ReadBpsPerGB
   - WriteBpsPerGB
   - BaseVolSizeBytes
   There are 4 base qos parameters among them, when users apply for
   a volume capacity equal to or less than BaseVolSizebytes, use base
   qos limit. For the portion of capacity exceeding BaseVolSizebytes,
   QoS will be increased in steps set per GB. If the step size parameter
   per GB is not provided, only base QoS limit will be used and not associated
   with capacity size.
4. If PVC has resize request, adjust the QoS limit
   according to the QoS parameters after resizing.

Signed-off-by: Yite Gu <guyite@bytedance.com>
  • Loading branch information
@YiteGu
YiteGu committed Dec 12, 2024
1 parent 3c63fea commit c07a41a
Show file tree
Hide file tree
Showing 2 changed files with 272 additions and 0 deletions.
88 changes: 88 additions & 0 deletions internal/rbd/controllerserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,22 @@ func (cs *ControllerServer) parseVolCreateRequest(
return nil, status.Error(codes.InvalidArgument, err.Error())
}

// Get QosParameters from SC if qos configuration existing in SC
baseQosVolSize := ""
params := req.GetParameters()
if v, ok := params[baseVolSizeBytes]; ok && v != "" {
baseQosVolSize = v
}
rbdQosParameters := parseQosParams(params)
for _, param := range rbdQosParameters {
if param.provide {
err := calcQosBasedOnCapacity(ctx, rbdVol, param.rbdQosType, param.rbdBaseQosLimit, param.rbdQosPerGB, baseQosVolSize)
if err != nil {
return nil, status.Error(codes.InvalidArgument, err.Error())
}
}
}

err = rbdVol.Connect(cr)
if err != nil {
log.ErrorLog(ctx, "failed to connect to volume %v: %v", rbdVol.RbdImageName, err)
Expand All @@ -243,6 +259,32 @@ func (cs *ControllerServer) parseVolCreateRequest(
return rbdVol, nil
}

func parseQosParams(
params map[string]string,
) (map[string]*rbdQos) {
rbdQosParameters := map[string]*rbdQos{
baseReadIOPS: {rbdQosReadIopsLimit, "", readIopsPerGB, "", false},
baseWriteIOPS: {rbdQosWriteIopsLimit, "", writeIopsPerGB, "", false},
baseReadBytesPerSecond: {rbdQosReadBpsLimit, "", readBpsPerGB, "", false},
baseWriteBytesPerSecond: {rbdQosWriteBpsLimit, "", writeBpsPerGB, "", false},
}
for k, v := range params {
if param, ok := rbdQosParameters[k]; ok && v != "" {
param.rbdBaseQosLimit = v
param.provide = true
for p, q := range params {
if p == param.rbdQosPerGBType {
if q != "" {
param.rbdQosPerGB = q
}
}
}
}
}

return rbdQosParameters
}

func (rbdVol *rbdVolume) ToCSI(ctx context.Context) (*csi.Volume, error) {
vol := &csi.Volume{
VolumeId: rbdVol.VolID,
Expand Down Expand Up @@ -424,6 +466,19 @@ func (cs *ControllerServer) CreateVolume(
return nil, err
}

// Set rbd qos for image
if len(rbdVol.QosParameters) != 0 && rbdVol.Mounter == rbdNbdMounter {
err = setRbdImageQos(ctx, rbdVol)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
}
// Save qos parameters from SC in image medatate, we will use it while resize volume
err = saveQosToRbdImage(ctx, rbdVol, req.GetParameters())
if err != nil {
return nil, err
}

// Set Metadata on PV Create
metadata := k8s.GetVolumeMetadata(req.GetParameters())
err = rbdVol.setAllMetadata(metadata)
Expand Down Expand Up @@ -1607,6 +1662,11 @@ func (cs *ControllerServer) ControllerExpandVolume(

return nil, status.Error(codes.Internal, err.Error())
}
// we need adjust rbd qos after resize volume
err = cs.adjustRbdImageQos(ctx, rbdVol)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
}

return &csi.ControllerExpandVolumeResponse{
Expand All @@ -1615,6 +1675,34 @@ func (cs *ControllerServer) ControllerExpandVolume(
}, nil
}

func (cs *ControllerServer) adjustRbdImageQos(
ctx context.Context,
rbdVol *rbdVolume,
) error {
rbdQosParameters, baseQosVolSize, err := getRbdImageQos(ctx, rbdVol)
if err != nil {
log.ErrorLog(ctx, "get image metadata failed: %v", err)

return err
}
for _, param := range rbdQosParameters {
err = calcQosBasedOnCapacity(ctx, rbdVol, param.rbdQosType, param.rbdBaseQosLimit, param.rbdQosPerGB, baseQosVolSize)
if err != nil {

return err
}
}
err = setRbdImageQos(ctx, rbdVol)
if err != nil {
log.ErrorLog(ctx, "set image metadata failed: %v", err)

return err
}
log.DebugLog(ctx, "adjust rbd image qos successfully")

return nil
}

// ControllerPublishVolume is a dummy publish implementation to mimic a successful attach operation being a NOOP.
func (cs *ControllerServer) ControllerPublishVolume(
ctx context.Context,
Expand Down
184 changes: 184 additions & 0 deletions internal/rbd/rbd_util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,35 @@ const (

// clusterNameKey cluster Key, set on RBD image.
clusterNameKey = "csi.ceph.com/cluster/name"

// Qos parameters name of StorageClass.
baseReadIOPS = "BaseReadIOPS"
baseWriteIOPS = "BaseWriteIOPS"
baseReadBytesPerSecond = "BaseReadBytesPerSecond"
baseWriteBytesPerSecond = "BaseWriteBytesPerSecond"
readIopsPerGB = "ReadIopsPerGB"
writeIopsPerGB = "WriteIopsPerGB"
readBpsPerGB = "ReadBpsPerGB"
writeBpsPerGB = "WriteBpsPerGB"
baseVolSizeBytes = "BaseVolSizeBytes"

// Qos type name of rbd image.
rbdQosReadIopsLimit = "rbd_qos_read_iops_limit"
rbdQosWriteIopsLimit = "rbd_qos_write_iops_limit"
rbdQosReadBpsLimit = "rbd_qos_read_bps_limit"
rbdQosWriteBpsLimit = "rbd_qos_write_bps_limit"
metadataConfPrefix = "conf_"

// The params use to calc qos based on capacity.
rbdBaseQosReadIopsLimit = "rbd_base_qos_read_iops_limit"
rbdBaseQosWriteIopsLimit = "rbd_base_qos_write_iops_limit"
rbdBaseQosReadBpsLimit = "rbd_base_qos_read_bps_limit"
rbdBaseQosWriteBpsLimit = "rbd_base_qos_write_bps_limit"
rbdReadIopsPerGB = "rbd_read_iops_per_gb"
rbdWriteIopsPerGB = "rbd_write_iops_per_gb"
rbdReadBpsPerGB = "rbd_read_bps_per_gb"
rbdWriteBpsPerGB = "rbd_write_bps_per_gb"
rbdBaseQosVolSize = "rbd_base_qos_vol_size"
)

// rbdImage contains common attributes and methods for the rbdVolume and
Expand Down Expand Up @@ -148,6 +177,9 @@ type rbdImage struct {
EnableMetadata bool
// ParentInTrash indicates the parent image is in trash.
ParentInTrash bool

// RBD QoS configuration
QosParameters map[string]string
}

// check that rbdVolume implements the types.Volume interface.
Expand Down Expand Up @@ -212,6 +244,14 @@ type migrationVolID struct {
clusterID string
}

type rbdQos struct {
rbdQosType string
rbdBaseQosLimit string
rbdQosPerGBType string
rbdQosPerGB string
provide bool
}

var (
supportedFeatures = map[string]imageFeature{
librbd.FeatureNameLayering: {
Expand Down Expand Up @@ -2273,3 +2313,147 @@ func (ri *rbdImage) GetClusterID(ctx context.Context) (string, error) {

return ri.ClusterID, nil
}

func calcQosBasedOnCapacity(
ctx context.Context,
rbdVol *rbdVolume,
qosType string,
baseQosLimit string,
qosPerGB string,
baseQosVolSize string,
) error {
if rbdVol.QosParameters == nil {
rbdVol.QosParameters = make(map[string]string)
}

// Don't set qos if base limit empty
if baseQosLimit == "" {
return nil
}
baseQosLimitInt, err := strconv.ParseInt(baseQosLimit, 10, 64)
if err != nil {
log.ErrorLog(ctx, "%v", err)

return err
}

// if provide qosPerGB and baseQosVolSize, we will set qos based on capacity,
// otherwise, we only set base qos limit.
if qosPerGB != "" && baseQosVolSize != "" {
qosPerGBInt, err := strconv.ParseInt(qosPerGB, 10, 64)
if err != nil {
log.ErrorLog(ctx, "%v", err)
return err
}

baseQosVolSizeInt, err := strconv.ParseInt(baseQosVolSize, 10, 64)
if err != nil {
log.ErrorLog(ctx, "%v", err)
return err
}

if rbdVol.VolSize <= baseQosVolSizeInt {
rbdVol.QosParameters[qosType] = baseQosLimit
} else {
capacityQos := (rbdVol.VolSize - baseQosVolSizeInt) / int64(oneGB) * qosPerGBInt
v := baseQosLimitInt + capacityQos
rbdVol.QosParameters[qosType] = strconv.FormatInt(v, 10)
}
} else {
rbdVol.QosParameters[qosType] = baseQosLimit
}

return nil
}

func setRbdImageQos(
ctx context.Context,
rbdVol *rbdVolume,
) error {
for k, v := range rbdVol.QosParameters {
err := rbdVol.SetMetadata(metadataConfPrefix+k, v)
if err != nil {
log.ErrorLog(ctx, "failed to set rbd qos, %s: %s, %v", k, v, err)
return err
}
}

return nil
}

func getRbdImageQos(
ctx context.Context,
rbdVol *rbdVolume,
) (map[string]rbdQos, string, error) {
QosParams := map[string]struct {
rbdQosType string
rbdQosPerGBType string
}{
rbdBaseQosReadIopsLimit: {rbdQosReadIopsLimit, rbdReadIopsPerGB},
rbdBaseQosWriteIopsLimit: {rbdQosWriteIopsLimit, rbdWriteIopsPerGB},
rbdBaseQosReadBpsLimit: {rbdQosReadBpsLimit, rbdReadBpsPerGB},
rbdBaseQosWriteBpsLimit: {rbdQosWriteBpsLimit, rbdWriteBpsPerGB},
}
rbdQosParameters := make(map[string]rbdQos)
var err error
for k, param := range QosParams {
baseLimit, err := rbdVol.GetMetadata(k)
if errors.Is(err, librbd.ErrNotFound) {
// if base qos dose not exist, skipping.
continue
} else if err != nil {
log.ErrorLog(ctx, "failed to get metadata: %v", err)
return nil, "", err
}
perGBLimit, err := rbdVol.GetMetadata(param.rbdQosPerGBType)
if errors.Is(err, librbd.ErrNotFound) {
// rbdQosPerGBType does not exist, set it empty.
perGBLimit = ""
} else if err != nil {
log.ErrorLog(ctx, "failed to get metadata: %v", err)
return nil, "", err
}
rbdQosParameters[k] = rbdQos{param.rbdQosType, baseLimit, param.rbdQosPerGBType, perGBLimit, true}
}
baseQosVolSize, err := rbdVol.GetMetadata(rbdBaseQosVolSize)
if errors.Is(err, librbd.ErrNotFound) {
// rbdBaseQosVolSize does not exist, set it empty.
baseQosVolSize = ""
} else if err != nil {
log.ErrorLog(ctx, "failed to get metadata: %v", err)
return nil, "", err
}

return rbdQosParameters, baseQosVolSize, nil
}

func saveQosToRbdImage(
ctx context.Context,
rbdVol *rbdVolume,
params map[string]string,
) error {
needSaveQosParameters := map[string]string{
baseReadIOPS: rbdBaseQosReadIopsLimit,
baseWriteIOPS: rbdBaseQosWriteIopsLimit,
baseReadBytesPerSecond: rbdBaseQosReadBpsLimit,
baseWriteBytesPerSecond: rbdBaseQosWriteBpsLimit,
readIopsPerGB: rbdReadIopsPerGB,
writeIopsPerGB: rbdWriteIopsPerGB,
readBpsPerGB: rbdReadBpsPerGB,
writeBpsPerGB: rbdWriteBpsPerGB,
baseVolSizeBytes: rbdBaseQosVolSize,
}
for k, v := range params {
if param, ok := needSaveQosParameters[k]; ok {
if v != "" {
err := rbdVol.SetMetadata(param, v)
if err != nil {
log.ErrorLog(ctx, "failed to save metadata, %s: %s, %v", k, v, err)
return err
}
}
}
}

return nil
}

0 comments on commit c07a41a

Please sign in to comment.