NFS tree quota support

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.2.3-0.20230517135918-9920e636bff1
 	github.com/dell/gocsi v1.7.0
 	github.com/dell/gofsutil v1.12.0
-	github.com/dell/goscaleio v1.11.1-0.20230724113841-f10386ad0bb4
+	github.com/dell/goscaleio v1.11.1-0.20230724122254-8f8b41ad4aad
 	github.com/fsnotify/fsnotify v1.5.1
 	github.com/golang/protobuf v1.5.3
 	github.com/google/uuid v1.3.0

go.sum

@@ -114,12 +114,10 @@ github.com/dell/gocsi v1.7.0 h1:fMQO2zwAXCaIsUoPCcnnuPMwfQMoaI1/0aqkQVndlxU=
 github.com/dell/gocsi v1.7.0/go.mod h1:X/8Ll8qqKAKCenmd1gPJMUvUmgY8cK0LiS8Pck12UaU=
 github.com/dell/gofsutil v1.12.0 h1:oo2YHfGFKHvHS1urtqjOIKpaHwcdyqacwKHLXzUg33M=
 github.com/dell/gofsutil v1.12.0/go.mod h1:mGMN5grVDtHv2imNw5+gFr2RmCqeyYgBFBldUbHtV78=
-github.com/dell/goscaleio v1.11.1-0.20230707063208-b67372e0f8d0 h1:3GqvTjqCAAG6y8FDUtKRR9q/Uj5lKD1hFD6w7FTzhww=
-github.com/dell/goscaleio v1.11.1-0.20230707063208-b67372e0f8d0/go.mod h1:dMTrHnXSsPus+Kd9mrs0JuyrCndoKvFP/bbEdc21Bi8=
-github.com/dell/goscaleio v1.11.1-0.20230722160317-b47d2ec46caa h1:3qX09PA+g/UjglChU7OLE7D25O9n0ZsF9ERVvhKy6dg=
-github.com/dell/goscaleio v1.11.1-0.20230722160317-b47d2ec46caa/go.mod h1:dMTrHnXSsPus+Kd9mrs0JuyrCndoKvFP/bbEdc21Bi8=
-github.com/dell/goscaleio v1.11.1-0.20230724113841-f10386ad0bb4 h1:6IDogX2aGiWQvybl3cWwtFdrLwiXE753sG6mDZr9Q3g=
-github.com/dell/goscaleio v1.11.1-0.20230724113841-f10386ad0bb4/go.mod h1:dMTrHnXSsPus+Kd9mrs0JuyrCndoKvFP/bbEdc21Bi8=
+github.com/dell/goscaleio v1.11.1-0.20230721055528-55caf6d14be6 h1:xsB1Ergjq3C4DrIhixHfBsoJ8lYj9uY/HsTVqDD9EN8=
+github.com/dell/goscaleio v1.11.1-0.20230721055528-55caf6d14be6/go.mod h1:dMTrHnXSsPus+Kd9mrs0JuyrCndoKvFP/bbEdc21Bi8=
+github.com/dell/goscaleio v1.11.1-0.20230724122254-8f8b41ad4aad h1:3yLaVslMtLDGrWBkR88F+9tKgJjEKWYrlj7f0aVQd9k=
+github.com/dell/goscaleio v1.11.1-0.20230724122254-8f8b41ad4aad/go.mod h1:dMTrHnXSsPus+Kd9mrs0JuyrCndoKvFP/bbEdc21Bi8=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=

helm/csi-vxflexos/templates/controller.yaml

@@ -405,6 +405,12 @@ spec:
             - name: X_CSI_POWERFLEX_EXTERNAL_ACCESS
               value: "{{ .Values.externalAccess }}"
             {{- end }}
+            {{- if hasKey .Values "enableQuota" }}
+            {{- if eq .Values.enableQuota true}}
+            - name: X_CSI_QUOTA_ENABLED
+              value: "{{ .Values.enableQuota }}"
+            {{- end }}
+            {{- end }}
           volumeMounts:
             - name: socket-dir
               mountPath: /var/run/csi

helm/csi-vxflexos/values.yaml

@@ -66,6 +66,14 @@ imagePullPolicy: IfNotPresent
 # Default value: "0777"
 nfsAcls: "0777"
 
+# enableQuota: a boolean that, when enabled, will set quota limit for a newly provisioned NFS volume.
+# Allowed values:
+#   true: set quota for volume
+#   false: do not set quota for volume
+# Optional: true
+# Default value: none
+enableQuota: "false"
+
 # "enablesnapshotcgdelete"- a boolean that, when enabled, will delete all snapshots in a consistency group
 # everytime a snap in the group is deleted
 # Allowed values: true, false
@@ -74,15 +82,15 @@ enablesnapshotcgdelete: "false"
 
 # "enablelistvolumesnapshot" - a boolean that, when enabled, will allow list volume operation to include snapshots (since creating a volume
 # from a snap actually results in a new snap)
-# It is recommend this be false unless instructed otherwise.
+# It is recommended this be false unless instructed otherwise.
 # Allowed values: true, false
 # Default value: none
 enablelistvolumesnapshot: "false"
 
 # Setting allowRWOMultiPodAccess to "true" will allow multiple pods on the same node
 # to access the same RWO volume. This behavior conflicts with the CSI specification version 1.3
 # NodePublishVolume descrition that requires an error to be returned in this case.
-# However some other CSI drivers support this behavior and some customers desire this behavior.
+# However, some other CSI drivers support this behavior and some customers desire this behavior.
 # Kubernetes could make a change at their discretion that would preclude our ability to support this option.
 # Customers use this option at their own risk.
 # You should leave this set as "false" unless instructed to change it by Dell support.

samples/storageclass/storageclass-nfs.yaml

@@ -81,6 +81,28 @@ parameters:
   # Optional: true
   # Default value: "0777"
   # nfsAcls: "0777"
+
+  # path: relative path to the root of the associated filesystem.
+  # Allowed values: string
+  # Optional: true
+  # Default value: None
+  # Examples: /fs, /csi
+  # path: /csi
+
+  # softLimit: set soft limit to quota.
+  # Specified as a percentage
+  # Allowed values: int
+  # Optional: true
+  # Default value: 0, unlimited quota
+  # softLimit: "80"
+
+  # gracePeriod: Grace period of tree quota, must be mentioned along with softLimit, in seconds.
+  # Soft Limit can be exceeded until the grace period.
+  # No hard limit when set to -1.
+  # Allowed values: int
+  # Optional: true
+  # Default value : 0
+  # gracePeriod: "86400"
 
 # volumeBindingMode determines how volume binding and dynamic provisioning should occur
 # Allowed values:

service/controller.go

@@ -73,10 +73,23 @@ const (
 
 	// NFSExportLocalPath is the local path for NFSExport
 	NFSExportLocalPath = "/"
+
 	// NFSExportNamePrefix is the prefix used for nfs exports created using
 	// csi-powerflex driver
 	NFSExportNamePrefix = "csishare-"
 
+	// KeyPath is the key used to get path of the associated filesystem
+	// from the volume create parameters map
+	KeyPath = "path"
+
+	// KeySoftLimit is the key used to get the soft limit of the filesystem
+	// from the volume create parameters map
+	KeySoftLimit = "softLimit"
+
+	// KeyGracePeriod is the key used to get the grace period from the
+	// volume create parameters map
+	KeyGracePeriod = "gracePeriod"
+
 	// DefaultVolumeSizeKiB is default volume sgolang/protobuf/blob/master/ptypesize
 	// to create on a scaleIO cluster when no size is given, expressed in KiB
 	DefaultVolumeSizeKiB = 16 * kiBytesInGiB
@@ -361,9 +374,47 @@ func (s *service) CreateVolume(
 			return nil, status.Errorf(codes.Unknown, "Create Volume %s failed with error: %v", volName, err)
 		}
 
+		// set quota limits, if specified in NFS storage class
+		isQuotaEnabled := s.opts.IsQuotaEnabled
+		if isQuotaEnabled {
+			// get filesystem (NFS volume), newly created
+			fs, err := system.GetFileSystemByIDName(fsResp.ID, "")
+			if err != nil {
+				Log.Debugf("Find Volume response error: %v", err)
+				return nil, status.Errorf(codes.Unknown, "Find Volume response error: %v", err)
+			}
+			path, ok := params[KeyPath]
+			if !ok {
+				return nil, status.Errorf(codes.InvalidArgument, "`%s` is a required parameter", KeyPath)
+			}
+
+			softLimit, ok := params[KeySoftLimit]
+			if !ok {
+				return nil, status.Errorf(codes.InvalidArgument, "`%s` is a required parameter", KeySoftLimit)
+			}
+
+			gracePeriod, ok := params[KeyGracePeriod]
+			if !ok {
+				return nil, status.Errorf(codes.InvalidArgument, "`%s` is a required parameter", KeyGracePeriod)
+			}
+
+			// create quota for the filesystem
+			quotaID, err := s.createQuota(fsResp.ID, path, softLimit, gracePeriod, int(size), isQuotaEnabled, systemID)
+			if err != nil {
+				// roll back, delete the newly created volume
+				if err = system.DeleteFileSystem(fs.Name); err != nil {
+					return nil, status.Errorf(codes.Internal,
+						"rollback (deleting volume '%s') failed with error : '%v'", fs.Name, err.Error())
+				}
+				return nil, fmt.Errorf("error creating quota ('%s', '%d' bytes), abort, also successfully rolled back by deleting the newly created volume", fs.Name, size)
+			}
+			Log.Infof("Tree quota set for: %d bytes on directory: '%s', quota ID: %s", size, path, quotaID)
+		}
+
 		newFs, err := system.GetFileSystemByIDName(fsResp.ID, "")
 		if err != nil {
-			Log.Debugf("Find Volume response: %v Error: %v", newFs, err)
+			Log.Debugf("Find Volume response error: %v", err)
+			return nil, status.Errorf(codes.Unknown, "Find Volume response error: %v", err)
 		}
 		if newFs != nil {
 			vi := s.getCSIVolumeFromFilesystem(newFs, systemID)
@@ -524,6 +575,99 @@ func (s *service) CreateVolume(
 	return nil, status.Errorf(codes.NotFound, "Volume/Filesystem not found after create. %v", err)
 }
 
+func (s *service) createQuota(fsID, path, softLimit, gracePeriod string, size int, isQuotaEnabled bool, systemID string) (string, error) {
+	system, err := s.adminClients[systemID].FindSystem(systemID, "", "")
+	if err != nil {
+		return "", err
+	}
+
+	// enabling quota on FS
+	fs, err := system.GetFileSystemByIDName(fsID, "")
+	if err != nil {
+		Log.Debugf("Find Volume response error: %v", err)
+		return "", status.Errorf(codes.Unknown, "Find Volume response error: %v", err)
+	}
+
+	var softLimitInt, gracePeriodInt int64
+	gracePeriodInt, err = strconv.ParseInt(gracePeriod, 10, 64)
+	if err != nil {
+		Log.Debugf("Invalid gracePeriod value. Setting it to default.")
+		gracePeriodInt = 0
+	}
+
+	// converting soft limit from percentage to value
+	if softLimit != "" {
+		softi, err := strconv.ParseInt(softLimit, 10, 64)
+		if err != nil {
+			Log.Debugf("Invalid softLimit value. Setting it to default.")
+			softLimitInt = 0
+		} else {
+			softLimitInt = (softi * int64(size)) / 100
+		}
+	}
+
+	// modify FS to set quota
+	fsModify := &siotypes.FSModify{
+		IsQuotaEnabled: isQuotaEnabled,
+	}
+
+	err = system.ModifyFileSystem(fsModify, fs.ID)
+	if err != nil {
+		Log.Debugf("Modify filesystem failed with error: %v", err)
+		return "", status.Errorf(codes.Unknown, "Modify filesystem failed with error: %v", err)
+	}
+
+	fs, err = system.GetFileSystemByIDName(fsID, "")
+	if err != nil {
+		Log.Debugf("Find Volume response error: %v", err)
+		return "", status.Errorf(codes.Unknown, "Find Volume response error: %v", err)
+	}
+
+	// need to set the quota based on the requested pv size
+	// if a size isn't requested, skip creating the quota
+	if size <= 0 {
+		Log.Debugf("Quotas is enabled, but storage size is not requested, skip creating quotas for volume '%s'", fsID)
+		return "", nil
+	}
+
+	// Check if softLimit < 100
+	if int(softLimitInt) >= size {
+		Log.Warnf("SoftLimit thresholds must be smaller than the hard threshold. Setting it to default for Volume '%s'", fsID)
+		softLimitInt, gracePeriodInt = 0, 0
+	}
+
+	//Check if grace period is set along with soft limit
+	if (softLimitInt != 0) && (gracePeriodInt == 0) {
+		Log.Warnf("Grace period must be configured along with soft limit. Setting it to default for Volume '%s'", fsID)
+		softLimitInt, gracePeriodInt = 0, 0
+	}
+
+	Log.Debugf("Begin to set quota for FS '%s', size '%d', quota enabled: '%t'", fsID, size, isQuotaEnabled)
+	// log all parameters used in CreateTreeQuota call
+	fields := map[string]interface{}{
+		"FileSystemID": fsID,
+		"Path":         path,
+		"HardLimit":    size,
+		"SoftLimit":    softLimitInt,
+		"GracePeriod":  gracePeriodInt,
+	}
+	Log.WithFields(fields).Info("Executing CreateTreeQuota with following fields")
+
+	createQuotaParams := &siotypes.TreeQuotaCreate{
+		FileSystemID: fsID,
+		Path:         path,
+		HardLimit:    size,
+		SoftLimit:    int(softLimitInt),
+		GracePeriod:  int(gracePeriodInt),
+	}
+	quota, err := system.CreateTreeQuota(createQuotaParams)
+	if err != nil {
+		Log.Debugf("Creating quota failed with error: %v", err)
+		return "", status.Errorf(codes.Unknown, "Creating quota failed with error: %v", err)
+	}
+	return quota.ID, nil
+}
+
 // Copies the interesting parameters to the output map.
 func copyInterestingParameters(parameters, out map[string]string) {
 	for _, str := range interestingParameters {
@@ -2617,6 +2761,33 @@ func (s *service) ControllerExpandVolume(ctx context.Context, req *csi.Controlle
 			return nil, status.Error(codes.Internal, err.Error())
 		}
 
+		// update tree quota hard limit and soft limit if pvc size has changed
+
+		isQuotaEnabled := s.opts.IsQuotaEnabled
+		if isQuotaEnabled && fs.IsQuotaEnabled {
+			treeQuota, err := system.GetTreeQuotaByFSID(fsID)
+			if err != nil {
+				Log.Errorf("Fetching tree quota for filesystem failed, error: %s", err.Error())
+				return nil, status.Error(codes.Internal, err.Error())
+			}
+
+			// Modify Tree Quota
+			updatedSoftLimit := treeQuota.SoftLimit * (requestedSize / treeQuota.HardLimit)
+			treeQuotaID := treeQuota.ID
+			Log.Infof("Modifying tree quota ID %s for filesystem ID: %s", treeQuotaID, fsID)
+			quotaModify := &siotypes.TreeQuotaModify{
+				HardLimit: requestedSize,
+				SoftLimit: updatedSoftLimit,
+			}
+
+			err = system.ModifyTreeQuota(quotaModify, treeQuotaID)
+			if err != nil {
+				Log.Errorf("Modifying tree quota for filesystem failed, error: %s", err.Error())
+				return nil, status.Error(codes.Internal, err.Error())
+			}
+			Log.Infof("Tree quota modified successfully.")
+		}
+
 		csiResp := &csi.ControllerExpandVolumeResponse{
 			CapacityBytes:         int64(requestedSize),
 			NodeExpansionRequired: false,

service/envvars.go

@@ -63,6 +63,10 @@ const (
 
 	// EnvExternalAccess is used to specify additional entries for host to access NFS volumes.
 	EnvExternalAccess = "X_CSI_POWERFLEX_EXTERNAL_ACCESS"
+
 	// EnvMaxVolumesPerNode specifies maximum number of volumes that controller can publish to the node.
 	EnvMaxVolumesPerNode = "X_CSI_MAX_VOLUMES_PER_NODE"
+
+	// EnvQuotaEnabled enables setting of quota for NFS volumes.
+	EnvQuotaEnabled = "X_CSI_QUOTA_ENABLED"
 )

service/features/controller_publish_unpublish.feature

@@ -552,3 +552,62 @@ Feature: VxFlex OS CSI interface
     And I call UnpublishVolume
     And no error was received
     Then the number of SDC mappings is 0
+
+  Scenario: Create NFS volume, enable quota with all key parameters
+    Given a VxFlexOS service
+    And I enable quota for filesystem
+    And I set quota with path "/fs" softLimit "20" graceperiod "86400"
+    And I call CreateVolumeSize nfs "vol-inttest-nfs" "10"
+    Then a valid CreateVolumeResponse is returned
+
+  Scenario: Create NFS volume, enable quota without path key
+    Given a VxFlexOS service
+    And I enable quota for filesystem
+    And I specify NoPath
+    And I call CreateVolumeSize nfs "vol-inttest-nfs" "10"
+    Then the error contains "rpc error: code = InvalidArgument desc = `path` is a required parameter"
+
+  Scenario: Create NFS volume, enable quota without soft limit key
+    Given a VxFlexOS service
+    And I enable quota for filesystem
+    And I specify NoSoftLimit
+    And I call CreateVolumeSize nfs "vol-inttest-nfs" "10"
+    Then the error contains "rpc error: code = InvalidArgument desc = `softLimit` is a required parameter"
+
+  Scenario: Create NFS volume, enable quota without grace period key
+    Given a VxFlexOS service
+    And I enable quota for filesystem
+    And I specify NoGracePeriod
+    And I call CreateVolumeSize nfs "vol-inttest-nfs" "10"
+    Then the error contains "rpc error: code = InvalidArgument desc = `gracePeriod` is a required parameter"
+
+  Scenario: Create NFS volume, create quota error
+    Given a VxFlexOS service
+    And I enable quota for filesystem
+    And I set quota with path "/fs" softLimit "20" graceperiod "86400"
+    And I induce error "CreateQuotaError"
+    And I call CreateVolumeSize nfs "vol-inttest-nfs" "10"
+    Then the error contains "error creating quota ('vol-inttest-nfs', '10737418240' bytes), abort, also successfully rolled back by deleting the newly created volume"
+
+  Scenario: Create NFS volume, invalid soft limit, set to default
+    Given a VxFlexOS service
+    And I enable quota for filesystem
+    And I set quota with path "/fs" softLimit "abc" graceperiod "86400"
+    And I call CreateVolumeSize nfs "vol-inttest-nfs" "10"
+    Then a valid CreateVolumeResponse is returned
+
+  Scenario: Create NFS volume, invalid grace period, set to default
+    Given a VxFlexOS service
+    And I enable quota for filesystem
+    And I set quota with path "/fs" softLimit "20" graceperiod "xyz"
+    And I call CreateVolumeSize nfs "vol-inttest-nfs" "10"
+    Then a valid CreateVolumeResponse is returned
+
+  Scenario: Create NFS volume, enable quota, with FS quota disabled
+    Given a VxFlexOS service
+    And I enable quota for filesystem
+    And I set quota with path "/fs" softLimit "20" graceperiod "86400"
+    And a capability with voltype "mount" access "single-node-single-writer" fstype "nfs"
+    And I induce error "FSQuotaError"
+    When I call CreateVolumeSize nfs "vol-inttest-nfs" "8"
+    Then the error contains "error creating quota "

service/features/filesystem.json.template

@@ -19,7 +19,7 @@
     "is_smb_notify_on_write_enabled": false,
     "smb_notify_on_change_dir_depth": 512,
     "is_async_MTime_enabled": false,
-    "is_quota_enabled": true,
+    "is_quota_enabled": __IS_QUOTA_ENABLED__,
     "grace_period": 86400,
     "default_hard_limit": 6442450944,
     "default_soft_limit": 3221225472,