Node Publish volume changes for the nfs support

Dockerfile

@@ -35,6 +35,7 @@ RUN microdnf update -y && \
         kmod \
         libaio \
         numactl \
+        nfs-utils \
         xfsprogs && \
     microdnf clean all
 ENTRYPOINT ["/csi-vxflexos.sh"]

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.2.2
 	github.com/dell/gocsi v1.7.0
 	github.com/dell/gofsutil v1.12.0
-	github.com/dell/goscaleio v1.10.1-0.20230502150156-f467d3984623
+	github.com/dell/goscaleio v1.10.1-0.20230515095359-d145e54cb0fe
 	github.com/fsnotify/fsnotify v1.5.1
 	github.com/golang/protobuf v1.5.3
 	github.com/google/uuid v1.3.0

go.sum

@@ -114,8 +114,8 @@ github.com/dell/gocsi v1.7.0 h1:fMQO2zwAXCaIsUoPCcnnuPMwfQMoaI1/0aqkQVndlxU=
 github.com/dell/gocsi v1.7.0/go.mod h1:X/8Ll8qqKAKCenmd1gPJMUvUmgY8cK0LiS8Pck12UaU=
 github.com/dell/gofsutil v1.12.0 h1:oo2YHfGFKHvHS1urtqjOIKpaHwcdyqacwKHLXzUg33M=
 github.com/dell/gofsutil v1.12.0/go.mod h1:mGMN5grVDtHv2imNw5+gFr2RmCqeyYgBFBldUbHtV78=
-github.com/dell/goscaleio v1.10.1-0.20230502150156-f467d3984623 h1:PQwO9B6aI5IF1NI9KjxRZCFm5Mk2YkB21b9SEcytmgI=
-github.com/dell/goscaleio v1.10.1-0.20230502150156-f467d3984623/go.mod h1:dMTrHnXSsPus+Kd9mrs0JuyrCndoKvFP/bbEdc21Bi8=
+github.com/dell/goscaleio v1.10.1-0.20230515095359-d145e54cb0fe h1:xG7XkoRorM/I9RhbQrFxDOdy5m9TLfYAZhaAX9vH138=
+github.com/dell/goscaleio v1.10.1-0.20230515095359-d145e54cb0fe/go.mod h1:dMTrHnXSsPus+Kd9mrs0JuyrCndoKvFP/bbEdc21Bi8=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=

samples/storageclass/storageclass-nfs.yaml

@@ -94,4 +94,4 @@ allowedTopologies:
 - matchLabelExpressions:
   - key: csi-vxflexos.dellemc.com/<SYSTEM_ID>-nfs # Insert System ID
     values:
-    - csi-vxflexos.dellemc.com    
+    - "true"   

service/mount.go

@@ -331,6 +331,118 @@ func publishVolume(
 	return nil
 }
 
+func publishNFS(ctx context.Context, req *csi.NodePublishVolumeRequest, nfsExportUrl string) error {
+	volCap := req.GetVolumeCapability()
+
+	if volCap == nil {
+		return status.Error(codes.InvalidArgument,
+			"Volume Capability is required")
+	}
+
+	am := volCap.GetAccessMode()
+
+	if am == nil {
+		return status.Error(codes.InvalidArgument,
+			"Volume Access Mode is required")
+	}
+
+	mountVol := volCap.GetMount()
+
+	if mountVol == nil {
+		return status.Error(codes.InvalidArgument, "Invalid access type")
+	}
+
+	var mntOptions []string
+	mntOptions = mountVol.GetMountFlags()
+	Log.Infof("The mountOptions received are: %s", mntOptions)
+
+	target := req.GetTargetPath()
+	if target == "" {
+		return status.Error(codes.InvalidArgument,
+			"Target Path is required")
+	}
+
+	// make sure target is created
+	_, err := mkdir(target)
+
+	if err != nil {
+		return status.Error(codes.FailedPrecondition, fmt.Sprintf("Could not create '%s': '%s'", target, err.Error()))
+	}
+	roFlag := req.GetReadonly()
+	rwOption := "rw"
+	if roFlag {
+		rwOption = "ro"
+	}
+
+	mntOptions = append(mntOptions, rwOption)
+
+	fields := map[string]interface{}{
+		"ID":         req.VolumeId,
+		"TargetPath": target,
+		"ExportPath": nfsExportUrl,
+		"AccessMode": am.GetMode(),
+	}
+	Log.WithFields(fields).Info("Node publish volume params ")
+
+	mnts, err := gofsutil.GetMounts(ctx)
+	if err != nil {
+		return status.Errorf(codes.Internal,
+			"could not reliably determine existing mount status: '%s'",
+			err.Error())
+	}
+
+	if len(mnts) != 0 {
+		for _, m := range mnts {
+			// check for idempotency
+			//same volume
+			if m.Device == nfsExportUrl {
+				if m.Path == target {
+					//as per specs, T1=T2, P1=P2 - return OK
+					if contains(m.Opts, rwOption) {
+						Log.WithFields(fields).Debug(
+							"mount already in place with same options")
+						return nil
+					}
+					//T1=T2, P1!=P2 - return AlreadyExists
+					Log.WithFields(fields).Error("Mount point already in use by device with different options")
+					return status.Error(codes.AlreadyExists, "Mount point already in use by device with different options")
+				}
+				//T1!=T2, P1==P2 || P1 != P2 - return FailedPrecondition for single node
+				if am.GetMode() == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER ||
+					am.GetMode() == csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY ||
+					am.GetMode() == csi.VolumeCapability_AccessMode_SINGLE_NODE_SINGLE_WRITER {
+					Log.WithFields(fields).Error("Mount point already in use for same device")
+					return status.Error(codes.FailedPrecondition, "Mount point already in use for same device")
+				}
+			}
+		}
+	}
+
+	Log.Infof("The mountOptions being used for mount are: %s", mntOptions)
+	if err := gofsutil.Mount(context.Background(), nfsExportUrl, target, "nfs", mntOptions...); err != nil {
+		var count = 0
+		var errmsg = err.Error()
+		//Both substring validation is for NFSv3 and NFSv4 errors resp.
+		for (strings.Contains(strings.ToLower(errmsg), "access denied by server while mounting") || (strings.Contains(strings.ToLower(errmsg), "no such file or directory"))) && count < 5 {
+			time.Sleep(2 * time.Second)
+			Log.Infof("Mount re-trial attempt-%d", count)
+			err = gofsutil.Mount(context.Background(), nfsExportUrl, target, "nfs", mntOptions...)
+			if err != nil {
+				errmsg = err.Error()
+			} else {
+				break
+			}
+			count++
+		}
+		if err != nil {
+			Log.Errorf("%v", err)
+			return err
+		}
+	}
+	return nil
+
+}
+
 func handlePrivFSMount(
 	ctx context.Context,
 	accMode *csi.VolumeCapability_AccessMode,

service/node.go

@@ -142,6 +142,13 @@ func (s *service) NodePublishVolume(
 	}
 	Log.Printf("[NodePublishVolume] csiVolID: %s", csiVolID)
 
+	// Check for NFS protocol
+	fsType := volumeContext[KeyFsType]
+	isNFS := false
+	if fsType == "nfs" {
+		isNFS = true
+	}
+
 	volID := getVolumeIDFromCsiVolumeID(csiVolID)
 	Log.Printf("[NodePublishVolume] volumeID: %s", volID)
 
@@ -170,6 +177,42 @@ func (s *service) NodePublishVolume(
 
 	}
 
+	if isNFS {
+		fsID := getFilesystemIDFromCsiVolumeID(csiVolID)
+
+		fs, err := s.getFilesystemByID(fsID, systemID)
+		if err != nil {
+			if strings.EqualFold(err.Error(), sioGatewayVolumeNotFound) || strings.Contains(err.Error(), "must be a hexadecimal number") {
+				return nil, status.Error(codes.NotFound,
+					"filesystem not found")
+			}
+			return nil, status.Errorf(codes.Internal,
+				"failure checking filesystem status before controller publish: %s",
+				err.Error())
+		}
+
+		client := s.adminClients[systemID]
+
+		NFSExport, err := s.getNFSExport(fs, client)
+
+		if err != nil {
+			return nil, err
+		}
+
+		fileInterface, err := s.getFileInterface(systemID, fs, client)
+		if err != nil {
+			return nil, err
+		}
+
+		path := fmt.Sprintf("%s:%s", fileInterface.IPAddress, NFSExport.Path)
+
+		if err := publishNFS(ctx, req, path); err != nil {
+			return nil, err
+		}
+
+		return &csi.NodePublishVolumeResponse{}, nil
+	}
+
 	sdcMappedVol, err := s.getSDCMappedVol(volID, systemID, publishGetMappedVolMaxRetry)
 	if err != nil {
 		return nil, status.Error(codes.InvalidArgument, err.Error())

service/service.go

@@ -892,6 +892,45 @@ func getFilesystemIDFromCsiVolumeID(csiVolID string) string {
 	return ""
 }
 
+func (s *service) getNFSExport(fs *siotypes.FileSystem, client *goscaleio.Client) (*siotypes.NFSExport, error) {
+
+	nfsExportList, err := client.GetNFSExport()
+
+	if err != nil {
+		return nil, err
+	}
+
+	for _, nfsExport := range nfsExportList {
+		if nfsExport.FileSystemID == fs.ID {
+			return &nfsExport, nil
+		}
+	}
+
+	return nil, status.Errorf(codes.NotFound, "NFS Export for the file system: %s not found", fs.Name)
+
+}
+
+func (s *service) getFileInterface(systemID string, fs *siotypes.FileSystem, client *goscaleio.Client) (*siotypes.FileInterface, error) {
+	system, err := client.FindSystem(systemID, "", "")
+
+	if err != nil {
+		return nil, err
+	}
+
+	nas, err := system.GetNASByIDName(fs.NasServerID, "")
+
+	if err != nil {
+		return nil, err
+	}
+
+	fileInterface, err := system.GetFileInterface(nas.CurrentPreferredIPv4InterfaceID)
+
+	if err != nil {
+		return nil, err
+	}
+	return fileInterface, err
+}
+
 // getSystemIDFromCsiVolumeId returns PowerFlex volume ID from CSI volume ID
 func (s *service) getSystemIDFromCsiVolumeID(csiVolID string) string {
 	containsHyphen := strings.Contains(csiVolID, "/")
@@ -952,7 +991,7 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli
 
 	// Create NFS export if it doesn't exist
 	if !nfsExportExists {
-		Log.Debugf("NFS Export does not exist for fs: %s ,proceeding to create NFS Export", fs)
+		Log.Debugf("NFS Export does not exist for fs: %s ,proceeding to create NFS Export", fs.Name)
 		resp, err := client.CreateNFSExport(&siotypes.NFSExportCreate{
 			Name:         nfsExportName,
 			FileSystemID: fs.ID,