diff --git a/Dockerfile b/Dockerfile index 179dc22b..293851a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,6 +35,7 @@ RUN microdnf update -y && \ kmod \ libaio \ numactl \ + nfs-utils \ xfsprogs && \ microdnf clean all ENTRYPOINT ["/csi-vxflexos.sh"] diff --git a/go.mod b/go.mod index 6dd31381..1735da25 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.2.2 github.com/dell/gocsi v1.7.0 github.com/dell/gofsutil v1.12.0 - github.com/dell/goscaleio v1.10.1-0.20230502150156-f467d3984623 + github.com/dell/goscaleio v1.10.1-0.20230515095359-d145e54cb0fe github.com/fsnotify/fsnotify v1.5.1 github.com/golang/protobuf v1.5.3 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index d93020fe..7b7bc971 100644 --- a/go.sum +++ b/go.sum @@ -114,8 +114,8 @@ github.com/dell/gocsi v1.7.0 h1:fMQO2zwAXCaIsUoPCcnnuPMwfQMoaI1/0aqkQVndlxU= github.com/dell/gocsi v1.7.0/go.mod h1:X/8Ll8qqKAKCenmd1gPJMUvUmgY8cK0LiS8Pck12UaU= github.com/dell/gofsutil v1.12.0 h1:oo2YHfGFKHvHS1urtqjOIKpaHwcdyqacwKHLXzUg33M= github.com/dell/gofsutil v1.12.0/go.mod h1:mGMN5grVDtHv2imNw5+gFr2RmCqeyYgBFBldUbHtV78= -github.com/dell/goscaleio v1.10.1-0.20230502150156-f467d3984623 h1:PQwO9B6aI5IF1NI9KjxRZCFm5Mk2YkB21b9SEcytmgI= -github.com/dell/goscaleio v1.10.1-0.20230502150156-f467d3984623/go.mod h1:dMTrHnXSsPus+Kd9mrs0JuyrCndoKvFP/bbEdc21Bi8= +github.com/dell/goscaleio v1.10.1-0.20230515095359-d145e54cb0fe h1:xG7XkoRorM/I9RhbQrFxDOdy5m9TLfYAZhaAX9vH138= +github.com/dell/goscaleio v1.10.1-0.20230515095359-d145e54cb0fe/go.mod h1:dMTrHnXSsPus+Kd9mrs0JuyrCndoKvFP/bbEdc21Bi8= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= diff --git a/samples/storageclass/storageclass-nfs.yaml b/samples/storageclass/storageclass-nfs.yaml index a89568ef..3a3a2ceb 100644 --- a/samples/storageclass/storageclass-nfs.yaml +++ b/samples/storageclass/storageclass-nfs.yaml @@ -94,4 +94,4 @@ allowedTopologies: - matchLabelExpressions: - key: csi-vxflexos.dellemc.com/-nfs # Insert System ID values: - - csi-vxflexos.dellemc.com + - "true" diff --git a/service/mount.go b/service/mount.go index 313c4687..7d6a9329 100644 --- a/service/mount.go +++ b/service/mount.go @@ -331,6 +331,118 @@ func publishVolume( return nil } +func publishNFS(ctx context.Context, req *csi.NodePublishVolumeRequest, nfsExportUrl string) error { + volCap := req.GetVolumeCapability() + + if volCap == nil { + return status.Error(codes.InvalidArgument, + "Volume Capability is required") + } + + am := volCap.GetAccessMode() + + if am == nil { + return status.Error(codes.InvalidArgument, + "Volume Access Mode is required") + } + + mountVol := volCap.GetMount() + + if mountVol == nil { + return status.Error(codes.InvalidArgument, "Invalid access type") + } + + var mntOptions []string + mntOptions = mountVol.GetMountFlags() + Log.Infof("The mountOptions received are: %s", mntOptions) + + target := req.GetTargetPath() + if target == "" { + return status.Error(codes.InvalidArgument, + "Target Path is required") + } + + // make sure target is created + _, err := mkdir(target) + + if err != nil { + return status.Error(codes.FailedPrecondition, fmt.Sprintf("Could not create '%s': '%s'", target, err.Error())) + } + roFlag := req.GetReadonly() + rwOption := "rw" + if roFlag { + rwOption = "ro" + } + + mntOptions = append(mntOptions, rwOption) + + fields := map[string]interface{}{ + "ID": req.VolumeId, + "TargetPath": target, + "ExportPath": nfsExportUrl, + "AccessMode": am.GetMode(), + } + Log.WithFields(fields).Info("Node publish volume params ") + + mnts, err := gofsutil.GetMounts(ctx) + if err != nil { + return status.Errorf(codes.Internal, + "could not reliably determine existing mount status: '%s'", + err.Error()) + } + + if len(mnts) != 0 { + for _, m := range mnts { + // check for idempotency + //same volume + if m.Device == nfsExportUrl { + if m.Path == target { + //as per specs, T1=T2, P1=P2 - return OK + if contains(m.Opts, rwOption) { + Log.WithFields(fields).Debug( + "mount already in place with same options") + return nil + } + //T1=T2, P1!=P2 - return AlreadyExists + Log.WithFields(fields).Error("Mount point already in use by device with different options") + return status.Error(codes.AlreadyExists, "Mount point already in use by device with different options") + } + //T1!=T2, P1==P2 || P1 != P2 - return FailedPrecondition for single node + if am.GetMode() == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER || + am.GetMode() == csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY || + am.GetMode() == csi.VolumeCapability_AccessMode_SINGLE_NODE_SINGLE_WRITER { + Log.WithFields(fields).Error("Mount point already in use for same device") + return status.Error(codes.FailedPrecondition, "Mount point already in use for same device") + } + } + } + } + + Log.Infof("The mountOptions being used for mount are: %s", mntOptions) + if err := gofsutil.Mount(context.Background(), nfsExportUrl, target, "nfs", mntOptions...); err != nil { + var count = 0 + var errmsg = err.Error() + //Both substring validation is for NFSv3 and NFSv4 errors resp. + for (strings.Contains(strings.ToLower(errmsg), "access denied by server while mounting") || (strings.Contains(strings.ToLower(errmsg), "no such file or directory"))) && count < 5 { + time.Sleep(2 * time.Second) + Log.Infof("Mount re-trial attempt-%d", count) + err = gofsutil.Mount(context.Background(), nfsExportUrl, target, "nfs", mntOptions...) + if err != nil { + errmsg = err.Error() + } else { + break + } + count++ + } + if err != nil { + Log.Errorf("%v", err) + return err + } + } + return nil + +} + func handlePrivFSMount( ctx context.Context, accMode *csi.VolumeCapability_AccessMode, diff --git a/service/node.go b/service/node.go index 2167e823..afe5a0f2 100644 --- a/service/node.go +++ b/service/node.go @@ -142,6 +142,13 @@ func (s *service) NodePublishVolume( } Log.Printf("[NodePublishVolume] csiVolID: %s", csiVolID) + // Check for NFS protocol + fsType := volumeContext[KeyFsType] + isNFS := false + if fsType == "nfs" { + isNFS = true + } + volID := getVolumeIDFromCsiVolumeID(csiVolID) Log.Printf("[NodePublishVolume] volumeID: %s", volID) @@ -170,6 +177,42 @@ func (s *service) NodePublishVolume( } + if isNFS { + fsID := getFilesystemIDFromCsiVolumeID(csiVolID) + + fs, err := s.getFilesystemByID(fsID, systemID) + if err != nil { + if strings.EqualFold(err.Error(), sioGatewayVolumeNotFound) || strings.Contains(err.Error(), "must be a hexadecimal number") { + return nil, status.Error(codes.NotFound, + "filesystem not found") + } + return nil, status.Errorf(codes.Internal, + "failure checking filesystem status before controller publish: %s", + err.Error()) + } + + client := s.adminClients[systemID] + + NFSExport, err := s.getNFSExport(fs, client) + + if err != nil { + return nil, err + } + + fileInterface, err := s.getFileInterface(systemID, fs, client) + if err != nil { + return nil, err + } + + path := fmt.Sprintf("%s:%s", fileInterface.IPAddress, NFSExport.Path) + + if err := publishNFS(ctx, req, path); err != nil { + return nil, err + } + + return &csi.NodePublishVolumeResponse{}, nil + } + sdcMappedVol, err := s.getSDCMappedVol(volID, systemID, publishGetMappedVolMaxRetry) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) diff --git a/service/service.go b/service/service.go index 1f66fa1f..49025ce0 100644 --- a/service/service.go +++ b/service/service.go @@ -892,6 +892,45 @@ func getFilesystemIDFromCsiVolumeID(csiVolID string) string { return "" } +func (s *service) getNFSExport(fs *siotypes.FileSystem, client *goscaleio.Client) (*siotypes.NFSExport, error) { + + nfsExportList, err := client.GetNFSExport() + + if err != nil { + return nil, err + } + + for _, nfsExport := range nfsExportList { + if nfsExport.FileSystemID == fs.ID { + return &nfsExport, nil + } + } + + return nil, status.Errorf(codes.NotFound, "NFS Export for the file system: %s not found", fs.Name) + +} + +func (s *service) getFileInterface(systemID string, fs *siotypes.FileSystem, client *goscaleio.Client) (*siotypes.FileInterface, error) { + system, err := client.FindSystem(systemID, "", "") + + if err != nil { + return nil, err + } + + nas, err := system.GetNASByIDName(fs.NasServerID, "") + + if err != nil { + return nil, err + } + + fileInterface, err := system.GetFileInterface(nas.CurrentPreferredIPv4InterfaceID) + + if err != nil { + return nil, err + } + return fileInterface, err +} + // getSystemIDFromCsiVolumeId returns PowerFlex volume ID from CSI volume ID func (s *service) getSystemIDFromCsiVolumeID(csiVolID string) string { containsHyphen := strings.Contains(csiVolID, "/") @@ -952,7 +991,7 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli // Create NFS export if it doesn't exist if !nfsExportExists { - Log.Debugf("NFS Export does not exist for fs: %s ,proceeding to create NFS Export", fs) + Log.Debugf("NFS Export does not exist for fs: %s ,proceeding to create NFS Export", fs.Name) resp, err := client.CreateNFSExport(&siotypes.NFSExportCreate{ Name: nfsExportName, FileSystemID: fs.ID,