diff --git a/Makefile b/Makefile index 4e73a048..42ebd712 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ ifndef NOTES NOTES= endif ifndef TAGMSG - TAGMSG="CSI Spec 1.5" + TAGMSG="CSI Spec 1.6" endif clean: diff --git a/README.md b/README.md index 894853ef..5d380b9e 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ ## Description CSI Driver for PowerStore is part of the [CSM (Container Storage Modules)](https://github.com/dell/csm) open-source suite of Kubernetes storage enablers for Dell products. CSI Driver for PowerStore is a Container Storage Interface (CSI) driver that provides support for provisioning persistent storage using Dell PowerStore storage array. -It supports CSI specification version 1.5. +It supports CSI specification version 1.6. This project may be compiled as a stand-alone binary using Golang that, when run, provides a valid CSI endpoint. It also can be used as a precompiled container image. diff --git a/go.mod b/go.mod index cb058208..d510fb09 100644 --- a/go.mod +++ b/go.mod @@ -5,18 +5,18 @@ go 1.21 require ( github.com/akutz/gosync v0.1.0 github.com/apparentlymart/go-cidr v1.1.0 - github.com/container-storage-interface/spec v1.5.0 + github.com/container-storage-interface/spec v1.6.0 github.com/dell/csi-metadata-retriever v1.5.0 github.com/dell/dell-csi-extensions/common v1.2.0 github.com/dell/dell-csi-extensions/podmon v1.2.0 github.com/dell/dell-csi-extensions/replication v1.5.0 github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.3.0 github.com/dell/gobrick v1.9.0 - github.com/dell/gocsi v1.8.0 + github.com/dell/gocsi v1.8.1-0.20230915044639-4bab90258ed0 github.com/dell/gofsutil v1.13.1 github.com/dell/goiscsi v1.8.0 github.com/dell/gonvme v1.5.0 - github.com/dell/gopowerstore v1.13.0 + github.com/dell/gopowerstore v1.13.1-0.20231012074319-4009fe74aa49 github.com/fsnotify/fsnotify v1.5.4 github.com/go-openapi/strfmt v0.21.3 github.com/golang/mock v1.6.0 @@ -34,6 +34,7 @@ require ( github.com/uber/jaeger-lib v2.4.1+incompatible golang.org/x/net v0.14.0 google.golang.org/grpc v1.57.0 + google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/apimachinery v0.26.1 k8s.io/client-go v0.26.1 @@ -102,7 +103,6 @@ require ( google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878 // indirect - google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.66.6 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect diff --git a/go.sum b/go.sum index a258f146..6718d751 100644 --- a/go.sum +++ b/go.sum @@ -102,8 +102,9 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/container-storage-interface/spec v1.5.0 h1:lvKxe3uLgqQeVQcrnL2CPQKISoKjTJxojEs9cBk+HXo= github.com/container-storage-interface/spec v1.5.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= +github.com/container-storage-interface/spec v1.6.0 h1:vwN9uCciKygX/a0toYryoYD5+qI9ZFeAMuhEEKO+JBA= +github.com/container-storage-interface/spec v1.6.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= @@ -131,16 +132,16 @@ github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.3.0 h1:yXocN5AwXzrJXG github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.3.0/go.mod h1:EWT6KIoauXYlcGiss60KwlnTwxFI6KCt3hklW0HZIOc= github.com/dell/gobrick v1.9.0 h1:kx69ygz1QV/uCAyIx9pX9gqiwDK7I4WOv5ZUs2zcfPg= github.com/dell/gobrick v1.9.0/go.mod h1:NK9V+t6LYMWAgHaT4hJiv8FYQdsWzZDz78hir6GAiTI= -github.com/dell/gocsi v1.8.0 h1:0qsC/Ts6QeAWBBVaQvFrZYBXdoR7bmjrDUb3QpcMfHM= -github.com/dell/gocsi v1.8.0/go.mod h1:X/8Ll8qqKAKCenmd1gPJMUvUmgY8cK0LiS8Pck12UaU= +github.com/dell/gocsi v1.8.1-0.20230915044639-4bab90258ed0 h1:EL+IakztajSHXZXIv2RvXcLP4E5FGqrZsAuoq3ZpMpA= +github.com/dell/gocsi v1.8.1-0.20230915044639-4bab90258ed0/go.mod h1:dclFEZScxwDjv/jdZ74CH5Pr2rYTDRllmX9zlU0Lzq8= github.com/dell/gofsutil v1.13.1 h1:hu26rfykH0gvpSxPe5lTBVCHZA3m896/iO+2Ekz0U7A= github.com/dell/gofsutil v1.13.1/go.mod h1:UPRuS1blrPnfT2K3nWRrLHIosZsBznDglovA6DRMmUI= github.com/dell/goiscsi v1.8.0 h1:kocGVOdgnufc6eGpfmwP66hyhY7OVgIafaS/+uM6ogU= github.com/dell/goiscsi v1.8.0/go.mod h1:PTlQGJaGKYgia95mGwwHSBgvfOr3BfLIjGNh1HT6p+s= github.com/dell/gonvme v1.5.0 h1:n73WeQSFaVOlAqjhtk5T3pbu7eZgMTLpPo8/8JymOJ8= github.com/dell/gonvme v1.5.0/go.mod h1:7MFbd7lWSaQwR5pf9ZnVZqhkAKkveSwQEO67jDBZuX0= -github.com/dell/gopowerstore v1.13.0 h1:KEATpl95GhGzO94zeo3+J7gxOrbLt55GZASHFYRVH5s= -github.com/dell/gopowerstore v1.13.0/go.mod h1:MJLVe9FLxhb/dXv8RuKtHwqGAAmRuAIMFqNOTK3ZMz0= +github.com/dell/gopowerstore v1.13.1-0.20231012074319-4009fe74aa49 h1:4omEzu7pEzUZPJpYql5sTy4qIw/+bt5GFsaVPu8jtyo= +github.com/dell/gopowerstore v1.13.1-0.20231012074319-4009fe74aa49/go.mod h1:MJLVe9FLxhb/dXv8RuKtHwqGAAmRuAIMFqNOTK3ZMz0= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index f548de6b..0a974e23 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -23,9 +23,11 @@ import ( "context" "errors" "fmt" + "net/http" "sort" "strconv" "strings" + "sync" "github.com/container-storage-interface/spec/lib/go/csi" "github.com/dell/csi-powerstore/v2/core" @@ -43,6 +45,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/wrapperspb" ) // Interface provides most important controller methods. @@ -72,6 +75,11 @@ type Service struct { K8sVisibilityAutoRegistration bool } +// maxVolumesSizeForArray - store the maxVolumesSizeForArray +var maxVolumesSizeForArray = make(map[string]int64) + +var mutex = &sync.Mutex{} + // Init is a method that initializes internal variables of controller service func (s *Service) Init() error { ctx := context.Background() @@ -898,16 +906,63 @@ func (s *Service) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) return nil, status.Errorf(codes.Internal, "can't find array with provided id %s", arrayID) } } - resp, err := arr.Client.GetCapacity(ctx) + capacity, err := arr.Client.GetCapacity(ctx) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } - + maxVolSize := getMaximumVolumeSize(ctx, arr) + if maxVolSize < 0 { + return &csi.GetCapacityResponse{ + AvailableCapacity: capacity, + }, nil + } + maxVol := wrapperspb.Int64(maxVolSize) return &csi.GetCapacityResponse{ - AvailableCapacity: resp, + AvailableCapacity: capacity, + MaximumVolumeSize: maxVol, }, nil } +func getMaximumVolumeSize(ctx context.Context, arr *array.PowerStoreArray) int64 { + valueInCache, found := getCachedMaximumVolumeSize(arr.GlobalID) + if !found || valueInCache < 0 { + defaultHeaders := arr.Client.GetCustomHTTPHeaders() + if defaultHeaders == nil { + defaultHeaders = make(http.Header) + } + customHeaders := defaultHeaders + customHeaders.Add("DELL-VISIBILITY", "internal") + arr.Client.SetCustomHTTPHeaders(customHeaders) + + value, err := arr.Client.GetMaxVolumeSize(ctx) + if err != nil { + log.Debug(fmt.Sprintf("GetMaxVolumeSize returning: %v for Array having GlobalId %s", err, arr.GlobalID)) + } + // reset custom header + customHeaders.Del("DELL-VISIBILITY") + arr.Client.SetCustomHTTPHeaders(customHeaders) + // Add a new entry to the MaximumVolumeSize + cacheMaximumVolumeSize(arr.GlobalID, value) + valueInCache = value + } + return valueInCache +} + +func getCachedMaximumVolumeSize(key string) (int64, bool) { + mutex.Lock() + defer mutex.Unlock() + + value, found := maxVolumesSizeForArray[key] + return value, found +} + +func cacheMaximumVolumeSize(key string, value int64) { + mutex.Lock() + defer mutex.Unlock() + + maxVolumesSizeForArray[key] = value +} + // ControllerGetCapabilities returns list of capabilities that are supported by the driver. func (s *Service) ControllerGetCapabilities(ctx context.Context, request *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { newCap := func(cap csi.ControllerServiceCapability_RPC_Type) *csi.ControllerServiceCapability { diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index aa15dc67..512bc98d 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -1206,7 +1206,7 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{}, nil) clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) clientMock.On("DeleteVolume", - mock.AnythingOfType("*context.emptyCtx"), + mock.Anything, mock.AnythingOfType("*gopowerstore.VolumeDelete"), validBaseVolID). Return(gopowerstore.EmptyResponse(""), nil) @@ -1224,7 +1224,7 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{}, nil) clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) clientMock.On("DeleteVolume", - mock.AnythingOfType("*context.emptyCtx"), + mock.Anything, mock.AnythingOfType("*gopowerstore.VolumeDelete"), validBaseVolID). Return(gopowerstore.EmptyResponse(""), nil) @@ -1258,7 +1258,7 @@ var _ = Describe("CSIControllerService", func() { req := &csi.DeleteVolumeRequest{VolumeId: validBlockVolumeID} clientMock.On("DeleteVolume", - mock.AnythingOfType("*context.emptyCtx"), + mock.Anything, mock.AnythingOfType("*gopowerstore.VolumeDelete"), validBaseVolID). Return(gopowerstore.EmptyResponse(""), nil) @@ -1274,7 +1274,7 @@ var _ = Describe("CSIControllerService", func() { It("should successfully delete nfs volume", func() { clientMock.On("GetFsSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.FileSystem{}, nil) clientMock.On("DeleteFS", - mock.AnythingOfType("*context.emptyCtx"), + mock.Anything, validBaseVolID). Return(gopowerstore.EmptyResponse(""), nil) clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). @@ -1367,7 +1367,7 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{}, nil) clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) clientMock.On("DeleteVolume", - mock.AnythingOfType("*context.emptyCtx"), + mock.Anything, mock.AnythingOfType("*gopowerstore.VolumeDelete"), validBaseVolID). Return(gopowerstore.EmptyResponse(""), gopowerstore.APIError{ @@ -1389,7 +1389,7 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("GetNFSExportByFileSystemID", mock.Anything, validBaseVolID). Return(gopowerstore.NFSExport{}, nil) clientMock.On("DeleteFS", - mock.AnythingOfType("*context.emptyCtx"), + mock.Anything, validBaseVolID). Return(gopowerstore.EmptyResponse(""), gopowerstore.APIError{ ErrorMsg: &api.ErrorMsg{ @@ -1411,7 +1411,7 @@ var _ = Describe("CSIControllerService", func() { clientMock.On("GetSnapshotsByVolumeID", mock.Anything, validBaseVolID).Return([]gopowerstore.Volume{}, nil) clientMock.On("GetVolumeGroupsByVolumeID", mock.Anything, validBaseVolID).Return(gopowerstore.VolumeGroups{}, nil) clientMock.On("DeleteVolume", - mock.AnythingOfType("*context.emptyCtx"), + mock.Anything, mock.AnythingOfType("*gopowerstore.VolumeDelete"), validBaseVolID). Return(gopowerstore.EmptyResponse(""), gopowerstore.APIError{ @@ -1736,7 +1736,7 @@ var _ = Describe("CSIControllerService", func() { Size: validVolSize, }, nil) clientMock.On("ModifyVolume", - mock.AnythingOfType("*context.emptyCtx"), + mock.Anything, mock.AnythingOfType("*gopowerstore.VolumeModify"), validBaseVolID). Return(gopowerstore.EmptyResponse(""), nil) @@ -1772,7 +1772,7 @@ var _ = Describe("CSIControllerService", func() { Size: validVolSize, }, nil) clientMock.On("ModifyVolume", - mock.AnythingOfType("*context.emptyCtx"), + mock.Anything, mock.AnythingOfType("*gopowerstore.VolumeModify"), validBaseVolID). Return(gopowerstore.EmptyResponse(""), e) @@ -1793,7 +1793,7 @@ var _ = Describe("CSIControllerService", func() { SizeTotal: validVolSize, }, nil) clientMock.On("ModifyFS", - mock.AnythingOfType("*context.emptyCtx"), + mock.Anything, mock.AnythingOfType("*gopowerstore.FSModify"), validBaseVolID). Return(gopowerstore.EmptyResponse(""), nil) @@ -1816,7 +1816,7 @@ var _ = Describe("CSIControllerService", func() { SizeTotal: validVolSize, }, nil) clientMock.On("ModifyFS", - mock.AnythingOfType("*context.emptyCtx"), + mock.Anything, mock.AnythingOfType("*gopowerstore.FSModify"), validBaseVolID). Return(gopowerstore.EmptyResponse(""), e) @@ -2569,7 +2569,7 @@ var _ = Describe("CSIControllerService", func() { mock.Anything, exportID).Return(gopowerstore.CreateResponse{}, nil) clientMock.On("DeleteFS", - mock.AnythingOfType("*context.emptyCtx"), + mock.Anything, validBaseVolID). Return(gopowerstore.EmptyResponse(""), nil) req := &csi.DeleteVolumeRequest{VolumeId: validNfsVolumeID} @@ -3231,7 +3231,10 @@ var _ = Describe("CSIControllerService", func() { Describe("calling GetCapacity()", func() { When("everything is ok and arrayip is provided", func() { It("should succeed", func() { + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) clientMock.On("GetCapacity", mock.Anything).Return(int64(123123123), nil) + clientMock.On("GetMaxVolumeSize", mock.Anything).Return(int64(-1), nil) req := &csi.GetCapacityRequest{ Parameters: map[string]string{ "arrayIP": "192.168.0.1", @@ -3245,7 +3248,10 @@ var _ = Describe("CSIControllerService", func() { When("everything is ok and array ip is not provided", func() { It("should succeed", func() { + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) clientMock.On("GetCapacity", mock.Anything).Return(int64(123123123), nil) + clientMock.On("GetMaxVolumeSize", mock.Anything).Return(int64(-1), nil) req := &csi.GetCapacityRequest{ Parameters: map[string]string{}, } @@ -3257,7 +3263,10 @@ var _ = Describe("CSIControllerService", func() { When("wrong arrayIP in params", func() { It("should fail with predefined errmsg", func() { + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) clientMock.On("GetCapacity", mock.Anything).Return(int64(123123123), nil) + clientMock.On("GetMaxVolumeSize", mock.Anything).Return(int64(-1), nil) req := &csi.GetCapacityRequest{ Parameters: map[string]string{ "arrayID": "10.10.10.10", @@ -3272,7 +3281,10 @@ var _ = Describe("CSIControllerService", func() { When("everything is correct, but API failed", func() { It("should fail with predefined errmsg", func() { + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) clientMock.On("GetCapacity", mock.Anything).Return(int64(123123123), errors.New("APIErrorUnexpected")) + clientMock.On("GetMaxVolumeSize", mock.Anything).Return(int64(-1), nil) req := &csi.GetCapacityRequest{ Parameters: map[string]string{ "arrayIP": "192.168.0.1", @@ -3284,6 +3296,55 @@ var _ = Describe("CSIControllerService", func() { Expect(err.Error()).To(ContainSubstring("APIErrorUnexpected")) }) }) + + When("everything is correct, but GetMaxVolumeSize API failed", func() { + It("MaximumVolumeSize should not be set in the response", func() { + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + clientMock.On("GetCapacity", mock.Anything).Return(int64(123123123), nil) + req := &csi.GetCapacityRequest{ + Parameters: map[string]string{}, + } + clientMock.On("GetMaxVolumeSize", mock.Anything).Return(int64(-1), errors.New("APIErrorUnexpected")) + + res, err := ctrlSvc.GetCapacity(context.Background(), req) + Expect(res).ToNot(BeNil()) + Expect(err).To(BeNil()) + }) + }) + + When("negative MaximumVolumeSize", func() { + It("MaximumVolumeSize should not be set in the response", func() { + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + clientMock.On("GetCapacity", mock.Anything).Return(int64(123123123), nil) + req := &csi.GetCapacityRequest{ + Parameters: map[string]string{}, + } + clientMock.On("GetMaxVolumeSize", mock.Anything).Return(int64(-1), nil) + + res, err := ctrlSvc.GetCapacity(context.Background(), req) + Expect(res.MaximumVolumeSize).To(BeNil()) + Expect(err).To(BeNil()) + }) + }) + + When("non negative MaximumVolumeSize", func() { + It("MaximumVolumeSize should be set in the response", func() { + clientMock.On("SetCustomHTTPHeaders", mock.Anything).Return(nil) + clientMock.On("GetCustomHTTPHeaders").Return(make(http.Header)) + clientMock.On("GetCapacity", mock.Anything).Return(int64(123123123), nil) + req := &csi.GetCapacityRequest{ + Parameters: map[string]string{}, + } + clientMock.On("GetMaxVolumeSize", mock.Anything).Return(int64(100000), nil) + res, err := ctrlSvc.GetCapacity(context.Background(), req) + + Expect(res.MaximumVolumeSize).ToNot(BeNil()) + Expect(err).To(BeNil()) + }) + }) + }) Describe("calling ValidateVolumeCapabilities()", func() {