Skip to content

Commit

Permalink
Merge pull request #364 from dell/usr/harshitap26/node_labels_integra…
Browse files Browse the repository at this point in the history 
…tiontests

Integration Tests for Node Topologies
  • Loading branch information
@harshitap26
harshitap26 authored Dec 6, 2024
2 parents f20f990 + 3ca1972 commit 366b0eb
Show file tree
Hide file tree
Showing 4 changed files with 160 additions and 29 deletions.
4 changes: 4 additions & 0 deletions env.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,14 @@ export NFS_STORAGE_POOL=""
export SDC_GUID=$(/bin/emc/scaleio/drv_cfg --query_guid)
# Alternate GUID is for another system for testing expose volume to multiple hosts
export ALT_GUID=
export X_CSI_POWERFLEX_KUBE_NODE_NAME="node1"

# Interface variables
export NODE_INTERFACES="nodeName:interfaceName"

# Node Label variables
export ZONE_LABEL_KEY=""

#Debug variables for goscaleio library
export GOSCALEIO_SHOWHTTP="true"

Expand Down
13 changes: 13 additions & 0 deletions test/integration/features/integration.feature
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1004,3 +1004,16 @@ Scenario: Create zone volume with invalid zone information
Examples:
| errormsg |
| "no zone topology found in accessibility requirements" |

@zone-integration
Scenario: call NodeGetInfo
Given a VxFlexOS service
And I call NodeGetInfo with ""
Then a NodeGetInfo is returned with zone topology
And a NodeGetInfo is returned with system topology

@zone-integration
Scenario: call NodeGetInfo
Given a VxFlexOS service
And I call NodeGetInfo with "invalid_zone_key"
Then a NodeGetInfo is returned without zone topology "invalid_zone_key"
21 changes: 12 additions & 9 deletions test/integration/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,21 +31,22 @@ import (
)

const (
datafile = "/tmp/datafile"
datadir = "/tmp/datadir"
configFile = "../../config.json"
datafile = "/tmp/datafile"
datadir = "/tmp/datadir"
configFile = "../../config.json"
zoneConfigFile = "features/array-config/multi-az"
)

var grpcClient *grpc.ClientConn

func init() {
func readConfigFile(filePath string) {
/* load array config and give proper errors if not ok*/
if _, err := os.Stat(configFile); err == nil {
if _, err := os.ReadFile(configFile); err != nil {
err = fmt.Errorf("DEBUG integration pre requisites missing %s with multi array configuration file ", configFile)
if _, err := os.Stat(filePath); err == nil {
if _, err := os.ReadFile(filePath); err != nil {
err = fmt.Errorf("DEBUG integration pre requisites missing %s with multi array configuration file ", filePath)
panic(err)
}
f, err := os.Open(configFile)
f, err := os.Open(filePath)
r := bufio.NewReader(f)
mdms := make([]string, 0)
line, isPrefix, err := r.ReadLine()
Expand All @@ -70,6 +71,7 @@ func init() {
}

func TestIntegration(t *testing.T) {
readConfigFile(configFile)
var stop func()
ctx := context.Background()
fmt.Printf("calling startServer")
Expand Down Expand Up @@ -118,10 +120,11 @@ func TestIntegration(t *testing.T) {
}

func TestZoneIntegration(t *testing.T) {
readConfigFile(zoneConfigFile)
var stop func()
ctx := context.Background()
fmt.Printf("calling startServer")
grpcClient, stop = startServer(ctx, "features/array-config/multi-az")
grpcClient, stop = startServer(ctx, zoneConfigFile)
fmt.Printf("back from startServer")
time.Sleep(5 * time.Second)

Expand Down
151 changes: 131 additions & 20 deletions test/integration/step_defs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ type feature struct {
createVolumeRequest *csi.CreateVolumeRequest
publishVolumeRequest *csi.ControllerPublishVolumeRequest
nodePublishVolumeRequest *csi.NodePublishVolumeRequest
nodeGetInfoRequest *csi.NodeGetInfoRequest
nodeGetInfoResponse *csi.NodeGetInfoResponse
listVolumesResponse *csi.ListVolumesResponse
listSnapshotsResponse *csi.ListSnapshotsResponse
capability *csi.VolumeCapability
Expand All @@ -103,7 +105,7 @@ func (f *feature) getGoscaleioClient() (client *goscaleio.Client, err error) {
if f.arrays == nil {
fmt.Printf("Initialize ArrayConfig from %s:\n", configFile)
var err error
f.arrays, err = f.getArrayConfig()
f.arrays, err = f.getArrayConfig(configFile)
if err != nil {
return nil, errors.New("Get multi array config failed " + err.Error())
}
Expand Down Expand Up @@ -137,19 +139,19 @@ func (f *feature) getGoscaleioClient() (client *goscaleio.Client, err error) {
// there is no way to call service.go methods from here
// hence copy same method over there , this is used to get all arrays and pick different
// systemID to test with see method iSetAnotherSystemID
func (f *feature) getArrayConfig() (map[string]*ArrayConnectionData, error) {
func (f *feature) getArrayConfig(filePath string) (map[string]*ArrayConnectionData, error) {
arrays := make(map[string]*ArrayConnectionData)

_, err := os.Stat(configFile)
_, err := os.Stat(filePath)
if err != nil {
if os.IsNotExist(err) {
return nil, fmt.Errorf("File %s does not exist", configFile)
return nil, fmt.Errorf("File %s does not exist", filePath)
}
}

config, err := os.ReadFile(filepath.Clean(configFile))
config, err := os.ReadFile(filepath.Clean(filePath))
if err != nil {
return nil, fmt.Errorf("File %s errors: %v", configFile, err)
return nil, fmt.Errorf("File %s errors: %v", filePath, err)
}

if string(config) != "" {
Expand All @@ -160,7 +162,7 @@ func (f *feature) getArrayConfig() (map[string]*ArrayConnectionData, error) {
}

if len(jsonCreds) == 0 {
return nil, fmt.Errorf("no arrays are provided in configFile %s", configFile)
return nil, fmt.Errorf("no arrays are provided in configFile %s", filePath)
}

noOfDefaultArray := 0
Expand Down Expand Up @@ -219,7 +221,7 @@ func (f *feature) getArrayConfig() (map[string]*ArrayConnectionData, error) {
arrays[c.SystemID] = &copyOfCred
}
} else {
return nil, fmt.Errorf("arrays details are not provided in configFile %s", configFile)
return nil, fmt.Errorf("arrays details are not provided in configFile %s", filePath)
}
return arrays, nil
}
Expand Down Expand Up @@ -318,6 +320,8 @@ func (f *feature) aVxFlexOSService() error {
f.errs = make([]error, 0)
f.createVolumeRequest = nil
f.publishVolumeRequest = nil
f.nodeGetInfoRequest = nil
f.nodeGetInfoResponse = nil
f.listVolumesResponse = nil
f.listSnapshotsResponse = nil
f.capability = nil
Expand Down Expand Up @@ -374,7 +378,7 @@ func (f *feature) aBasicNfsVolumeRequest(name string, size int64) error {
if f.arrays == nil {
fmt.Printf("Initialize ArrayConfig from %s:\n", configFile)
var err error
f.arrays, err = f.getArrayConfig()
f.arrays, err = f.getArrayConfig(configFile)
if err != nil {
return errors.New("Get multi array config failed " + err.Error())
}
Expand Down Expand Up @@ -439,7 +443,7 @@ func (f *feature) aBasicNfsVolumeRequestWithSizeLessThan3Gi(name string, size in
if f.arrays == nil {
fmt.Printf("Initialize ArrayConfig from %s:\n", configFile)
var err error
f.arrays, err = f.getArrayConfig()
f.arrays, err = f.getArrayConfig(configFile)
if err != nil {
return errors.New("Get multi array config failed " + err.Error())
}
Expand Down Expand Up @@ -504,7 +508,7 @@ func (f *feature) aNfsVolumeRequestWithQuota(volname string, volsize int64, path
if f.arrays == nil {
fmt.Printf("Initialize ArrayConfig from %s:\n", configFile)
var err error
f.arrays, err = f.getArrayConfig()
f.arrays, err = f.getArrayConfig(configFile)
if err != nil {
return errors.New("Get multi array config failed " + err.Error())
}
Expand Down Expand Up @@ -1263,7 +1267,7 @@ func (f *feature) iSetAnotherSystemName(systemType string) error {
if f.arrays == nil {
fmt.Printf("Initialize ArrayConfig from %s:\n", configFile)
var err error
f.arrays, err = f.getArrayConfig()
f.arrays, err = f.getArrayConfig(configFile)
if err != nil {
return errors.New("Get multi array config failed " + err.Error())
}
Expand Down Expand Up @@ -1294,7 +1298,7 @@ func (f *feature) iSetAnotherSystemID(systemType string) error {
if f.arrays == nil {
fmt.Printf("Initialize ArrayConfig from %s:\n", configFile)
var err error
f.arrays, err = f.getArrayConfig()
f.arrays, err = f.getArrayConfig(configFile)
if err != nil {
return errors.New("Get multi array config failed " + err.Error())
}
Expand Down Expand Up @@ -2100,7 +2104,7 @@ func (f *feature) aBasicNfsVolumeRequestWithWrongNasName(name string, size int64
if f.arrays == nil {
fmt.Printf("Initialize ArrayConfig from %s:\n", configFile)
var err error
f.arrays, err = f.getArrayConfig()
f.arrays, err = f.getArrayConfig(configFile)
if err != nil {
return errors.New("Get multi array config failed " + err.Error())
}
Expand Down Expand Up @@ -2162,7 +2166,7 @@ func (f *feature) aNfsCapabilityWithVoltypeAccessFstype(voltype, access, fstype
if f.arrays == nil {
fmt.Printf("Initialize ArrayConfig from %s:\n", configFile)
var err error
f.arrays, err = f.getArrayConfig()
f.arrays, err = f.getArrayConfig(configFile)
if err != nil {
return errors.New("Get multi array config failed " + err.Error())
}
Expand Down Expand Up @@ -2227,7 +2231,7 @@ func (f *feature) aNfsVolumeRequest(name string, size int64) error {
if f.arrays == nil {
fmt.Printf("Initialize ArrayConfig from %s:\n", configFile)
var err error
f.arrays, err = f.getArrayConfig()
f.arrays, err = f.getArrayConfig(configFile)
if err != nil {
return errors.New("Get multi array config failed " + err.Error())
}
Expand Down Expand Up @@ -2327,7 +2331,7 @@ func (f *feature) controllerPublishVolumeForNfsWithoutSDC(id string) error {
if f.arrays == nil {
fmt.Printf("Initialize ArrayConfig from %s:\n", configFile)
var err error
f.arrays, err = f.getArrayConfig()
f.arrays, err = f.getArrayConfig(configFile)
if err != nil {
return errors.New("Get multi array config failed " + err.Error())
}
Expand Down Expand Up @@ -2357,7 +2361,7 @@ func (f *feature) controllerPublishVolumeForNfs(id string, nodeIDEnvVar string)
if f.arrays == nil {
fmt.Printf("Initialize ArrayConfig from %s:\n", configFile)
var err error
f.arrays, err = f.getArrayConfig()
f.arrays, err = f.getArrayConfig(configFile)
if err != nil {
return errors.New("Get multi array config failed " + err.Error())
}
Expand Down Expand Up @@ -2609,7 +2613,7 @@ func (f *feature) ICallListFileSystemSnapshot() error {
if f.arrays == nil {
fmt.Printf("Initialize ArrayConfig from %s:\n", configFile)
var err error
f.arrays, err = f.getArrayConfig()
f.arrays, err = f.getArrayConfig(configFile)
if err != nil {
return errors.New("Get multi array config failed " + err.Error())
}
Expand Down Expand Up @@ -2721,7 +2725,7 @@ func (f *feature) checkNFS(_ context.Context, systemID string) (bool, error) {
return false, err
}
if ver >= 4.0 {
arrayConData, err := f.getArrayConfig()
arrayConData, err := f.getArrayConfig(configFile)
if err != nil {
return false, err
}
Expand All @@ -2734,6 +2738,109 @@ func (f *feature) checkNFS(_ context.Context, systemID string) (bool, error) {
return false, nil
}

func (f *feature) createFakeNodeLabels(zoneLabelKey string) error {
if zoneLabelKey == "" {
zoneLabelKey = os.Getenv("ZONE_LABEL_KEY")
}
nodeName := os.Getenv("X_CSI_POWERFLEX_KUBE_NODE_NAME")

node := &apiv1.Node{
ObjectMeta: v1.ObjectMeta{
Name: nodeName,
Labels: map[string]string{
zoneLabelKey: "zoneA",
"kubernetes.io/hostname": nodeName,
},
},
Status: apiv1.NodeStatus{
Conditions: []apiv1.NodeCondition{
{
Type: apiv1.NodeReady,
Status: apiv1.ConditionTrue,
},
},
},
}
_, err := service.K8sClientset.CoreV1().Nodes().Create(context.TODO(), node, v1.CreateOptions{})
if err != nil {
fmt.Printf("CreateNode returned error: %s\n", err.Error())
return err
}
return nil
}

func (f *feature) iCallNodeGetInfo(zoneLabelKey string) error {
fmt.Println("[iCallNodeGetInfo] Calling NodeGetInfo...")
_, err := f.nodeGetInfo(f.nodeGetInfoRequest, zoneLabelKey)
if err != nil {
fmt.Printf("NodeGetInfo returned error: %s\n", err.Error())
f.addError(err)
}
return nil
}

func (f *feature) nodeGetInfo(req *csi.NodeGetInfoRequest, zoneLabelKey string) (*csi.NodeGetInfoResponse, error) {
fmt.Println("[nodeGetInfo] Calling NodeGetInfo...")

ctx := context.Background()
client := csi.NewNodeClient(grpcClient)
var nodeResp *csi.NodeGetInfoResponse

clientSet := fake.NewSimpleClientset()
service.K8sClientset = clientSet

err := f.createFakeNodeLabels(zoneLabelKey)
if err != nil {
return nil, fmt.Errorf("failed to create fake node labels: %v", err)
}

nodeResp, err = client.NodeGetInfo(ctx, req)
f.nodeGetInfoResponse = nodeResp
return nodeResp, err
}

func (f *feature) aNodeGetInfoIsReturnedWithZoneTopology() error {
accessibility := f.nodeGetInfoResponse.GetAccessibleTopology()
log.Printf("Node Accessibility %+v", accessibility)
if _, ok := accessibility.Segments[os.Getenv("ZONE_LABEL_KEY")]; !ok {
return fmt.Errorf("zone not found")
}
return nil
}

func (f *feature) aNodeGetInfoIsReturnedWithoutZoneTopology(zoneLabelKey string) error {
accessibility := f.nodeGetInfoResponse.GetAccessibleTopology()
log.Printf("Node Accessibility %+v", accessibility)
if _, ok := accessibility.Segments[zoneLabelKey]; ok {
return fmt.Errorf("zone found")
}
return nil
}

func (f *feature) aNodeGetInfoIsReturnedWithSystemTopology() error {
accessibility := f.nodeGetInfoResponse.GetAccessibleTopology()
log.Printf("Node Accessibility %+v", accessibility)

var err error
f.arrays, err = f.getArrayConfig(zoneConfigFile)
if err != nil {
return fmt.Errorf("failed to get array config: %v", err)
}

labelAdded := false
for _, array := range f.arrays {
log.Printf("array systemID %+v", array.SystemID)
if _, ok := accessibility.Segments[service.Name+"/"+array.SystemID]; ok {
labelAdded = true
}
}

if !labelAdded {
return fmt.Errorf("topology with zone label not found")
}
return nil
}

func (f *feature) iCreateZoneRequest(name string) error {
req := f.createGenericZoneRequest(name)
req.AccessibilityRequirements = new(csi.TopologyRequirement)
Expand Down Expand Up @@ -2883,4 +2990,8 @@ func FeatureContext(s *godog.ScenarioContext) {
s.Step(`^I call DeleteSnapshotForFS$`, f.iCallDeleteSnapshotForFS)
s.Step(`^I create a zone volume request "([^"]*)"$`, f.iCreateZoneRequest)
s.Step(`^I create an invalid zone volume request$`, f.iCreateInvalidZoneRequest)
s.Step(`^I call NodeGetInfo with "([^"]*)"$`, f.iCallNodeGetInfo)
s.Step(`^a NodeGetInfo is returned with zone topology$`, f.aNodeGetInfoIsReturnedWithZoneTopology)
s.Step(`^a NodeGetInfo is returned without zone topology "([^"]*)"$`, f.aNodeGetInfoIsReturnedWithoutZoneTopology)
s.Step(`^a NodeGetInfo is returned with system topology$`, f.aNodeGetInfoIsReturnedWithSystemTopology)
}

0 comments on commit 366b0eb

Please sign in to comment.