diff --git a/.gitignore b/.gitignore index 7279dc751..b83bc5abe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ # Temporary Build Files build/_output build/_test -test/v200/schemaTest/tmp +test/**/tmp test/go/pkg # Created by https://www.gitignore.io/api/go,vim,emacs,visualstudiocode ### Emacs ### diff --git a/test/README.md b/test/README.md index 672f06715..62ef8e739 100644 --- a/test/README.md +++ b/test/README.md @@ -1,4 +1,4 @@ -# API Tests +# schemaTest The API tests are intended to provide a comprehensive verification of the devfile schemas. This includes: - Ensuring every possible attribute is valid. @@ -44,3 +44,27 @@ The test will read each of the test-xxxxxx.json files and run the tests defined 1. Modify the copied tests as needed for the new version as decsribed above. 1. Add `test/v201/schemaTest/tmp` to the .gitignore file. 1. Run the test + + +# apiTest + +A new test approach, shared with the library repository for testing valid devfiles. Basically the test creates lots of valid devfiles whith different content. The attributes which are set and the values to which they are set are randomized. These tests are a work in progress and the intent is to eventually replace schemaTest. + +## Test structure + +- `test/v200/apiTest/api-test.go`: The go unit test program +- `test/v200/utils/api/test-utils.go` : utilites, used by the test, which contain functions uniqiue to the api tests. +- `test/v200/utils/common/*-utils.go` : utilites, used by the test, which are also used by the library tests. Mostly contain the code to generate valid devfile content. + + +## Running tests + +from the `test/v200/apiTest/` directory run +- `go test -v` + +* The test will generate a set of valid devfile.yaml files in `test/v200/apiTest/tmp/api-test/ +* The test will generate a log file: `test/v200/apiTest/tmp/test.log` +* Each run of the test removes the `test/v200/apiTest/tmp` directory from the previous run. + + + diff --git a/test/v200/apiTest/api_test.go b/test/v200/apiTest/api_test.go new file mode 100644 index 000000000..f94b0bde2 --- /dev/null +++ b/test/v200/apiTest/api_test.go @@ -0,0 +1,80 @@ +package apiTest + +import ( + "testing" + + schema "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2" + apiUtils "github.com/devfile/api/v2/test/v200/utils/api" + commonUtils "github.com/devfile/api/v2/test/v200/utils/common" +) + +func Test_ExecCommand(t *testing.T) { + testContent := commonUtils.TestContent{} + testContent.CommandTypes = []schema.CommandType{schema.ExecCommandType} + testContent.EditContent = false + testContent.FileName = commonUtils.GetDevFileName() + apiUtils.RunTest(testContent, t) +} + +func Test_ApplyCommand(t *testing.T) { + testContent := commonUtils.TestContent{} + testContent.CommandTypes = []schema.CommandType{schema.ApplyCommandType} + testContent.EditContent = false + testContent.FileName = commonUtils.GetDevFileName() + apiUtils.RunTest(testContent, t) +} + +func Test_CompositeCommand(t *testing.T) { + testContent := commonUtils.TestContent{} + testContent.CommandTypes = []schema.CommandType{schema.CompositeCommandType} + testContent.EditContent = false + testContent.FileName = commonUtils.GetDevFileName() + apiUtils.RunTest(testContent, t) +} + +func Test_MultiCommand(t *testing.T) { + testContent := commonUtils.TestContent{} + testContent.CommandTypes = []schema.CommandType{schema.ExecCommandType, + schema.CompositeCommandType, + schema.ApplyCommandType} + testContent.EditContent = true + testContent.FileName = commonUtils.GetDevFileName() + apiUtils.RunTest(testContent, t) +} + +func Test_ContainerComponent(t *testing.T) { + testContent := commonUtils.TestContent{} + testContent.ComponentTypes = []schema.ComponentType{schema.ContainerComponentType} + testContent.EditContent = false + testContent.FileName = commonUtils.GetDevFileName() + apiUtils.RunTest(testContent, t) +} + +func Test_VolumeComponent(t *testing.T) { + testContent := commonUtils.TestContent{} + testContent.ComponentTypes = []schema.ComponentType{schema.VolumeComponentType} + testContent.FileName = commonUtils.GetDevFileName() + apiUtils.RunTest(testContent, t) +} + +func Test_MultiComponent(t *testing.T) { + testContent := commonUtils.TestContent{} + testContent.ComponentTypes = []schema.ComponentType{ + schema.ContainerComponentType, + schema.VolumeComponentType} + testContent.FileName = commonUtils.GetDevFileName() + apiUtils.RunTest(testContent, t) +} + +func Test_Everything(t *testing.T) { + testContent := commonUtils.TestContent{} + testContent.CommandTypes = []schema.CommandType{ + schema.ExecCommandType, + schema.CompositeCommandType, + schema.ApplyCommandType} + testContent.ComponentTypes = []schema.ComponentType{ + schema.ContainerComponentType, + schema.VolumeComponentType} + testContent.FileName = commonUtils.GetDevFileName() + apiUtils.RunTest(testContent, t) +} diff --git a/test/v200/utils/api/test_utils.go b/test/v200/utils/api/test_utils.go new file mode 100644 index 000000000..e450eb791 --- /dev/null +++ b/test/v200/utils/api/test_utils.go @@ -0,0 +1,175 @@ +package api + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "strconv" + "strings" + "testing" + + commonUtils "github.com/devfile/api/v2/test/v200/utils/common" + "github.com/santhosh-tekuri/jsonschema" + "sigs.k8s.io/yaml" +) + +const ( + // numDevfiles : the number of devfiles to create for each test + numDevfiles = 5 + + schemaFileName = "../../../schemas/latest/ide-targeted/devfile.json" +) + +var schemas = make(map[string]SchemaFile) + +// SchemaFile - represents the schema stucture +type SchemaFile struct { + Schema *jsonschema.Schema +} + +// DevfileValidator struct for DevfileValidator interface defined in common utils. +type DevfileValidator struct{} + +// WriteAndValidate implements Saved.DevfileValidator interface. +// writes to disk and validates the specified devfile +func (devfileValidator DevfileValidator) WriteAndValidate(devfile *commonUtils.TestDevfile) error { + err := writeDevfile(devfile) + if err != nil { + commonUtils.LogErrorMessage(fmt.Sprintf("Error writing file : %s : %v", devfile.FileName, err)) + } else { + err = validateDevfile(devfile) + if err != nil { + commonUtils.LogErrorMessage(fmt.Sprintf("Error vaidating file : %s : %v", devfile.FileName, err)) + } + } + return err +} + +// checkWithSchema checks the validity of a devfile against the schema. +func (schemaFile *SchemaFile) checkWithSchema(devfile string, expectedMessage string) error { + + // Read the created yaml file, ready for converison to json + devfileData, err := ioutil.ReadFile(devfile) + if err != nil { + commonUtils.LogErrorMessage(fmt.Sprintf(" FAIL: schema : unable to read %s: %v", devfile, err)) + return err + } + + // Convert the yaml file to json + devfileDataAsJSON, err := yaml.YAMLToJSON(devfileData) + if err != nil { + commonUtils.LogErrorMessage(fmt.Sprintf(" FAIL : %s : schema : failed to convert to json : %v", devfile, err)) + return err + } + + validationErr := schemaFile.Schema.Validate(bytes.NewReader(devfileDataAsJSON)) + if validationErr != nil { + if len(expectedMessage) > 0 { + if !strings.Contains(validationErr.Error(), expectedMessage) { + err = errors.New(commonUtils.LogErrorMessage(fmt.Sprintf(" FAIL : schema : %s : Did not fail as expected : %s got : %v", devfile, expectedMessage, validationErr))) + } else { + commonUtils.LogInfoMessage(fmt.Sprintf("PASS: schema : Expected Error received : %s", expectedMessage)) + } + } else { + err = errors.New(commonUtils.LogErrorMessage(fmt.Sprintf(" FAIL : schema : %s : Did not pass as expected, got : %v", devfile, validationErr))) + } + } else { + if len(expectedMessage) > 0 { + err = errors.New(commonUtils.LogErrorMessage(fmt.Sprintf(" FAIL : schema : %s : was valid - Expected Error not found : %v", devfile, validationErr))) + } else { + commonUtils.LogInfoMessage(fmt.Sprintf(" PASS : schema : %s : devfile was valid.", devfile)) + } + } + return err +} + +// getSchema downloads and saves a schema from the provided url +func getSchema(schemaFileName string) (SchemaFile, error) { + + var err error + schemaFile, found := schemas[schemaFileName] + if !found { + + schemaFile = SchemaFile{} + + // Prepare the schema file + compiler := jsonschema.NewCompiler() + // Use Draft 7, github.com/santhosh-tekuri/jsonschema provides 4,6 an 7 so use the latest + compiler.Draft = jsonschema.Draft7 + schemaFile.Schema, err = compiler.Compile(schemaFileName) + if err != nil { + commonUtils.LogErrorMessage(fmt.Sprintf("FAIL : Failed to compile schema %v", err)) + } else { + commonUtils.LogInfoMessage(fmt.Sprintf("Schema compiled from file: %s)", schemaFileName)) + schemas[schemaFileName] = schemaFile + } + } + return schemaFile, err +} + +// writeDevfile creates a devfile on disk for use in a test. +func writeDevfile(devfile *commonUtils.TestDevfile) error { + var err error + + fileName := devfile.FileName + if !strings.HasSuffix(fileName, ".yaml") { + fileName += ".yaml" + } + + commonUtils.LogInfoMessage(fmt.Sprintf("Marshall and write devfile %s", devfile.FileName)) + + c, marshallErr := yaml.Marshal(&(devfile.SchemaDevFile)) + + if marshallErr != nil { + err = errors.New(commonUtils.LogErrorMessage(fmt.Sprintf("Marshall devfile %s : %v", devfile.FileName, marshallErr))) + } else { + err = ioutil.WriteFile(fileName, c, 0644) + if err != nil { + commonUtils.LogErrorMessage(fmt.Sprintf("Write devfile %s : %v", devfile.FileName, err)) + } + } + return err +} + +// validateDevfile check the provided defile against the schema +func validateDevfile(devfile *commonUtils.TestDevfile) error { + + var err error + var schemaFile SchemaFile + + schemaFile, err = getSchema(schemaFileName) + if err != nil { + commonUtils.LogErrorMessage(fmt.Sprintf("Failed to get devfile schema : %v", err)) + } else { + err = schemaFile.checkWithSchema(devfile.FileName, "") + if err != nil { + commonUtils.LogErrorMessage(fmt.Sprintf("Verification with devfile schema failed : %v", err)) + } else { + commonUtils.LogInfoMessage(fmt.Sprintf("Devfile validated using JSONSchema schema : %s", devfile.FileName)) + } + } + + return err +} + +// RunTest : Runs a test to create and verify a devfile based on the content of the specified TestContent +func RunTest(testContent commonUtils.TestContent, t *testing.T) { + + commonUtils.LogMessage(fmt.Sprintf("Start test for %s", testContent.FileName)) + + validator := DevfileValidator{} + + devfileName := testContent.FileName + for i := 1; i <= numDevfiles; i++ { + + testContent.FileName = commonUtils.AddSuffixToFileName(devfileName, strconv.Itoa(i)) + + testDevfile, err := commonUtils.GetDevfile(testContent.FileName, nil, validator) + if err != nil { + t.Fatalf(commonUtils.LogMessage(fmt.Sprintf("Error creating devfile : %v", err))) + } + + testDevfile.RunTest(testContent, t) + } +} diff --git a/test/v200/utils/common/command_test_utils.go b/test/v200/utils/common/command_test_utils.go new file mode 100644 index 000000000..70d9fbdca --- /dev/null +++ b/test/v200/utils/common/command_test_utils.go @@ -0,0 +1,213 @@ +package common + +import ( + "fmt" + + schema "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2" +) + +// commandAdded adds a new command to the test schema data and notifies the follower +func (testDevFile *TestDevfile) commandAdded(command schema.Command) { + LogInfoMessage(fmt.Sprintf("command added Id: %s", command.Id)) + testDevFile.SchemaDevFile.Commands = append(testDevFile.SchemaDevFile.Commands, command) + if testDevFile.Follower != nil { + testDevFile.Follower.AddCommand(command) + } +} + +// commandUpdated notifies the follower of the command which has been updated +func (testDevFile *TestDevfile) commandUpdated(command schema.Command) { + LogInfoMessage(fmt.Sprintf("command updated Id: %s", command.Id)) + if testDevFile.Follower != nil { + testDevFile.Follower.UpdateCommand(command) + } +} + +// addEnv creates and returns a specifed number of env attributes in a schema structure +func addEnv(numEnv int) []schema.EnvVar { + commandEnvs := make([]schema.EnvVar, numEnv) + for i := 0; i < numEnv; i++ { + commandEnvs[i].Name = "Name_" + GetRandomString(5, false) + commandEnvs[i].Value = "Value_" + GetRandomString(5, false) + LogInfoMessage(fmt.Sprintf("Add Env: %s", commandEnvs[i])) + } + return commandEnvs +} + +// addAttributes creates returns a specifed number of attributes in a schema structure +func addAttributes(numAtrributes int) map[string]string { + attributes := make(map[string]string) + for i := 0; i < numAtrributes; i++ { + AttributeName := "Name_" + GetRandomString(6, false) + attributes[AttributeName] = "Value_" + GetRandomString(6, false) + LogInfoMessage(fmt.Sprintf("Add attribute : %s = %s", AttributeName, attributes[AttributeName])) + } + return attributes +} + +// addGroup creates and returns a group in a schema structure +func (testDevFile *TestDevfile) addGroup() *schema.CommandGroup { + + commandGroup := schema.CommandGroup{} + commandGroup.Kind = GetRandomGroupKind() + LogInfoMessage(fmt.Sprintf("group Kind: %s, default already set %t", commandGroup.Kind, testDevFile.GroupDefaults[commandGroup.Kind])) + // Ensure only one and at least one of each type are labelled as default + if !testDevFile.GroupDefaults[commandGroup.Kind] { + testDevFile.GroupDefaults[commandGroup.Kind] = true + commandGroup.IsDefault = true + } else { + commandGroup.IsDefault = false + } + LogInfoMessage(fmt.Sprintf("group isDefault: %t", commandGroup.IsDefault)) + return &commandGroup +} + +// AddCommand creates a command of a specified type in a schema structure and pupulates it with random attributes +func (testDevFile *TestDevfile) AddCommand(commandType schema.CommandType) schema.Command { + + var command *schema.Command + switch commandType { + case schema.ExecCommandType: + command = testDevFile.createExecCommand() + testDevFile.SetExecCommandValues(command) + case schema.CompositeCommandType: + command = testDevFile.createCompositeCommand() + testDevFile.SetCompositeCommandValues(command) + case schema.ApplyCommandType: + command = testDevFile.createApplyCommand() + testDevFile.SetApplyCommandValues(command) + } + return *command +} + +// createExecCommand creates and returns an empty exec command in a schema structure +func (testDevFile *TestDevfile) createExecCommand() *schema.Command { + + LogInfoMessage("Create an exec command :") + command := schema.Command{} + command.Id = GetRandomUniqueString(8, true) + LogInfoMessage(fmt.Sprintf("command Id: %s", command.Id)) + command.Exec = &schema.ExecCommand{} + testDevFile.commandAdded(command) + return &command + +} + +// SetExecCommandValues randomly sets/updates exec command attributes to random values +func (testDevFile *TestDevfile) SetExecCommandValues(command *schema.Command) { + + execCommand := command.Exec + + // exec command must be mentioned by a container component + execCommand.Component = testDevFile.GetContainerName() + + execCommand.CommandLine = GetRandomString(4, false) + " " + GetRandomString(4, false) + LogInfoMessage(fmt.Sprintf("....... commandLine: %s", execCommand.CommandLine)) + + // If group already leave it to make sure defaults are not deleted or added + if execCommand.Group == nil { + if GetRandomDecision(2, 1) { + execCommand.Group = testDevFile.addGroup() + } + } + + if GetBinaryDecision() { + execCommand.Label = GetRandomString(12, false) + LogInfoMessage(fmt.Sprintf("....... label: %s", execCommand.Label)) + } else { + execCommand.Label = "" + } + + if GetBinaryDecision() { + execCommand.WorkingDir = "./tmp" + LogInfoMessage(fmt.Sprintf("....... WorkingDir: %s", execCommand.WorkingDir)) + } else { + execCommand.WorkingDir = "" + } + + execCommand.HotReloadCapable = GetBinaryDecision() + LogInfoMessage(fmt.Sprintf("....... HotReloadCapable: %t", execCommand.HotReloadCapable)) + + if GetBinaryDecision() { + execCommand.Env = addEnv(GetRandomNumber(1, 4)) + } else { + execCommand.Env = nil + } + testDevFile.commandUpdated(*command) + +} + +// createCompositeCommand creates an empty composite command in a schema structure +func (testDevFile *TestDevfile) createCompositeCommand() *schema.Command { + + LogInfoMessage("Create a composite command :") + command := schema.Command{} + command.Id = GetRandomUniqueString(8, true) + LogInfoMessage(fmt.Sprintf("command Id: %s", command.Id)) + command.Composite = &schema.CompositeCommand{} + testDevFile.commandAdded(command) + + return &command +} + +// SetCompositeCommandValues randomly sets/updates composite command attributes to random values +func (testDevFile *TestDevfile) SetCompositeCommandValues(command *schema.Command) { + + compositeCommand := command.Composite + numCommands := GetRandomNumber(1, 3) + + for i := 0; i < numCommands; i++ { + execCommand := testDevFile.AddCommand(schema.ExecCommandType) + compositeCommand.Commands = append(compositeCommand.Commands, execCommand.Id) + LogInfoMessage(fmt.Sprintf("....... command %d of %d : %s", i, numCommands, execCommand.Id)) + } + + // If group already exists - leave it to make sure defaults are not deleted or added + if compositeCommand.Group == nil { + if GetRandomDecision(2, 1) { + compositeCommand.Group = testDevFile.addGroup() + } + } + + if GetBinaryDecision() { + compositeCommand.Label = GetRandomString(12, false) + LogInfoMessage(fmt.Sprintf("....... label: %s", compositeCommand.Label)) + } + + if GetBinaryDecision() { + compositeCommand.Parallel = true + LogInfoMessage(fmt.Sprintf("....... Parallel: %t", compositeCommand.Parallel)) + } + + testDevFile.commandUpdated(*command) +} + +// createApplyCommand creates an apply command in a schema structure +func (testDevFile *TestDevfile) createApplyCommand() *schema.Command { + + LogInfoMessage("Create a apply command :") + command := schema.Command{} + command.Id = GetRandomUniqueString(8, true) + LogInfoMessage(fmt.Sprintf("command Id: %s", command.Id)) + command.Apply = &schema.ApplyCommand{} + testDevFile.commandAdded(command) + return &command +} + +// SetApplyCommandValues randomly sets/updates apply command attributes to random values +func (testDevFile *TestDevfile) SetApplyCommandValues(command *schema.Command) { + applyCommand := command.Apply + + applyCommand.Component = testDevFile.GetContainerName() + + if GetRandomDecision(2, 1) { + applyCommand.Group = testDevFile.addGroup() + } + + if GetBinaryDecision() { + applyCommand.Label = GetRandomString(63, false) + LogInfoMessage(fmt.Sprintf("....... label: %s", applyCommand.Label)) + } + + testDevFile.commandUpdated(*command) +} diff --git a/test/v200/utils/common/component_test_utils.go b/test/v200/utils/common/component_test_utils.go new file mode 100644 index 000000000..2a8f5c903 --- /dev/null +++ b/test/v200/utils/common/component_test_utils.go @@ -0,0 +1,172 @@ +package common + +import ( + "fmt" + "strconv" + + schema "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2" +) + +// componentAdded adds a new component to the test schema data and to the parser data +func (devfile *TestDevfile) componentAdded(component schema.Component) { + LogInfoMessage(fmt.Sprintf("component added Name: %s", component.Name)) + devfile.SchemaDevFile.Components = append(devfile.SchemaDevFile.Components, component) + if devfile.Follower != nil { + devfile.Follower.AddComponent(component) + } +} + +// componetUpdated updates a component in the parser data +func (devfile *TestDevfile) componentUpdated(component schema.Component) { + LogInfoMessage(fmt.Sprintf("component updated Name: %s", component.Name)) + if devfile.Follower != nil { + devfile.Follower.UpdateComponent(component) + } +} + +// addVolume returns volumeMounts in a schema structure based on a specified number of volumes +func (devfile *TestDevfile) addVolume(numVols int) []schema.VolumeMount { + commandVols := make([]schema.VolumeMount, numVols) + for i := 0; i < numVols; i++ { + volumeComponent := devfile.AddComponent(schema.VolumeComponentType) + commandVols[i].Name = volumeComponent.Name + commandVols[i].Path = "/Path_" + GetRandomString(5, false) + LogInfoMessage(fmt.Sprintf("....... Add Volume: %s", commandVols[i])) + } + return commandVols +} + +// AddComponent adds a component of the specified type, with random attributes, to the devfile schema +func (devfile *TestDevfile) AddComponent(componentType schema.ComponentType) schema.Component { + + var component schema.Component + if componentType == schema.ContainerComponentType { + component = devfile.createContainerComponent() + devfile.SetContainerComponentValues(&component) + } else if componentType == schema.VolumeComponentType { + component = devfile.createVolumeComponent() + devfile.SetVolumeComponentValues(&component) + } + return component +} + +// createContainerComponent creates a container component, ready for attribute setting +func (devfile *TestDevfile) createContainerComponent() schema.Component { + + LogInfoMessage("Create a container component :") + component := schema.Component{} + component.Name = GetRandomUniqueString(8, true) + LogInfoMessage(fmt.Sprintf("....... Name: %s", component.Name)) + component.Container = &schema.ContainerComponent{} + devfile.componentAdded(component) + return component + +} + +// createVolumeComponent creates a volume component , ready for attribute setting +func (devfile *TestDevfile) createVolumeComponent() schema.Component { + + LogInfoMessage("Create a volume component :") + component := schema.Component{} + component.Name = GetRandomUniqueString(8, true) + LogInfoMessage(fmt.Sprintf("....... Name: %s", component.Name)) + component.Volume = &schema.VolumeComponent{} + devfile.componentAdded(component) + return component + +} + +// GetContainer returns the name of an existing, or newly created, container. +func (devfile *TestDevfile) GetContainerName() string { + + componentName := "" + for _, currentComponent := range devfile.SchemaDevFile.Components { + if currentComponent.Container != nil { + componentName = currentComponent.Name + LogInfoMessage(fmt.Sprintf("return existing container from GetContainerName : %s", componentName)) + break + } + } + + if componentName == "" { + component := devfile.createContainerComponent() + component.Container.Image = GetRandomUniqueString(GetRandomNumber(8, 18), false) + componentName = component.Name + LogInfoMessage(fmt.Sprintf("retrun new container from GetContainerName : %s", componentName)) + } + + return componentName +} + +// SetContainerComponentValues randomly sets/updates container component attributes to random values +func (devfile *TestDevfile) SetContainerComponentValues(component *schema.Component) { + + containerComponent := component.Container + + containerComponent.Image = GetRandomUniqueString(GetRandomNumber(8, 18), false) + + if GetBinaryDecision() { + numCommands := GetRandomNumber(1, 3) + containerComponent.Command = make([]string, numCommands) + for i := 0; i < numCommands; i++ { + containerComponent.Command[i] = GetRandomString(GetRandomNumber(4, 16), false) + LogInfoMessage(fmt.Sprintf("....... command %d of %d : %s", i, numCommands, containerComponent.Command[i])) + } + } + + if GetBinaryDecision() { + numArgs := GetRandomNumber(1, 3) + containerComponent.Args = make([]string, numArgs) + for i := 0; i < numArgs; i++ { + containerComponent.Args[i] = GetRandomString(GetRandomNumber(8, 18), false) + LogInfoMessage(fmt.Sprintf("....... arg %d of %d : %s", i, numArgs, containerComponent.Args[i])) + } + } + + containerComponent.DedicatedPod = GetBinaryDecision() + LogInfoMessage(fmt.Sprintf("....... DedicatedPod: %t", containerComponent.DedicatedPod)) + + if GetBinaryDecision() { + containerComponent.MemoryLimit = strconv.Itoa(GetRandomNumber(4, 124)) + "M" + LogInfoMessage(fmt.Sprintf("....... MemoryLimit: %s", containerComponent.MemoryLimit)) + } + + if GetBinaryDecision() { + setMountSources := GetBinaryDecision() + containerComponent.MountSources = &setMountSources + LogInfoMessage(fmt.Sprintf("....... MountSources: %t", *containerComponent.MountSources)) + + if setMountSources { + containerComponent.SourceMapping = "/" + GetRandomString(8, false) + LogInfoMessage(fmt.Sprintf("....... SourceMapping: %s", containerComponent.SourceMapping)) + } + } + + if GetBinaryDecision() { + containerComponent.Env = addEnv(GetRandomNumber(1, 4)) + } else { + containerComponent.Env = nil + } + + if len(containerComponent.VolumeMounts) == 0 { + if GetBinaryDecision() { + containerComponent.VolumeMounts = devfile.addVolume(GetRandomNumber(1, 4)) + } + } + + if GetBinaryDecision() { + containerComponent.Endpoints = devfile.CreateEndpoints() + } + + devfile.componentUpdated(*component) + +} + +// SetVolumeComponentValues randomly sets/updates volume component attributes to random values +func (devfile *TestDevfile) SetVolumeComponentValues(component *schema.Component) { + + component.Volume.Size = strconv.Itoa(4+GetRandomNumber(64, 256)) + "G" + LogInfoMessage(fmt.Sprintf("....... volumeComponent.Size: %s", component.Volume.Size)) + devfile.componentUpdated(*component) + +} diff --git a/test/v200/utils/common/endpoint-test-utils.go b/test/v200/utils/common/endpoint-test-utils.go new file mode 100644 index 000000000..8c114d5d9 --- /dev/null +++ b/test/v200/utils/common/endpoint-test-utils.go @@ -0,0 +1,84 @@ +package common + +import ( + "fmt" + + schema "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2" +) + +var Exposures = [...]schema.EndpointExposure{schema.PublicEndpointExposure, schema.InternalEndpointExposure, schema.NoneEndpointExposure} + +// getRandomExposure returns a random exposure value +func getRandomExposure() schema.EndpointExposure { + return Exposures[GetRandomNumber(1, len(Exposures))-1] +} + +var Protocols = [...]schema.EndpointProtocol{schema.HTTPEndpointProtocol, schema.HTTPSEndpointProtocol, schema.WSEndpointProtocol, schema.WSSEndpointProtocol, schema.TCPEndpointProtocol, schema.UDPEndpointProtocol} + +// getRandomProtocol returns a random protocol value +func getRandomProtocol() schema.EndpointProtocol { + return Protocols[GetRandomNumber(1, len(Protocols))-1] +} + +// getUniquePort return a port value not previously used in that same devfile +func (devfile *TestDevfile) getUniquePort() int { + + // max sure a lot of unique ports exist + maxPorts := len(devfile.UsedPorts) + 5000 + + var port int + used := true + for used { + port = GetRandomNumber(1, maxPorts) + _, used = devfile.UsedPorts[port] + } + devfile.UsedPorts[port] = true + return port +} + +// CreateEndpoints creates and returns a randon number of endpoints in a schema structure +func (devfile *TestDevfile) CreateEndpoints() []schema.Endpoint { + + numEndpoints := GetRandomNumber(1, 5) + endpoints := make([]schema.Endpoint, numEndpoints) + + commonPort := devfile.getUniquePort() + + for i := 0; i < numEndpoints; i++ { + + endpoint := schema.Endpoint{} + + endpoint.Name = GetRandomUniqueString(GetRandomNumber(5, 24), true) + LogInfoMessage(fmt.Sprintf(" ....... add endpoint %d name : %s", i, endpoint.Name)) + + if GetBinaryDecision() { + endpoint.TargetPort = devfile.getUniquePort() + } else { + endpoint.TargetPort = commonPort + } + LogInfoMessage(fmt.Sprintf(" ....... add endpoint %d targetPort: %d", i, endpoint.TargetPort)) + + if GetBinaryDecision() { + endpoint.Exposure = getRandomExposure() + LogInfoMessage(fmt.Sprintf(" ....... add endpoint %d exposure: %s", i, endpoint.Exposure)) + } + + if GetBinaryDecision() { + endpoint.Protocol = getRandomProtocol() + LogInfoMessage(fmt.Sprintf(" ....... add endpoint %d protocol: %s", i, endpoint.Protocol)) + } + + endpoint.Secure = GetBinaryDecision() + LogInfoMessage(fmt.Sprintf(" ....... add endpoint %d secure: %t", i, endpoint.Secure)) + + if GetBinaryDecision() { + endpoint.Path = "/Path_" + GetRandomString(GetRandomNumber(3, 15), false) + LogInfoMessage(fmt.Sprintf(" ....... add endpoint %d path: %s", i, endpoint.Path)) + } + + endpoints[i] = endpoint + + } + + return endpoints +} diff --git a/test/v200/utils/common/test_utils.go b/test/v200/utils/common/test_utils.go new file mode 100644 index 000000000..e53ea473c --- /dev/null +++ b/test/v200/utils/common/test_utils.go @@ -0,0 +1,278 @@ +package common + +import ( + "fmt" + "io" + "log" + "math/rand" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + schema "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2" + header "github.com/devfile/api/v2/pkg/devfile" +) + +const ( + // maxCommands : The maximum number of commands to include in a generated devfile + maxCommands = 10 + // maxComponents : The maximum number of components to include in a generated devfile + maxComponents = 10 + + defaultTempDir = "./tmp/" + logFileName = "test.log" + // logToFileOnly - If set to false the log output will also be output to the console + logToFileOnly = true +) + +var ( + // tmpDir temporary directory in use + tmpDir string + testLogger *log.Logger +) + +// DevfileFollower interface implemented by the parser tests for updating the parser data +type DevfileFollower interface { + AddCommand(schema.Command) error + UpdateCommand(schema.Command) + AddComponent(schema.Component) error + UpdateComponent(schema.Component) + AddProject(schema.Project) error + UpdateProject(schema.Project) + AddStarterProject(schema.StarterProject) error + UpdateStarterProject(schema.StarterProject) + AddEvent(schema.Events) error + UpdateEvent(schema.Events) + SetParent(schema.Parent) error + UpdateParent(schema.Parent) + + SetMetaData(header.DevfileMetadata) error + UpdateMetaData(header.DevfileMetadata) + SetSchemaVersion(string) +} + +// DevfileValidator interface implemented by the parser and api tests for verifying generated devfiles +type DevfileValidator interface { + WriteAndValidate(*TestDevfile) error +} + +// TestContent - structure used by a test to configure the tests to run +type TestContent struct { + CommandTypes []schema.CommandType + ComponentTypes []schema.ComponentType + FileName string + EditContent bool +} + +// init creates: +// - the temporary directory used by the test to store logs and generated devfiles. +// - the log file +func init() { + tmpDir = defaultTempDir + if _, err := os.Stat(tmpDir); !os.IsNotExist(err) { + os.RemoveAll(tmpDir) + } + if err := os.Mkdir(tmpDir, 0755); err != nil { + fmt.Printf("Failed to create temp directory, will use current directory : %v ", err) + tmpDir = "./" + } + f, err := os.OpenFile(filepath.Join(tmpDir, logFileName), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + fmt.Printf("Error creating Log file : %v", err) + } else { + if logToFileOnly { + testLogger = log.New(f, "", log.LstdFlags|log.Lmicroseconds) + } else { + writer := io.MultiWriter(f, os.Stdout) + testLogger = log.New(writer, "", log.LstdFlags|log.Lmicroseconds) + } + testLogger.Println("Test Starting:") + } + + rand.Seed(time.Now().UnixNano()) + +} + +// CreateTempDir creates a specified sub directory under the temp directory if it does not exist. +// Returns the name of the created directory. +func CreateTempDir(subdir string) string { + tempDir := tmpDir + subdir + "/" + var err error + if _, err = os.Stat(tempDir); os.IsNotExist(err) { + err = os.Mkdir(tempDir, 0755) + } + if err != nil { + // if cannot create subdirectory just use the base tmp directory + LogErrorMessage(fmt.Sprintf("Failed to create temp directory %s will use %s : %v", tempDir, tmpDir, err)) + tempDir = tmpDir + } + return tempDir +} + +// GetDevFileName returns a qualified name of a devfile for use in a test. +// The devfile will be in a temporary directory and is named using the calling function's name. +func GetDevFileName() string { + pc, fn, _, ok := runtime.Caller(1) + if !ok { + return tmpDir + "DefaultDevfile" + } + + testFile := filepath.Base(fn) + testFileExtension := filepath.Ext(testFile) + subdir := testFile[0 : len(testFile)-len(testFileExtension)] + destDir := CreateTempDir(subdir) + callerName := runtime.FuncForPC(pc).Name() + pos1 := strings.LastIndex(callerName, "Test_") + devfileName := destDir + callerName[pos1:len(callerName)] + ".yaml" + + LogInfoMessage(fmt.Sprintf("GetDevFileName : %s", devfileName)) + + return devfileName +} + +// AddSuffixToFileName adds a specified suffix to the name of a specified file. +// For example if the file is devfile.yaml and the suffix is 1, the result is devfile1.yaml +func AddSuffixToFileName(fileName string, suffix string) string { + pos1 := strings.LastIndex(fileName, ".yaml") + newFileName := fileName[0:pos1] + suffix + ".yaml" + LogInfoMessage(fmt.Sprintf("Add suffix %s to fileName %s : %s", suffix, fileName, newFileName)) + return newFileName +} + +// LogMessage logs the specified message and returns the message logged +func LogMessage(message string) string { + if testLogger != nil { + testLogger.Println(message) + } else { + fmt.Printf("Logger not available: %s", message) + } + return message +} + +var errorPrefix = "..... ERROR : " +var infoPrefix = "INFO :" + +// LogErrorMessage logs the specified message as an error message and returns the message logged +func LogErrorMessage(message string) string { + var errMessage []string + errMessage = append(errMessage, errorPrefix, message) + return LogMessage(fmt.Sprint(errMessage)) +} + +// LogInfoMessage logs the specified message as an info message and returns the message logged +func LogInfoMessage(message string) string { + var infoMessage []string + infoMessage = append(infoMessage, infoPrefix, message) + return LogMessage(fmt.Sprint(infoMessage)) +} + +// TestDevfile is a structure used to track a test devfile and its contents +type TestDevfile struct { + SchemaDevFile schema.Devfile + FileName string + GroupDefaults map[schema.CommandGroupKind]bool + UsedPorts map[int]bool + Follower DevfileFollower + Validator DevfileValidator +} + +var StringCount int = 0 + +// GetRandomUniqueString returns a unique random string which is n characters long plus an integer to ensure uniqueness +// If lower is set to true a lower case string is returned. +func GetRandomUniqueString(n int, lower bool) string { + StringCount++ + return fmt.Sprintf("%s%04d", GetRandomString(n, lower), StringCount) +} + +const schemaBytes = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + +// GetRandomString returns a random string which is n characters long. +// If lower is set to true a lower case string is returned. +func GetRandomString(n int, lower bool) string { + b := make([]byte, n) + for i := range b { + b[i] = schemaBytes[rand.Intn(len(schemaBytes)-1)] + } + randomString := string(b) + if lower { + randomString = strings.ToLower(randomString) + } + return randomString +} + +var GroupKinds = [...]schema.CommandGroupKind{schema.BuildCommandGroupKind, schema.RunCommandGroupKind, schema.TestCommandGroupKind, schema.DebugCommandGroupKind} + +// GetRandomGroupKind return random group kind. One of "build", "run", "test" or "debug" +func GetRandomGroupKind() schema.CommandGroupKind { + return GroupKinds[GetRandomNumber(1, len(GroupKinds))-1] +} + +// GetBinaryDecision randomly returns true or false +func GetBinaryDecision() bool { + return GetRandomDecision(1, 1) +} + +// GetRandomDecision randomly returns true or false, but weighted to one or the other. +// For example if success is set to 2 and failure to 1, true is twice as likely to be returned. +func GetRandomDecision(success int, failure int) bool { + return rand.Intn(success+failure) > failure-1 +} + +// GetRandomNumber randomly returns an integer between 1 and the number specified. +func GetRandomNumber(min int, max int) int { + if min == max { + return 1 + } else if min > max { + return rand.Intn(max) + 1 + } + return rand.Intn(max-min) + min + 1 +} + +// GetDevfile returns a structure used to represent a specific devfile in a test +func GetDevfile(fileName string, follower DevfileFollower, validator DevfileValidator) (TestDevfile, error) { + + var err error + testDevfile := TestDevfile{} + testDevfile.SchemaDevFile = schema.Devfile{} + testDevfile.FileName = fileName + testDevfile.SchemaDevFile.SchemaVersion = "2.0.0" + testDevfile.GroupDefaults = make(map[schema.CommandGroupKind]bool) + for _, kind := range GroupKinds { + testDevfile.GroupDefaults[kind] = false + } + testDevfile.UsedPorts = make(map[int]bool) + testDevfile.Validator = validator + testDevfile.Follower = follower + + return testDevfile, err +} + +// Runs a test to create and verify a devfile based on the content of the specified TestContent +func (testDevfile *TestDevfile) RunTest(testContent TestContent, t *testing.T) { + + if len(testContent.CommandTypes) > 0 { + numCommands := GetRandomNumber(1, maxCommands) + for i := 0; i < numCommands; i++ { + commandIndex := GetRandomNumber(1, len(testContent.CommandTypes)) + testDevfile.AddCommand(testContent.CommandTypes[commandIndex-1]) + } + } + + if len(testContent.ComponentTypes) > 0 { + numComponents := GetRandomNumber(1, maxComponents) + for i := 0; i < numComponents; i++ { + componentIndex := GetRandomNumber(1, len(testContent.ComponentTypes)) + testDevfile.AddComponent(testContent.ComponentTypes[componentIndex-1]) + } + } + + err := testDevfile.Validator.WriteAndValidate(testDevfile) + if err != nil { + t.Fatalf(LogErrorMessage(fmt.Sprintf("ERROR verifying devfile : %s : %v", testContent.FileName, err))) + } + +}