First stage of new tests which share logic with library tests (#348)

* add new tests shared with library

* Code review comments part 2

.gitignore

@@ -1,7 +1,7 @@
 # Temporary Build Files
 build/_output
 build/_test
-test/v200/schemaTest/tmp
+test/**/tmp
 test/go/pkg
 # Created by https://www.gitignore.io/api/go,vim,emacs,visualstudiocode
 ### Emacs ###

test/README.md

@@ -1,4 +1,4 @@
-# API Tests
+# schemaTest
 
 The API tests are intended to provide a comprehensive verification of the devfile schemas. This includes:
 - Ensuring every possible attribute is valid.
@@ -44,3 +44,27 @@ The test will read each of the test-xxxxxx.json files and run the tests defined
 1. Modify the copied tests as needed for the new version as decsribed above.
 1. Add `test/v201/schemaTest/tmp` to the .gitignore file.
 1. Run the test
+
+
+# apiTest
+
+A new test approach, shared with the library repository for testing valid devfiles. Basically the test creates lots of valid devfiles whith different content. The attributes which are set and the values to which they are set are randomized. These tests are a work in progress and the intent is to eventually replace schemaTest.  
+
+## Test structure
+
+- `test/v200/apiTest/api-test.go`: The go unit test program
+- `test/v200/utils/api/test-utils.go` : utilites, used by the test, which contain functions uniqiue to the api tests.
+- `test/v200/utils/common/*-utils.go` : utilites, used by the test, which are also used by the library tests. Mostly contain the code to generate valid devfile content.
+
+
+## Running tests
+
+from the `test/v200/apiTest/` directory run
+- `go test -v`
+
+* The test will generate a set of valid devfile.yaml files in `test/v200/apiTest/tmp/api-test/
+* The test will generate a log file:  `test/v200/apiTest/tmp/test.log`
+* Each run of the test removes the  `test/v200/apiTest/tmp` directory from the previous run.
+
+
+

test/v200/apiTest/api_test.go

@@ -0,0 +1,80 @@
+package apiTest
+
+import (
+	"testing"
+
+	schema "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
+	apiUtils "github.com/devfile/api/v2/test/v200/utils/api"
+	commonUtils "github.com/devfile/api/v2/test/v200/utils/common"
+)
+
+func Test_ExecCommand(t *testing.T) {
+	testContent := commonUtils.TestContent{}
+	testContent.CommandTypes = []schema.CommandType{schema.ExecCommandType}
+	testContent.EditContent = false
+	testContent.FileName = commonUtils.GetDevFileName()
+	apiUtils.RunTest(testContent, t)
+}
+
+func Test_ApplyCommand(t *testing.T) {
+	testContent := commonUtils.TestContent{}
+	testContent.CommandTypes = []schema.CommandType{schema.ApplyCommandType}
+	testContent.EditContent = false
+	testContent.FileName = commonUtils.GetDevFileName()
+	apiUtils.RunTest(testContent, t)
+}
+
+func Test_CompositeCommand(t *testing.T) {
+	testContent := commonUtils.TestContent{}
+	testContent.CommandTypes = []schema.CommandType{schema.CompositeCommandType}
+	testContent.EditContent = false
+	testContent.FileName = commonUtils.GetDevFileName()
+	apiUtils.RunTest(testContent, t)
+}
+
+func Test_MultiCommand(t *testing.T) {
+	testContent := commonUtils.TestContent{}
+	testContent.CommandTypes = []schema.CommandType{schema.ExecCommandType,
+		schema.CompositeCommandType,
+		schema.ApplyCommandType}
+	testContent.EditContent = true
+	testContent.FileName = commonUtils.GetDevFileName()
+	apiUtils.RunTest(testContent, t)
+}
+
+func Test_ContainerComponent(t *testing.T) {
+	testContent := commonUtils.TestContent{}
+	testContent.ComponentTypes = []schema.ComponentType{schema.ContainerComponentType}
+	testContent.EditContent = false
+	testContent.FileName = commonUtils.GetDevFileName()
+	apiUtils.RunTest(testContent, t)
+}
+
+func Test_VolumeComponent(t *testing.T) {
+	testContent := commonUtils.TestContent{}
+	testContent.ComponentTypes = []schema.ComponentType{schema.VolumeComponentType}
+	testContent.FileName = commonUtils.GetDevFileName()
+	apiUtils.RunTest(testContent, t)
+}
+
+func Test_MultiComponent(t *testing.T) {
+	testContent := commonUtils.TestContent{}
+	testContent.ComponentTypes = []schema.ComponentType{
+		schema.ContainerComponentType,
+		schema.VolumeComponentType}
+	testContent.FileName = commonUtils.GetDevFileName()
+	apiUtils.RunTest(testContent, t)
+}
+
+func Test_Everything(t *testing.T) {
+	testContent := commonUtils.TestContent{}
+	testContent.CommandTypes = []schema.CommandType{
+		schema.ExecCommandType,
+		schema.CompositeCommandType,
+		schema.ApplyCommandType}
+	testContent.ComponentTypes = []schema.ComponentType{
+		schema.ContainerComponentType,
+		schema.VolumeComponentType}
+	testContent.FileName = commonUtils.GetDevFileName()
+	apiUtils.RunTest(testContent, t)
+}

test/v200/utils/api/test_utils.go

@@ -0,0 +1,175 @@
+package api
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"strconv"
+	"strings"
+	"testing"
+
+	commonUtils "github.com/devfile/api/v2/test/v200/utils/common"
+	"github.com/santhosh-tekuri/jsonschema"
+	"sigs.k8s.io/yaml"
+)
+
+const (
+	// numDevfiles : the number of devfiles to create for each test
+	numDevfiles = 5
+
+	schemaFileName = "../../../schemas/latest/ide-targeted/devfile.json"
+)
+
+var schemas = make(map[string]SchemaFile)
+
+// SchemaFile - represents the schema stucture
+type SchemaFile struct {
+	Schema *jsonschema.Schema
+}
+
+// DevfileValidator struct for DevfileValidator interface defined in common utils.
+type DevfileValidator struct{}
+
+// WriteAndValidate implements Saved.DevfileValidator interface.
+// writes to disk and validates the specified devfile
+func (devfileValidator DevfileValidator) WriteAndValidate(devfile *commonUtils.TestDevfile) error {
+	err := writeDevfile(devfile)
+	if err != nil {
+		commonUtils.LogErrorMessage(fmt.Sprintf("Error writing file : %s : %v", devfile.FileName, err))
+	} else {
+		err = validateDevfile(devfile)
+		if err != nil {
+			commonUtils.LogErrorMessage(fmt.Sprintf("Error vaidating file : %s : %v", devfile.FileName, err))
+		}
+	}
+	return err
+}
+
+// checkWithSchema checks the validity of a devfile against the schema.
+func (schemaFile *SchemaFile) checkWithSchema(devfile string, expectedMessage string) error {
+
+	// Read the created yaml file, ready for converison to json
+	devfileData, err := ioutil.ReadFile(devfile)
+	if err != nil {
+		commonUtils.LogErrorMessage(fmt.Sprintf("  FAIL: schema : unable to read %s: %v", devfile, err))
+		return err
+	}
+
+	// Convert the yaml file to json
+	devfileDataAsJSON, err := yaml.YAMLToJSON(devfileData)
+	if err != nil {
+		commonUtils.LogErrorMessage(fmt.Sprintf("  FAIL : %s : schema : failed to convert to json : %v", devfile, err))
+		return err
+	}
+
+	validationErr := schemaFile.Schema.Validate(bytes.NewReader(devfileDataAsJSON))
+	if validationErr != nil {
+		if len(expectedMessage) > 0 {
+			if !strings.Contains(validationErr.Error(), expectedMessage) {
+				err = errors.New(commonUtils.LogErrorMessage(fmt.Sprintf("  FAIL : schema : %s : Did not fail as expected : %s  got : %v", devfile, expectedMessage, validationErr)))
+			} else {
+				commonUtils.LogInfoMessage(fmt.Sprintf("PASS: schema :  Expected Error received : %s", expectedMessage))
+			}
+		} else {
+			err = errors.New(commonUtils.LogErrorMessage(fmt.Sprintf("  FAIL : schema : %s : Did not pass as expected, got : %v", devfile, validationErr)))
+		}
+	} else {
+		if len(expectedMessage) > 0 {
+			err = errors.New(commonUtils.LogErrorMessage(fmt.Sprintf("  FAIL : schema : %s :  was valid - Expected Error not found : %v", devfile, validationErr)))
+		} else {
+			commonUtils.LogInfoMessage(fmt.Sprintf("  PASS : schema : %s : devfile was valid.", devfile))
+		}
+	}
+	return err
+}
+
+// getSchema downloads and saves a schema from the provided url
+func getSchema(schemaFileName string) (SchemaFile, error) {
+
+	var err error
+	schemaFile, found := schemas[schemaFileName]
+	if !found {
+
+		schemaFile = SchemaFile{}
+
+		// Prepare the schema file
+		compiler := jsonschema.NewCompiler()
+		// Use Draft 7, github.com/santhosh-tekuri/jsonschema provides 4,6 an 7 so use the latest
+		compiler.Draft = jsonschema.Draft7
+		schemaFile.Schema, err = compiler.Compile(schemaFileName)
+		if err != nil {
+			commonUtils.LogErrorMessage(fmt.Sprintf("FAIL : Failed to compile schema  %v", err))
+		} else {
+			commonUtils.LogInfoMessage(fmt.Sprintf("Schema compiled from file: %s)", schemaFileName))
+			schemas[schemaFileName] = schemaFile
+		}
+	}
+	return schemaFile, err
+}
+
+// writeDevfile creates a devfile on disk for use in a test.
+func writeDevfile(devfile *commonUtils.TestDevfile) error {
+	var err error
+
+	fileName := devfile.FileName
+	if !strings.HasSuffix(fileName, ".yaml") {
+		fileName += ".yaml"
+	}
+
+	commonUtils.LogInfoMessage(fmt.Sprintf("Marshall and write devfile %s", devfile.FileName))
+
+	c, marshallErr := yaml.Marshal(&(devfile.SchemaDevFile))
+
+	if marshallErr != nil {
+		err = errors.New(commonUtils.LogErrorMessage(fmt.Sprintf("Marshall devfile %s : %v", devfile.FileName, marshallErr)))
+	} else {
+		err = ioutil.WriteFile(fileName, c, 0644)
+		if err != nil {
+			commonUtils.LogErrorMessage(fmt.Sprintf("Write devfile %s : %v", devfile.FileName, err))
+		}
+	}
+	return err
+}
+
+// validateDevfile check the provided defile against the schema
+func validateDevfile(devfile *commonUtils.TestDevfile) error {
+
+	var err error
+	var schemaFile SchemaFile
+
+	schemaFile, err = getSchema(schemaFileName)
+	if err != nil {
+		commonUtils.LogErrorMessage(fmt.Sprintf("Failed to get devfile schema : %v", err))
+	} else {
+		err = schemaFile.checkWithSchema(devfile.FileName, "")
+		if err != nil {
+			commonUtils.LogErrorMessage(fmt.Sprintf("Verification with devfile schema failed : %v", err))
+		} else {
+			commonUtils.LogInfoMessage(fmt.Sprintf("Devfile validated using JSONSchema schema : %s", devfile.FileName))
+		}
+	}
+
+	return err
+}
+
+// RunTest : Runs a test to create and verify a devfile based on the content of the specified TestContent
+func RunTest(testContent commonUtils.TestContent, t *testing.T) {
+
+	commonUtils.LogMessage(fmt.Sprintf("Start test for %s", testContent.FileName))
+
+	validator := DevfileValidator{}
+
+	devfileName := testContent.FileName
+	for i := 1; i <= numDevfiles; i++ {
+
+		testContent.FileName = commonUtils.AddSuffixToFileName(devfileName, strconv.Itoa(i))
+
+		testDevfile, err := commonUtils.GetDevfile(testContent.FileName, nil, validator)
+		if err != nil {
+			t.Fatalf(commonUtils.LogMessage(fmt.Sprintf("Error creating devfile : %v", err)))
+		}
+
+		testDevfile.RunTest(testContent, t)
+	}
+}