Merge pull request #42 from kevinrizza/validation

OperatorHub io validator

cmd/operator-verify/manifests/cmd.go

@@ -1,9 +1,6 @@
 package manifests
 
 import (
-	"fmt"
-	"os"
-
 	"github.com/operator-framework/api/pkg/manifests"
 	"github.com/operator-framework/api/pkg/validation"
 	"github.com/operator-framework/api/pkg/validation/errors"
@@ -13,33 +10,57 @@ import (
 )
 
 func NewCmd() *cobra.Command {
-	return &cobra.Command{
+	rootCmd := &cobra.Command{
 		Use:   "manifests",
 		Short: "Validates all manifests in a directory",
-		Long: `'operator-verify manifests' validates all manifests in the supplied directory
+		Long: `'operator-verify manifests' validates a bundle in the supplied directory
 and prints errors and warnings corresponding to each manifest found to be
 invalid. Manifests are only validated if a validator for that manifest
 type/kind, ex. CustomResourceDefinition, is implemented in the Operator
 validation library.`,
-		Run: func(cmd *cobra.Command, args []string) {
-			if len(args) != 1 {
-				log.Fatalf("command %s requires exactly one argument", cmd.CommandPath())
-			}
-			bundle, err := manifests.GetBundleFromDir(args[0])
-			if err != nil {
-				log.Fatalf("Error generating bundle from directory %s", err.Error())
-			}
-			results := validation.AllValidators.Validate(bundle)
-			nonEmptyResults := []errors.ManifestResult{}
-			for _, result := range results {
-				if result.HasError() || result.HasWarn() {
-					nonEmptyResults = append(nonEmptyResults, result)
-				}
-			}
-			if len(nonEmptyResults) != 0 {
-				fmt.Println(nonEmptyResults)
-				os.Exit(1)
-			}
-		},
+		Run: manifestsFunc,
+	}
+
+	rootCmd.Flags().Bool("operatorhub_validate", false, "enable optional UI validation for operatorhub.io")
+
+	return rootCmd
+}
+
+func manifestsFunc(cmd *cobra.Command, args []string) {
+	bundle, err := manifests.GetBundleFromDir(args[0])
+	if err != nil {
+		log.Fatalf("Error generating bundle from directory: %s", err.Error())
+	}
+	if bundle == nil {
+		log.Fatalf("Error generating bundle from directory")
+	}
+
+	operatorHubValidate, err := cmd.Flags().GetBool("operatorhub_validate")
+	if err != nil {
+		log.Fatalf("Unable to parse operatorhub_validate parameter")
+	}
+
+	validators := validation.DefaultBundleValidators
+	if operatorHubValidate {
+		validators = validators.WithValidators(validation.OperatorHubValidator)
+	}
+
+	results := validators.Validate(bundle.ObjectsToValidate()...)
+	nonEmptyResults := []errors.ManifestResult{}
+	for _, result := range results {
+		if result.HasError() || result.HasWarn() {
+			nonEmptyResults = append(nonEmptyResults, result)
+		}
+	}
+
+	for _, result := range nonEmptyResults {
+		for _, err := range result.Errors {
+			log.Errorf(err.Error())
+		}
+		for _, err := range result.Warnings {
+			log.Warnf(err.Error())
+		}
 	}
+
+	return
 }

pkg/manifests/bundle.go

@@ -20,3 +20,17 @@ type Bundle struct {
 	V1CRDs         []*apiextensionsv1.CustomResourceDefinition
 	Dependencies   []*Dependency
 }
+
+func (b *Bundle) ObjectsToValidate() []interface{} {
+	objs := []interface{}{}
+	for _, crd := range b.V1CRDs {
+		objs = append(objs, crd)
+	}
+	for _, crd := range b.V1beta1CRDs {
+		objs = append(objs, crd)
+	}
+	objs = append(objs, b.CSV)
+	objs = append(objs, b)
+
+	return objs
+}

pkg/manifests/bundleloader.go

@@ -18,8 +18,9 @@ import (
 
 // bundleLoader loads a bundle directory from disk
 type bundleLoader struct {
-	dir    string
-	bundle *Bundle
+	dir      string
+	bundle   *Bundle
+	foundCSV bool
 }
 
 func NewBundleLoader(dir string) bundleLoader {
@@ -34,6 +35,12 @@ func (b *bundleLoader) LoadBundle() error {
 		errs = append(errs, err)
 	}
 
+	if !b.foundCSV {
+		errs = append(errs, fmt.Errorf("unable to find a csv in bundle directory %s", b.dir))
+	} else if b.bundle == nil {
+		errs = append(errs, fmt.Errorf("unable to load bundle from directory %s", b.dir))
+	}
+
 	return utilerrors.NewAggregate(errs)
 }
 
@@ -82,6 +89,8 @@ func (b *bundleLoader) LoadBundleWalkFunc(path string, f os.FileInfo, err error)
 		return nil
 	}
 
+	b.foundCSV = true
+
 	var errs []error
 	bundle, err := loadBundle(csv.GetName(), filepath.Dir(path))
 	if err != nil {

pkg/validation/internal/csv.go

@@ -46,7 +46,7 @@ func validateCSV(csv *v1alpha1.ClusterServiceVersion) errors.ManifestResult {
 	// validate installModes
 	result.Add(validateInstallModes(csv)...)
 	// check missing optional/mandatory fields.
-	result.Add(checkFields(csv)...)
+	result.Add(checkFields(*csv)...)
 	return result
 }
 
@@ -67,7 +67,7 @@ func parseCSVNameFormat(name string) (string, semver.Version, error) {
 }
 
 // checkFields runs checkEmptyFields and returns its errors.
-func checkFields(csv *v1alpha1.ClusterServiceVersion) (errs []errors.Error) {
+func checkFields(csv v1alpha1.ClusterServiceVersion) (errs []errors.Error) {
 	result := errors.ManifestResult{}
 	checkEmptyFields(&result, reflect.ValueOf(csv), "")
 	return append(result.Errors, result.Warnings...)

pkg/validation/internal/operatorhub.go

@@ -0,0 +1,152 @@
+package internal
+
+import (
+	"fmt"
+	"net/mail"
+	"net/url"
+	"strings"
+
+	"github.com/operator-framework/api/pkg/manifests"
+	"github.com/operator-framework/api/pkg/operators/v1alpha1"
+	"github.com/operator-framework/api/pkg/validation/errors"
+	interfaces "github.com/operator-framework/api/pkg/validation/interfaces"
+)
+
+var OperatorHubValidator interfaces.Validator = interfaces.ValidatorFunc(validateOperatorHub)
+
+var validCapabilities = map[string]struct{}{
+	"Basic Install":     struct{}{},
+	"Seamless Upgrades": struct{}{},
+	"Full Lifecycle":    struct{}{},
+	"Deep Insights":     struct{}{},
+	"Auto Pilot":        struct{}{},
+}
+
+var validMediatypes = map[string]struct{}{
+	"image/gif":     struct{}{},
+	"image/jpeg":    struct{}{},
+	"image/png":     struct{}{},
+	"image/svg+xml": struct{}{},
+}
+
+var validCategories = map[string]struct{}{
+	"AI/Machine Learning":    struct{}{},
+	"Application Runtime":    struct{}{},
+	"Big Data":               struct{}{},
+	"Cloud Provider":         struct{}{},
+	"Developer Tools":        struct{}{},
+	"Database":               struct{}{},
+	"Integration & Delivery": struct{}{},
+	"Logging & Tracing":      struct{}{},
+	"Monitoring":             struct{}{},
+	"Networking":             struct{}{},
+	"OpenShift Optional":     struct{}{},
+	"Security":               struct{}{},
+	"Storage":                struct{}{},
+	"Streaming & Messaging":  struct{}{},
+}
+
+func validateOperatorHub(objs ...interface{}) (results []errors.ManifestResult) {
+	for _, obj := range objs {
+		switch v := obj.(type) {
+		case *manifests.Bundle:
+			results = append(results, validateBundleOperatorHub(v))
+		}
+	}
+	return results
+}
+
+func validateBundleOperatorHub(bundle *manifests.Bundle) errors.ManifestResult {
+	result := errors.ManifestResult{Name: bundle.Name}
+
+	if bundle == nil {
+		result.Add(errors.ErrInvalidBundle("Bundle is nil", nil))
+		return result
+	}
+
+	if bundle.CSV == nil {
+		result.Add(errors.ErrInvalidBundle("Bundle csv is nil", bundle.Name))
+		return result
+	}
+
+	errs := validateHubCSVSpec(*bundle.CSV)
+	for _, err := range errs {
+		result.Add(errors.ErrInvalidCSV(err.Error(), bundle.CSV.GetName()))
+	}
+
+	return result
+}
+
+func validateHubCSVSpec(csv v1alpha1.ClusterServiceVersion) []error {
+	var errs []error
+
+	if csv.Spec.Provider.Name == "" {
+		errs = append(errs, fmt.Errorf("csv.Spec.Provider.Name not specified"))
+	}
+
+	for _, maintainer := range csv.Spec.Maintainers {
+		if maintainer.Name == "" || maintainer.Email == "" {
+			errs = append(errs, fmt.Errorf("csv.Spec.Maintainers elements should contain both name and email"))
+		}
+		if maintainer.Email != "" {
+			_, err := mail.ParseAddress(maintainer.Email)
+			if err != nil {
+				errs = append(errs, fmt.Errorf("csv.Spec.Maintainers email %s is invalid: %v", maintainer.Email, err))
+			}
+		}
+	}
+
+	for _, link := range csv.Spec.Links {
+		if link.Name == "" || link.URL == "" {
+			errs = append(errs, fmt.Errorf("csv.Spec.Links elements should contain both name and url"))
+		}
+		if link.URL != "" {
+			_, err := url.ParseRequestURI(link.URL)
+			if err != nil {
+				errs = append(errs, fmt.Errorf("csv.Spec.Links url %s is invalid: %v", link.URL, err))
+			}
+		}
+	}
+
+	if csv.GetAnnotations() == nil {
+		csv.SetAnnotations(make(map[string]string))
+	}
+
+	if capability, ok := csv.ObjectMeta.Annotations["capabilities"]; ok {
+		if _, ok := validCapabilities[capability]; !ok {
+			errs = append(errs, fmt.Errorf("csv.Metadata.Annotations.Capabilities %s is not a valid capabilities level", capability))
+		}
+	}
+
+	if csv.Spec.Icon != nil {
+		// only one icon is allowed
+		if len(csv.Spec.Icon) != 1 {
+			errs = append(errs, fmt.Errorf("csv.Spec.Icon should only have one element"))
+		}
+
+		icon := csv.Spec.Icon[0]
+		if icon.MediaType == "" || icon.Data == "" {
+			errs = append(errs, fmt.Errorf("csv.Spec.Icon elements should contain both data and mediatype"))
+		}
+
+		if icon.MediaType != "" {
+			if _, ok := validMediatypes[icon.MediaType]; !ok {
+				errs = append(errs, fmt.Errorf("csv.Spec.Icon %s does not have a valid mediatype", icon.MediaType))
+			}
+		}
+	} else {
+		errs = append(errs, fmt.Errorf("csv.Spec.Icon not specified"))
+	}
+
+	if categories, ok := csv.ObjectMeta.Annotations["categories"]; ok {
+		categorySlice := strings.Split(categories, ",")
+
+		for _, category := range categorySlice {
+			if _, ok := validCategories[category]; !ok {
+				errs = append(errs, fmt.Errorf("csv.Metadata.Annotations.Categories %s is not a valid category", category))
+			}
+		}
+	}
+
+	return errs
+}

pkg/validation/internal/operatorhub_test.go

@@ -0,0 +1,59 @@
+package internal
+
+import (
+	"testing"
+
+	"github.com/operator-framework/api/pkg/manifests"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestValidateBundleOperatorHub(t *testing.T) {
+	var table = []struct {
+		description string
+		directory   string
+		hasError    bool
+		errStrings  []string
+	}{
+		{
+			description: "registryv1 bundle/valid bundle",
+			directory:   "./testdata/valid_bundle",
+			hasError:    false,
+		},
+		{
+			description: "registryv1 bundle/invald bundle operatorhubio",
+			directory:   "./testdata/invalid_bundle_operatorhub",
+			hasError:    true,
+			errStrings: []string{
+				`Error: Value : (etcdoperator.v0.9.4) csv.Spec.Provider.Name not specified`,
+				`Error: Value : (etcdoperator.v0.9.4) csv.Spec.Maintainers elements should contain both name and email`,
+				`Error: Value : (etcdoperator.v0.9.4) csv.Spec.Maintainers email invalidemail is invalid: mail: missing '@' or angle-addr`,
+				`Error: Value : (etcdoperator.v0.9.4) csv.Spec.Links elements should contain both name and url`,
+				`Error: Value : (etcdoperator.v0.9.4) csv.Spec.Links url https//coreos.com/operators/etcd/docs/latest/ is invalid: parse https//coreos.com/operators/etcd/docs/latest/: invalid URI for request`,
+				`Error: Value : (etcdoperator.v0.9.4) csv.Metadata.Annotations.Capabilities Installs and stuff is not a valid capabilities level`,
+				`Error: Value : (etcdoperator.v0.9.4) csv.Spec.Icon should only have one element`,
+				`Error: Value : (etcdoperator.v0.9.4) csv.Metadata.Annotations.Categories Magic is not a valid category`,
+			},
+		},
+	}
+
+	for _, tt := range table {
+		// Validate the bundle object
+		bundle, err := manifests.GetBundleFromDir(tt.directory)
+		require.NoError(t, err)
+
+		results := OperatorHubValidator.Validate(bundle)
+
+		if len(results) > 0 {
+			require.Equal(t, results[0].HasError(), tt.hasError)
+			if results[0].HasError() {
+				require.Equal(t, len(tt.errStrings), len(results[0].Errors))
+
+				for _, err := range results[0].Errors {
+					errString := err.Error()
+					require.Contains(t, tt.errStrings, errString)
+				}
+			}
+		}
+	}
+}

pkg/validation/internal/testdata/invalid_bundle_operatorhub/etcdbackups.etcd.database.coreos.com.crd.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: etcdbackups.etcd.database.coreos.com
+spec:
+  group: etcd.database.coreos.com
+  names:
+    kind: EtcdBackup
+    listKind: EtcdBackupList
+    plural: etcdbackups
+    singular: etcdbackup
+  scope: Namespaced
+  version: v1beta2