From 04ae799bd035bab6c32bbe4bd1ac38b49ecad978 Mon Sep 17 00:00:00 2001 From: Bailin He Date: Wed, 4 Oct 2023 01:26:24 +0000 Subject: [PATCH] add validator --- .../extension_resource_definitions.go | 15 +- pkg/jsonschema/compiler.go | 75 +++++ pkg/jsonschema/doc.go | 3 + pkg/jsonschema/errors.go | 13 + pkg/jsonschema/unique_constraint.go | 229 +++++++++++++++ pkg/jsonschema/unique_constraint_test.go | 271 ++++++++++++++++++ 6 files changed, 603 insertions(+), 3 deletions(-) create mode 100644 pkg/jsonschema/compiler.go create mode 100644 pkg/jsonschema/doc.go create mode 100644 pkg/jsonschema/errors.go create mode 100644 pkg/jsonschema/unique_constraint.go create mode 100644 pkg/jsonschema/unique_constraint_test.go diff --git a/pkg/api/v1alpha1/extension_resource_definitions.go b/pkg/api/v1alpha1/extension_resource_definitions.go index 19a1dd5b..7993fc11 100644 --- a/pkg/api/v1alpha1/extension_resource_definitions.go +++ b/pkg/api/v1alpha1/extension_resource_definitions.go @@ -15,10 +15,9 @@ import ( "github.com/metal-toolbox/governor-api/internal/dbtools" "github.com/metal-toolbox/governor-api/internal/models" events "github.com/metal-toolbox/governor-api/pkg/events/v1alpha1" + "github.com/metal-toolbox/governor-api/pkg/jsonschema" "github.com/volatiletech/sqlboiler/v4/boil" "github.com/volatiletech/sqlboiler/v4/queries/qm" - - jsonschema "github.com/santhosh-tekuri/jsonschema/v5" ) // ExtensionResourceDefinition is the extension resource definition response @@ -218,7 +217,17 @@ func (r *Router) createExtensionResourceDefinition(c *gin.Context) { schema = string(req.Schema) } - if _, err := jsonschema.CompileString("https://governor/s.json", schema); err != nil { + compiler := jsonschema.NewCompiler( + extensionID, req.SlugPlural, req.Version, + jsonschema.WithUniqueConstraint( + c.Request.Context(), + &models.ExtensionResourceDefinition{}, + nil, + nil, + ), + ) + + if _, err := compiler.Compile(schema); err != nil { sendError(c, http.StatusBadRequest, "ERD schema is not valid: "+err.Error()) return } diff --git a/pkg/jsonschema/compiler.go b/pkg/jsonschema/compiler.go new file mode 100644 index 00000000..c3883ced --- /dev/null +++ b/pkg/jsonschema/compiler.go @@ -0,0 +1,75 @@ +package jsonschema + +import ( + "context" + "fmt" + "strings" + + "github.com/metal-toolbox/governor-api/internal/models" + "github.com/santhosh-tekuri/jsonschema/v5" + "github.com/volatiletech/sqlboiler/v4/boil" +) + +// Compiler is a struct for a JSON schema compiler +type Compiler struct { + jsonschema.Compiler + + extensionID string + erdSlugPlural string + version string +} + +// Option is a functional configuration option for JSON schema compiler +type Option func(c *Compiler) + +// NewCompiler configures and creates a new JSON schema compiler +func NewCompiler( + extensionID, slugPlural, version string, + opts ...Option, +) *Compiler { + c := &Compiler{*jsonschema.NewCompiler(), extensionID, slugPlural, version} + + for _, opt := range opts { + opt(c) + } + + return c +} + +// WithUniqueConstraint enables the unique constraint extension for a JSON +// schema. An extra `unique` field can be added to the JSON schema, and the +// Validator will ensure that the combination of every properties in the +// array is unique within the given extension resource definition. +// Note that unique constraint validation will be skipped if db is nil. +func WithUniqueConstraint( + ctx context.Context, + extensionResourceDefinition *models.ExtensionResourceDefinition, + resourceID *string, + db boil.ContextExecutor, +) Option { + return func(c *Compiler) { + c.RegisterExtension( + "uniqueConstraint", + JSONSchemaUniqueConstraint, + &UniqueConstraintCompiler{extensionResourceDefinition, resourceID, ctx, db}, + ) + } +} + +func (c *Compiler) schemaURL() string { + return fmt.Sprintf( + "https://governor/extensions/%s/erds/%s/%s/schema.json", + c.extensionID, c.erdSlugPlural, c.version, + ) +} + +// Compile compiles the schema string +func (c *Compiler) Compile(schema string) (*jsonschema.Schema, error) { + url := c.schemaURL() + + if err := c.AddResource(url, strings.NewReader(schema)); err != nil { + return nil, err + } + + return c.Compiler.Compile(url) +} diff --git a/pkg/jsonschema/doc.go b/pkg/jsonschema/doc.go new file mode 100644 index 00000000..241cb991 --- /dev/null +++ b/pkg/jsonschema/doc.go @@ -0,0 +1,3 @@ +// Package jsonschema provides a JSON schema validator that is tailored for +// validations of governor's Extension Resources +package jsonschema diff --git a/pkg/jsonschema/errors.go b/pkg/jsonschema/errors.go new file mode 100644 index 00000000..4ebed065 --- /dev/null +++ b/pkg/jsonschema/errors.go @@ -0,0 +1,13 @@ +package jsonschema + +import "errors" + +var ( + // ErrInvalidUniqueProperty is returned when the schema's unique property + // is invalid + ErrInvalidUniqueProperty = errors.New(`property "unique" is invalid`) + + // ErrUniqueConstraintViolation is returned when an object violates the unique + // constrain + ErrUniqueConstraintViolation = errors.New("unique constraint violation") +) diff --git a/pkg/jsonschema/unique_constraint.go b/pkg/jsonschema/unique_constraint.go new file mode 100644 index 00000000..fb2eeb48 --- /dev/null +++ b/pkg/jsonschema/unique_constraint.go @@ -0,0 +1,229 @@ +package jsonschema + +import ( + "context" + "fmt" + "reflect" + + "github.com/metal-toolbox/governor-api/internal/models" + "github.com/santhosh-tekuri/jsonschema/v5" + "github.com/volatiletech/sqlboiler/v4/boil" + "github.com/volatiletech/sqlboiler/v4/queries/qm" +) + +// JSONSchemaUniqueConstraint is a JSON schema extension that provides a +// "unique" property of type array +var JSONSchemaUniqueConstraint = jsonschema.MustCompileString( + "https://governor/json-schemas/unique.json", + `{ + "properties": { + "unique": { + "type": "array", + "items": { + "type": "string" + } + } + } + }`, +) + +// UniqueConstraintSchema is the schema struct for the unique constraint JSON schema extension +type UniqueConstraintSchema struct { + UniqueFieldTypesMap map[string]string + ERD *models.ExtensionResourceDefinition + ResourceID *string + ctx context.Context + db boil.ContextExecutor +} + +// UniqueConstraintSchema implements jsonschema.ExtSchema +var _ jsonschema.ExtSchema = (*UniqueConstraintSchema)(nil) + +// Validate checks the uniqueness of the provided value against a database +// to ensure the unique constraint is satisfied. +func (s *UniqueConstraintSchema) Validate(_ jsonschema.ValidationContext, v interface{}) error { + // Skip validation if no database is provided + if s.db == nil { + return nil + } + + // Skip validation if no constraint is provided + if len(s.UniqueFieldTypesMap) == 0 { + return nil + } + + // Try to assert the provided value as a map, skip validation otherwise + mappedValue, ok := v.(map[string]interface{}) + if !ok { + return nil + } + + qms := []qm.QueryMod{} + + if s.ResourceID != nil { + qms = append(qms, qm.Where("id != ?", *s.ResourceID)) + } + + for k, value := range mappedValue { + // Convert the value to string + v := fmt.Sprint(value) + + fieldType, exists := s.UniqueFieldTypesMap[k] + if !exists { + continue + } + + if fieldType == "string" { + v = fmt.Sprintf(`"%s"`, v) + } + + qms = append(qms, qm.Where(`resource->? = ?`, k, v)) + } + + exists, err := s.ERD.SystemExtensionResources(qms...).Exists(s.ctx, s.db) + if err != nil { + return &jsonschema.ValidationError{ + Message: err.Error(), + } + } + + if exists { + return &jsonschema.ValidationError{ + InstanceLocation: s.ERD.Name, + KeywordLocation: "unique", + Message: ErrUniqueConstraintViolation.Error(), + } + } + + return nil +} + +// UniqueConstraintCompiler is the compiler struct for the unique constraint JSON schema extension +type UniqueConstraintCompiler struct { + ERD *models.ExtensionResourceDefinition + ResourceID *string + ctx context.Context + db boil.ContextExecutor +} + +// UniqueConstraintCompiler implements jsonschema.ExtCompiler +var _ jsonschema.ExtCompiler = (*UniqueConstraintCompiler)(nil) + +// Compile compiles the unique constraint JSON schema extension +func (uc *UniqueConstraintCompiler) Compile( + _ jsonschema.CompilerContext, m map[string]interface{}, +) (jsonschema.ExtSchema, error) { + unique, ok := m["unique"] + if !ok { + // If "unique" is not in the map, skip processing + return nil, nil + } + + uniqueFields, err := assertStringSlice(unique) + if err != nil { + return nil, err + } + + if len(uniqueFields) == 0 { + // unique property is not provided, skip + return nil, nil + } + + requiredFields, err := assertStringSlice(m["required"]) + if err != nil { + return nil, err + } + + requiredMap := make(map[string]bool, len(requiredFields)) + for _, f := range requiredFields { + requiredMap[f] = true + } + + propertiesMap, ok := m["properties"].(map[string]interface{}) + if !ok { + return nil, fmt.Errorf( + `%w: cannot apply unique constraint when "properties" is not provided or invalid`, + ErrInvalidUniqueProperty, + ) + } + + return uc.compileUniqueConstraint(uniqueFields, requiredMap, propertiesMap) +} + +func (uc *UniqueConstraintCompiler) compileUniqueConstraint( + uniqueFields []string, requiredMap map[string]bool, propertiesMap map[string]interface{}, +) (jsonschema.ExtSchema, error) { + // map fieldName => fieldType + resultUniqueFields := make(map[string]string) + + for _, fieldName := range uniqueFields { + if !requiredMap[fieldName] { + return nil, fmt.Errorf( + `%w: unique property needs to be a required property, "%s" is not in "required"`, + ErrInvalidUniqueProperty, + fieldName, + ) + } + + prop, ok := propertiesMap[fieldName] + if !ok { + return nil, fmt.Errorf( + `%w: missing property definition for unique field "%s"`, + ErrInvalidUniqueProperty, + fieldName, + ) + } + + fieldType, ok := prop.(map[string]interface{})["type"].(string) + if !ok || !isValidType(fieldType) { + return nil, fmt.Errorf( + `%w: invalid type "%s" for unique field "%s"`, + ErrInvalidUniqueProperty, + fieldType, + fieldName, + ) + } + + resultUniqueFields[fieldName] = fieldType + } + + return &UniqueConstraintSchema{resultUniqueFields, uc.ERD, uc.ResourceID, uc.ctx, uc.db}, nil +} + +// Checks if the provided field type is valid for unique constraints +func isValidType(fieldType string) bool { + _, ok := map[string]bool{ + "string": true, "number": true, "integer": true, "boolean": true, + }[fieldType] + + return ok +} + +// helper function to assert string slice type +func assertStringSlice(value interface{}) ([]string, error) { + values, ok := value.([]interface{}) + if !ok { + return nil, fmt.Errorf( + `%w: unable to convert %v to string array`, + ErrInvalidUniqueProperty, + reflect.TypeOf(value), + ) + } + + strs := make([]string, len(values)) + + for i, v := range values { + str, ok := v.(string) + if !ok { + return nil, fmt.Errorf( + `%w: unable to convert %v to string`, + ErrInvalidUniqueProperty, + reflect.TypeOf(v), + ) + } + + strs[i] = str + } + + return strs, nil +} diff --git a/pkg/jsonschema/unique_constraint_test.go b/pkg/jsonschema/unique_constraint_test.go new file mode 100644 index 00000000..d81cee4e --- /dev/null +++ b/pkg/jsonschema/unique_constraint_test.go @@ -0,0 +1,271 @@ +package jsonschema + +import ( + "context" + "database/sql" + "testing" + + "github.com/cockroachdb/cockroach-go/v2/testserver" + dbm "github.com/metal-toolbox/governor-api/db" + "github.com/metal-toolbox/governor-api/internal/models" + "github.com/pressly/goose/v3" + "github.com/santhosh-tekuri/jsonschema/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "github.com/volatiletech/sqlboiler/v4/boil" + "github.com/volatiletech/sqlboiler/v4/queries/qm" +) + +type UniqueConstrainTestSuite struct { + suite.Suite + + db *sql.DB +} + +func (s *UniqueConstrainTestSuite) seedTestDB() error { + testData := []string{ + `INSERT INTO extensions (id, name, description, enabled, slug, status) + VALUES ('00000001-0000-0000-0000-000000000001', 'Test Extension', 'some extension', true, 'test-extension', 'online');`, + ` + INSERT INTO extension_resource_definitions (id, name, description, enabled, slug_singular, slug_plural, version, scope, schema, extension_id) + VALUES ('00000001-0000-0000-0000-000000000002', 'Test Resource', 'some-description', true, 'test-resource', 'test-resources', 'v1', 'system', + '{"$id": "v1.person.test-ex-1","$schema": "https://json-schema.org/draft/2020-12/schema","title": "Person","type": "object","unique": ["firstName", "lastName"],"required": ["firstName", "lastName"],"properties": {"firstName": {"type": "string","description": "The person''s first name.","ui": {"hide": true}},"lastName": {"type": "string","description": "The person''s last name."},"age": {"description": "Age in years which must be equal to or greater than zero.","type": "integer","minimum": 0}}}'::jsonb, + '00000001-0000-0000-0000-000000000001'); + `, + `INSERT INTO system_extension_resources (id, resource, extension_resource_definition_id) + VALUES ('00000001-0000-0000-0000-000000000003', '{"age": 10, "firstName": "Hello", "lastName": "World"}'::jsonb, '00000001-0000-0000-0000-000000000002');`, + } + + for _, q := range testData { + _, err := s.db.Query(q) + if err != nil { + return err + } + } + + return nil +} + +func (s *UniqueConstrainTestSuite) SetupSuite() { + ts, err := testserver.NewTestServer() + if err != nil { + panic(err) + } + + s.db, err = sql.Open("postgres", ts.PGURL().String()) + if err != nil { + panic(err) + } + + goose.SetBaseFS(dbm.Migrations) + + if err := goose.Up(s.db, "migrations"); err != nil { + panic("migration failed - could not set up test db") + } + + if err := s.seedTestDB(); err != nil { + panic("db setup failed - could not seed test db: " + err.Error()) + } +} + +func (s *UniqueConstrainTestSuite) TestCompile() { + tests := []struct { + name string + inputMap map[string]interface{} + expectedErr string + }{ + { + name: "no unique key", + inputMap: map[string]interface{}{}, + expectedErr: "", + }, + { + name: "invalid unique field type", + inputMap: map[string]interface{}{ + "unique": 1234, + }, + expectedErr: "unable to convert", + }, + { + name: "unique exists but empty", + inputMap: map[string]interface{}{ + "unique": []interface{}{}, + }, + expectedErr: "", + }, + { + name: "unique exists but required invalid", + inputMap: map[string]interface{}{ + "unique": []interface{}{"a"}, + "required": 1234, + }, + expectedErr: "unable to convert", + }, + { + name: "missing properties", + inputMap: map[string]interface{}{ + "unique": []interface{}{"a"}, + "required": []interface{}{"a"}, + }, + expectedErr: "cannot apply unique constraint when \"properties\" is not provided or invalid", + }, + + { + name: "invalid properties type", + inputMap: map[string]interface{}{ + "unique": []interface{}{"a"}, + "required": []interface{}{"a"}, + "properties": "invalidType", + }, + expectedErr: "cannot apply unique constraint when \"properties\" is not provided or invalid", + }, + { + name: "valid unique, required, and properties", + inputMap: map[string]interface{}{ + "unique": []interface{}{"a"}, + "required": []interface{}{"a"}, + "properties": map[string]interface{}{ + "a": map[string]interface{}{ + "type": "string", + }, + }, + }, + expectedErr: "", + }, + { + name: "unique field not in properties", + inputMap: map[string]interface{}{ + "unique": []interface{}{"b"}, + "required": []interface{}{"b"}, + "properties": map[string]interface{}{ + "a": map[string]interface{}{ + "type": "string", + }, + }, + }, + expectedErr: "missing property definition for unique field", + }, + { + name: "unique property not in required", + inputMap: map[string]interface{}{ + "unique": []interface{}{"a"}, + "required": []interface{}{"b"}, + "properties": map[string]interface{}{ + "a": map[string]interface{}{"type": "string"}, + "b": map[string]interface{}{"type": "string"}, + }, + }, + expectedErr: "unique property needs to be a required property", + }, + } + + for _, tt := range tests { + s.Suite.T().Run(tt.name, func(t *testing.T) { + uc := &UniqueConstraintCompiler{ + ctx: context.Background(), + db: nil, + } + _, err := uc.Compile(jsonschema.CompilerContext{}, tt.inputMap) + if tt.expectedErr == "" { + assert.Nil(t, err) + } else { + assert.Contains(t, err.Error(), tt.expectedErr) + } + }) + } +} + +func (s *UniqueConstrainTestSuite) TestValidate() { + resourceID := "00000001-0000-0000-0000-000000000003" + + tests := []struct { + name string + db boil.ContextExecutor + resourceID *string + value interface{} + uniqueFields map[string]string + expectedErr string + existsReturn bool + existsErr error + }{ + { + name: "no DB provided", + db: nil, + value: map[string]interface{}{"firstName": "test1", "lastName": "test11"}, + uniqueFields: map[string]string{"firstName": "string", "lastName": "string"}, + expectedErr: "", + }, + { + name: "value not a map", + db: s.db, + value: "not-a-map", + uniqueFields: map[string]string{"firstName": "string", "lastName": "string"}, + expectedErr: "", + }, + { + name: "value matches uniqueFields (string)", + db: s.db, + value: map[string]interface{}{"firstName": "Hello", "lastName": "World"}, + uniqueFields: map[string]string{"firstName": "string", "lastName": "string"}, + expectedErr: "unique constraint violation", + }, + { + name: "allow self updates (string)", + db: s.db, + resourceID: &resourceID, + value: map[string]interface{}{"firstName": "Hello", "lastName": "World"}, + uniqueFields: map[string]string{"firstName": "string", "lastName": "string"}, + }, + { + name: "empty unique fields", + db: s.db, + value: map[string]interface{}{"firstName": "Hello", "lastName": "World", "age": 10}, + uniqueFields: map[string]string{}, + }, + { + name: "value matches uniqueFields (int)", + db: s.db, + value: map[string]interface{}{"firstName": "Hello", "age": 10}, + uniqueFields: map[string]string{"firstName": "string", "age": "int"}, + expectedErr: "unique constraint violation", + }, + { + name: "allow self updates (int)", + db: s.db, + resourceID: &resourceID, + value: map[string]interface{}{"firstName": "Hello", "age": 10}, + uniqueFields: map[string]string{"firstName": "string", "age": "int"}, + }, + } + + erd, err := models. + ExtensionResourceDefinitions(qm.Where("id = ?", "00000001-0000-0000-0000-000000000002")). + One(context.Background(), s.db) + if err != nil { + panic(err) + } + + for _, tt := range tests { + s.T().Run(tt.name, func(t *testing.T) { + schema := &UniqueConstraintSchema{ + UniqueFieldTypesMap: tt.uniqueFields, + ERD: erd, + ctx: context.Background(), + db: tt.db, + ResourceID: tt.resourceID, + } + + err := schema.Validate(jsonschema.ValidationContext{}, tt.value) + if tt.expectedErr == "" { + assert.Nil(t, err) + } else { + assert.NotNil(t, err) + assert.Contains(t, err.Error(), tt.expectedErr) + } + }) + } +} + +func TestUniqueConstraintSuite(t *testing.T) { + suite.Run(t, new(UniqueConstrainTestSuite)) +}