feat(indexer/base): schema and value validation

indexer/base/CHANGELOG.md

@@ -0,0 +1,37 @@
+<!--
+Guiding Principles:
+
+Changelogs are for humans, not machines.
+There should be an entry for every single version.
+The same types of changes should be grouped.
+Versions and sections should be linkable.
+The latest version comes first.
+The release date of each version is displayed.
+Mention whether you follow Semantic Versioning.
+
+Usage:
+
+Change log entries are to be added to the Unreleased section under the
+appropriate stanza (see below). Each entry should ideally include a tag and
+the Github issue reference in the following format:
+
+* (<tag>) \#<issue-number> message
+
+The issue numbers will later be link-ified during the release process so you do
+not have to worry about including a link manually, but you can if you wish.
+
+Types of changes (Stanzas):
+
+"Features" for new features.
+"Improvements" for changes in existing functionality.
+"Deprecated" for soon-to-be removed features.
+"Bug Fixes" for any bug fixes.
+"Client Breaking" for breaking Protobuf, gRPC and REST routes used by end-users.
+"CLI Breaking" for breaking CLI commands.
+"API Breaking" for breaking exported APIs used by developers building on SDK.
+Ref: https://keepachangelog.com/en/1.0.0/
+-->
+
+# Changelog
+
+## [Unreleased]

indexer/base/README.md

@@ -0,0 +1,5 @@
+# Indexer Base
+
+The indexer base module is designed to provide a stable, zero-dependency base layer for the built-in indexer functionality. Packages that integrate with the indexer should feel free to depend on this package without fear of any external dependencies being pulled in.
+
+The basic types for specifying index sources, targets and decoders are provided here along with a basic engine that ties these together. A package wishing to be an indexing source could accept an instance of `Engine` directly to be compatible with indexing. A package wishing to be a decoder can use the `Entity` and `Table` types. A package defining an indexing target should implement the `Indexer` interface.

indexer/base/decoder.go

@@ -0,0 +1,27 @@
+package indexerbase
+
+// DecodableModule is an interface that modules can implement to provide a ModuleDecoder.
+// Usually these modules would also implement appmodule.AppModule, but that is not included
+// to keep this package free of any dependencies.
+type DecodableModule interface {
+
+	// ModuleDecoder returns a ModuleDecoder for the module.
+	ModuleDecoder() (ModuleDecoder, error)
+}
+
+// ModuleDecoder is a struct that contains the schema and a KVDecoder for a module.
+type ModuleDecoder struct {
+	// Schema is the schema for the module.
+	Schema ModuleSchema
+
+	// KVDecoder is a function that decodes a key-value pair into an ObjectUpdate.
+	// If modules pass logical updates directly to the engine and don't require logical decoding of raw bytes,
+	// then this function should be nil.
+	KVDecoder KVDecoder
+}
+
+// KVDecoder is a function that decodes a key-value pair into an ObjectUpdate.
+// If the KV-pair doesn't represent an object update, the function should return false
+// as the second return value. Error should only be non-nil when the decoder expected
+// to parse a valid update and was unable to.
+type KVDecoder = func(key, value []byte) (ObjectUpdate, bool, error)

indexer/base/enum.go

@@ -0,0 +1,10 @@
+package indexerbase
+
+// EnumDefinition represents the definition of an enum type.
+type EnumDefinition struct {
+	// Name is the name of the enum type.
+	Name string
+
+	// Values is a list of distinct values that are part of the enum type.
+	Values []string
+}

indexer/base/field.go

@@ -0,0 +1,142 @@
+package indexerbase
+
+import "fmt"
+
+// Field represents a field in an object type.
+type Field struct {
+	// Name is the name of the field.
+	Name string
+
+	// Kind is the basic type of the field.
+	Kind Kind
+
+	// Nullable indicates whether null values are accepted for the field.
+	Nullable bool
+
+	// AddressPrefix is the address prefix of the field's kind, currently only used for Bech32AddressKind.
+	AddressPrefix string
+
+	// EnumDefinition is the definition of the enum type and is only valid when Kind is EnumKind.
+	EnumDefinition EnumDefinition
+}
+
+// Validate validates the field.
+func (c Field) Validate() error {
+	// non-empty name
+	if c.Name == "" {
+		return fmt.Errorf("field name cannot be empty")
+	}
+
+	// valid kind
+	if err := c.Kind.Validate(); err != nil {
+		return fmt.Errorf("invalid field type for %q: %w", c.Name, err)
+	}
+
+	// address prefix only valid with Bech32AddressKind
+	if c.Kind == Bech32AddressKind && c.AddressPrefix == "" {
+		return fmt.Errorf("missing address prefix for field %q", c.Name)
+	} else if c.Kind != Bech32AddressKind && c.AddressPrefix != "" {
+		return fmt.Errorf("address prefix is only valid for field %q with type Bech32AddressKind", c.Name)
+	}
+
+	// enum definition only valid with EnumKind
+	if c.Kind == EnumKind {
+		if err := c.EnumDefinition.Validate(); err != nil {
+			return fmt.Errorf("invalid enum definition for field %q: %w", c.Name, err)
+		}
+	} else if c.Kind != EnumKind && c.EnumDefinition.Name != "" && c.EnumDefinition.Values != nil {
+		return fmt.Errorf("enum definition is only valid for field %q with type EnumKind", c.Name)
+	}
+
+	return nil
+}
+
+// Validate validates the enum definition.
+func (e EnumDefinition) Validate() error {
+	if e.Name == "" {
+		return fmt.Errorf("enum definition name cannot be empty")
+	}
+	if len(e.Values) == 0 {
+		return fmt.Errorf("enum definition values cannot be empty")
+	}
+	seen := make(map[string]bool, len(e.Values))
+	for i, v := range e.Values {
+		if v == "" {
+			return fmt.Errorf("enum definition value at index %d cannot be empty for enum %s", i, e.Name)
+		}
+		if seen[v] {
+			return fmt.Errorf("duplicate enum definition value %q for enum %s", v, e.Name)
+		}
+		seen[v] = true
+	}
+	return nil
+}
+
+// ValidateValue validates that the value conforms to the field's kind and nullability.
+// It currently does not do any validation that IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind
+// values are valid for their respective types behind conforming to the correct go type.
+func (c Field) ValidateValue(value any) error {
+	if value == nil {
+		if !c.Nullable {
+			return fmt.Errorf("field %q cannot be null", c.Name)
+		}
+		return nil
+	}
+	return c.Kind.ValidateValueType(value)
+}
+
+// ValidateKey validates that the value conforms to the set of fields as a Key in an EntityUpdate.
+// See EntityUpdate.Key for documentation on the requirements of such values.
+func ValidateKey(fields []Field, value any) error {
+	if len(fields) == 0 {
+		return nil
+	}
+
+	if len(fields) == 1 {
+		return fields[0].ValidateValue(value)
+	}
+
+	values, ok := value.([]any)
+	if !ok {
+		return fmt.Errorf("expected slice of values for key fields, got %T", value)
+	}
+
+	if len(fields) != len(values) {
+		return fmt.Errorf("expected %d key fields, got %d values", len(fields), len(value.([]any)))
+	}
+	for i, field := range fields {
+		if err := field.ValidateValue(values[i]); err != nil {
+			return fmt.Errorf("invalid value for key field %q: %w", field.Name, err)
+		}
+	}
+	return nil
+}
+
+// ValidateValue validates that the value conforms to the set of fields as a Value in an EntityUpdate.
+// See EntityUpdate.Value for documentation on the requirements of such values.
+func ValidateValue(fields []Field, value any) error {
+	valueUpdates, ok := value.(ValueUpdates)
+	if ok {
+		fieldMap := map[string]Field{}
+		for _, field := range fields {
+			fieldMap[field.Name] = field
+		}
+		var errs []error
+		valueUpdates.Iterate(func(fieldName string, value any) bool {
+			field, ok := fieldMap[fieldName]
+			if !ok {
+				errs = append(errs, fmt.Errorf("unknown field %q in value updates", fieldName))
+			}
+			if err := field.ValidateValue(value); err != nil {
+				errs = append(errs, fmt.Errorf("invalid value for field %q: %w", fieldName, err))
+			}
+			return true
+		})
+		if len(errs) > 0 {
+			return fmt.Errorf("validation errors: %v", errs)
+		}
+		return nil
+	} else {
+		return ValidateKey(fields, value)
+	}
+}

indexer/base/go.mod

@@ -0,0 +1,7 @@
+module cosmossdk.io/indexer/base
+
+// NOTE: this go.mod should have zero dependencies and remain on go 1.12 to stay compatible
+// with all known production releases of the Cosmos SDK. This is to ensure that all historical
+// apps could be patched to support indexing if desired.
+
+go 1.12