resource templating

go.mod

@@ -29,6 +29,7 @@ require (
 	go.uber.org/atomic v1.9.0
 	go.uber.org/zap v1.19.1
 	golang.org/x/tools v0.7.0
+	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 	helm.sh/helm/v3 v3.11.1
 	k8s.io/api v0.26.0
@@ -47,7 +48,6 @@ require (
 	github.com/rivo/uniseg v0.4.4 // indirect
 	golang.org/x/mod v0.9.0 // indirect
 	golang.org/x/oauth2 v0.4.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
 
 replace github.com/smacker/go-tree-sitter => github.com/klothoplatform/go-tree-sitter v0.1.1

pkg/core/construct_graph.go

@@ -129,6 +129,9 @@ func (cg *ConstructGraph) RemoveDependency(source ResourceId, dest ResourceId) e
 func (cg *ConstructGraph) GetConstruct(key ResourceId) BaseConstruct {
 	return cg.underlying.GetVertex(key.String())
 }
+func (cg *ConstructGraph) GetDependency(source ResourceId, target ResourceId) *graph.Edge[BaseConstruct] {
+	return cg.underlying.GetEdge(source.String(), target.String())
+}
 
 func (cg *ConstructGraph) GetResource(id ResourceId) Resource {
 	c := cg.GetConstruct(id)
@@ -138,10 +141,6 @@ func (cg *ConstructGraph) GetResource(id ResourceId) Resource {
 	return nil
 }
 
-func (cg *ConstructGraph) GetDependency(source ResourceId, target ResourceId) *graph.Edge[BaseConstruct] {
-	return cg.underlying.GetEdge(source.String(), target.String())
-}
-
 func ListConstructs[C BaseConstruct](cg *ConstructGraph) []C {
 	var result []C
 	for _, v := range cg.underlying.GetAllVertices() {

pkg/core/errors.go

@@ -28,9 +28,12 @@ type (
 	}
 
 	OperationalResourceError struct {
-		Needs    []string
-		Resource Resource
-		Cause    error
+		Needs      []string
+		Count      int
+		Resource   Resource
+		Parent     Resource
+		MustCreate bool
+		Cause      error
 	}
 )
 

pkg/core/resources.go

@@ -1,6 +1,7 @@
 package core
 
 import (
+	"fmt"
 	"reflect"
 	"strings"
 
@@ -33,13 +34,13 @@ type (
 	// DeleteContext is supposed to tell us when we are able to delete a resource based on its dependencies
 	DeleteContext struct {
 		// RequiresNoUpstream is a boolean that tells us if deletion relies on there being no upstream resources
-		RequiresNoUpstream bool
+		RequiresNoUpstream bool `yaml:"requires_no_upstream" toml:"requires_no_upstream"`
 		// RequiresNoDownstream is a boolean that tells us if deletion relies on there being no downstream resources
-		RequiresNoDownstream bool
+		RequiresNoDownstream bool `yaml:"requires_no_downstream" toml:"requires_no_downstream"`
 		// RequiresExplicitDelete is a boolean that tells us if deletion relies on the resource being explicitly deleted
-		RequiresExplicitDelete bool
+		RequiresExplicitDelete bool `yaml:"requires_explicit_delete" toml:"requires_explicit_delete"`
 		// RequiresNoUpstreamOrDownstream is a boolean that tells us if deletion relies on there being no upstream or downstream resources
-		RequiresNoUpstreamOrDownstream bool
+		RequiresNoUpstreamOrDownstream bool `yaml:"requires_no_upstream_or_downstream" toml:"requires_no_upstream_or_downstream"`
 	}
 
 	// Resource describes a resource at the provider, infrastructure level
@@ -93,6 +94,110 @@ type (
 	}
 )
 
+type (
+	// ResourceTemplate defines how rules are handled by the engine in terms of making sure they are functional in the graph
+	ResourceTemplate struct {
+		// Provider refers to the resources provider
+		Provider string `json:"provider" yaml:"provider"`
+		// Type refers to the unique type identifier of the resource
+		Type string `json:"type" yaml:"type"`
+		// Rules defines a set of rules that must pass checks and actions which must be carried out to make a resource operational
+		Rules OperationalTempalte `json:"rules" yaml:"rules"`
+		// Configuration specifies how to act on any intrinsic values of a resource to make it operational
+		Configuration []Configuration `json:"configuration" yaml:"configuration"`
+		// SanitizationRules defines a set of rules that are used to ensure the resource's name is valid
+		NameSanitization Sanitization `json:"sanitization" yaml:"sanitization"`
+		// DeleteContext defines the context in which a resource can be deleted
+		DeleteContext DeleteContext `json:"delete_context" yaml:"delete_context"`
+	}
+
+	// OperationalTempalte defines a set of rules that must pass checks and actions which must be carried out to make a resource operational
+	OperationalTempalte struct {
+		// Downstream defines a set of rules that exist downstream of the resource that must pass checks and actions which must be carried out to make a resource operational
+		Downstream []OperationalRule `json:"downstream" yaml:"downstream"`
+		// Upstream defines a set of rules that exist upstream of the resource that must pass checks and actions which must be carried out to make a resource operational
+		Upstream []OperationalRule `json:"upstream" yaml:"upstream"`
+	}
+
+	// OperationalRule defines a rule that must pass checks and actions which must be carried out to make a resource operational
+	OperationalRule struct {
+		// Enforcement defines how the rule should be enforced
+		Enforcement OperationEnforcement `json:"enforcement" yaml:"enforcement"`
+		// ResourceTypes defines the resource types that the rule should be enforced on. Resource types must be specified if classifications is not specified
+		ResourceTypes []string `json:"resource_types" yaml:"resource_types"`
+		// Classifications defines the classifications that the rule should be enforced on. Classifications must be specified if resource types is not specified
+		Classifications []string `json:"classifications" yaml:"classifications"`
+		// SetField defines the field on the resource that should be set to the resource that satisfies the rule
+		SetField string `json:"set_field" yaml:"set_field"`
+		// RemoveDirectDependency defines if the direct dependency between the resource and the rule's resource(s) that satisfies the rule should be removed
+		RemoveDirectDependency bool `json:"remove_direct_dependency" yaml:"remove_direct_dependency"`
+		// NumNeeded defines the number of resources that must satisfy the rule
+		NumNeeded int `json:"num_needed" yaml:"num_needed"`
+		// Rules defines a set of sub rules that will be carried out based on the evaluation of the initial parent rule
+		Rules []OperationalRule `json:"rules" yaml:"rules"`
+		// UnsatisfiedAction defines what action should be taken if the rule is not satisfied
+		UnsatisfiedAction UnsatisfiedAction `json:"unsatisfied_action" yaml:"unsatisfied_action"`
+	}
+
+	// UnsatisfiedAction defines what action should be taken if the rule is not satisfied
+	UnsatisfiedAction struct {
+		// Operation defines what action should be taken if the rule is not satisfied
+		Operation UnsatisfiedActionOperation `json:"operation" yaml:"operation"`
+		// DefaultType defines the default type of resource that should be acted upon if the rule is not satisfied
+		DefaultType string `json:"default_type" yaml:"default_type"`
+		// Unique defines if the resource that is created should be unique
+		Unique bool `json:"unique" yaml:"unique"`
+	}
+
+	// Configuration defines how to act on any intrinsic values of a resource to make it operational
+	Configuration struct {
+		// Fields defines a field that should be set on the resource
+		Field string `json:"field" yaml:"field"`
+		// Value defines the value that should be set on the resource
+		Value any `json:"value" yaml:"value"`
+		// ZeroValueAllowed defines if the value can be set to the zero value of the field
+		ZeroValueAllowed bool `json:"zero_value_allowed" yaml:"zero_value_allowed"`
+	}
+
+	Sanitization struct {
+		Rules     []SanitizationRule `json:"rules" yaml:"rules"`
+		MaxLength int                `json:"max_length" yaml:"max_length"`
+		MinLength int                `json:"min_length" yaml:"min_length"`
+	}
+	SanitizationRule struct {
+		Pattern     string `json:"pattern" yaml:"pattern"`
+		Replacement string `json:"replacement" yaml:"replacement"`
+	}
+
+	// OperationEnforcement defines how the rule should be enforced
+	OperationEnforcement string
+	// UnsatisfiedActionOperation defines what action should be taken if the rule is not satisfied
+	UnsatisfiedActionOperation string
+)
+
+const (
+	// ExactlyOne defines that the rule should be enforced on exactly one resource
+	ExactlyOne OperationEnforcement = "exactly_one"
+	// Conditional defines that the rule should be enforced on a resource if it exists
+	Conditional OperationEnforcement = "conditional"
+	// AnyAvailable defines that the rule should be enforced on any available resource
+	AnyAvailable OperationEnforcement = "any_available"
+
+	// CreateUnsatisfiedResource defines that a resource should be created if the rule is not satisfied
+	CreateUnsatisfiedResource UnsatisfiedActionOperation = "create"
+	// ErrorUnsatisfiedResource defines that an error should be returned if the rule is not satisfied
+	ErrorUnsatisfiedResource UnsatisfiedActionOperation = "error"
+)
+
+func (or *OperationalRule) String() string {
+	if or.ResourceTypes != nil {
+		return fmt.Sprintf("%s %s", or.Enforcement, or.ResourceTypes)
+	} else if or.Classifications != nil {
+		return fmt.Sprintf("%s %s", or.Enforcement, or.Classifications)
+	}
+	return string(or.Enforcement)
+}
+
 const (
 	Compute Functionality = "compute"
 	Cluster Functionality = "cluster"

pkg/engine/classification/classification.go

@@ -98,7 +98,7 @@ func (c *ClassificationDocument) GetFunctionality(resource core.Resource) core.F
 func (c *ClassificationDocument) ResourceContainsClassifications(resource core.Resource, needs []string) bool {
 	classifications := c.GetClassification(resource)
 	for _, need := range needs {
-		if !collectionutil.Contains(classifications.Is, need) || resource.Id().Type == need {
+		if !collectionutil.Contains(classifications.Is, need) && resource.Id().Type != need {
 			return false
 		}
 	}

pkg/engine/dataflow.go

@@ -38,6 +38,8 @@ func (e *Engine) GetDataFlowDag() *core.ResourceGraph {
 		awsResources.LOAD_BALANCER_TYPE,
 		awsResources.CLOUDFRONT_DISTRIBUTION_TYPE,
 		awsResources.ROUTE_53_HOSTED_ZONE_TYPE,
+		awsResources.SNS_TOPIC_TYPE,
+		awsResources.SQS_QUEUE_TYPE,
 		k8sResources.DEPLOYMENT_TYPE,
 		k8sResources.POD_TYPE,
 		k8sResources.HELM_CHART_TYPE,

pkg/engine/engine.go

@@ -24,6 +24,8 @@ type (
 		ClassificationDocument *classification.ClassificationDocument
 		// The constructs which the engine understands
 		Constructs []core.Construct
+		// The templates that the engine uses to make resources operational
+		ResourceTemplates map[string]*core.ResourceTemplate
 		// The context of the engine
 		Context EngineContext
 	}
@@ -61,12 +63,19 @@ type (
 )
 
 func NewEngine(providers map[string]provider.Provider, kb knowledgebase.EdgeKB, constructs []core.Construct) *Engine {
-	return &Engine{
+	engine := &Engine{
 		Providers:              providers,
 		KnowledgeBase:          kb,
 		Constructs:             constructs,
 		ClassificationDocument: classification.BaseClassificationDocument,
 	}
+	engine.ResourceTemplates = make(map[string]*core.ResourceTemplate)
+	for _, p := range providers {
+		for resType, template := range p.GetOperationalTempaltes() {
+			engine.ResourceTemplates[fmt.Sprintf("%s:%s", p.Name(), resType)] = template
+		}
+	}
+	return engine
 }
 
 func (e *Engine) LoadClassifications(classificationPath string, fs embed.FS) error {

pkg/engine/enginetesting/kb_mock.go

@@ -5,8 +5,11 @@ import (
 )
 
 var MockKB = knowledgebase.Build(
-	knowledgebase.EdgeBuilder[*mockResource1, *mockResource2]{},
-	knowledgebase.EdgeBuilder[*mockResource1, *mockResource3]{},
-	knowledgebase.EdgeBuilder[*mockResource1, *mockResource4]{},
-	knowledgebase.EdgeBuilder[*mockResource2, *mockResource3]{},
+	knowledgebase.EdgeBuilder[*MockResource1, *MockResource2]{},
+	knowledgebase.EdgeBuilder[*MockResource1, *MockResource3]{},
+	knowledgebase.EdgeBuilder[*MockResource1, *MockResource4]{},
+	knowledgebase.EdgeBuilder[*MockResource2, *MockResource3]{},
+	// used for operational resource testing
+	knowledgebase.EdgeBuilder[*MockResource5, *MockResource1]{},
+	knowledgebase.EdgeBuilder[*MockResource5, *MockResource2]{},
 )

pkg/engine/enginetesting/provider_mock.go

@@ -10,13 +10,13 @@ type MockProvider struct {
 func (p *MockProvider) CreateResourceFromId(id core.ResourceId, dag *core.ConstructGraph) (core.Resource, error) {
 	switch id.Type {
 	case "mock1":
-		return &mockResource1{Name: id.Name}, nil
+		return &MockResource1{Name: id.Name}, nil
 	case "mock2":
-		return &mockResource2{Name: id.Name}, nil
+		return &MockResource2{Name: id.Name}, nil
 	case "mock3":
-		return &mockResource3{Name: id.Name}, nil
+		return &MockResource3{Name: id.Name}, nil
 	case "mock4":
-		return &mockResource4{Name: id.Name}, nil
+		return &MockResource4{Name: id.Name}, nil
 	}
 	return nil, nil
 }
@@ -25,13 +25,13 @@ func (p *MockProvider) ExpandConstruct(construct core.Construct, cg *core.Constr
 	case *core.ExecutionUnit:
 		switch constructType {
 		case "mock1":
-			mock1 := &mockResource1{Name: c.Name, ConstructRefs: core.BaseConstructSetOf(c)}
+			mock1 := &MockResource1{Name: c.Name, ConstructRefs: core.BaseConstructSetOf(c)}
 			dag.AddResource(mock1)
 			return []core.Resource{mock1}, nil
 		}
 		return nil, nil
 	case *core.Orm:
-		res := &mockResource3{Name: c.Name, ConstructRefs: core.BaseConstructSetOf(c)}
+		res := &MockResource3{Name: c.Name, ConstructRefs: core.BaseConstructSetOf(c)}
 		dag.AddResource(res)
 		return []core.Resource{res}, nil
 	}
@@ -40,12 +40,17 @@ func (p *MockProvider) ExpandConstruct(construct core.Construct, cg *core.Constr
 
 func (p *MockProvider) ListResources() []core.Resource {
 	return []core.Resource{
-		&mockResource1{},
-		&mockResource2{},
-		&mockResource3{},
-		&mockResource4{},
+		&MockResource1{},
+		&MockResource2{},
+		&MockResource3{},
+		&MockResource4{},
 	}
 }
+
+func (p *MockProvider) GetOperationalTempaltes() map[string]*core.ResourceTemplate {
+	return map[string]*core.ResourceTemplate{}
+}
+
 func (p *MockProvider) Name() string {
 	return "mock"
 }