Skip to content

Commit

Permalink
remove edge checks if edges can be removed (#832)
Browse files Browse the repository at this point in the history
  • Loading branch information
jhsinger-klotho authored Dec 20, 2023
1 parent ad21edf commit 90e0de6
Show file tree
Hide file tree
Showing 16 changed files with 1,384 additions and 180 deletions.
37 changes: 1 addition & 36 deletions pkg/engine2/constraints.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"errors"
"fmt"

"github.com/dominikbraun/graph"
construct "github.com/klothoplatform/klotho/pkg/construct2"
"github.com/klothoplatform/klotho/pkg/engine2/constraints"
"github.com/klothoplatform/klotho/pkg/engine2/reconciler"
Expand Down Expand Up @@ -120,46 +119,12 @@ func applyEdgeConstraint(ctx solution_context.SolutionContext, constraint constr
}
}

removePath := func() error {
paths, err := graph.AllPathsBetween(ctx.DataflowGraph(), constraint.Target.Source, constraint.Target.Target)
switch {
case errors.Is(err, graph.ErrTargetNotReachable):
return nil
case err != nil:
return err
}

var errs error

// first we will remove all dependencies that make up the paths from the constraints source to target
for _, path := range paths {
for i, res := range path {
if i == 0 {
continue
}
errs = errors.Join(errs, ctx.OperationalView().RemoveEdge(path[i-1], res))
}
}
if errs != nil {
return errs
}

// Next we will try to delete any node in those paths in case they no longer are required for the architecture
// We will pass the explicit field as false so that explicitly added resources do not get deleted
for _, path := range paths {
for _, resource := range path {
errs = errors.Join(errs, reconciler.RemoveResource(ctx, resource, false))
}
}
return errs
}

switch constraint.Operator {
case constraints.MustExistConstraintOperator:
return ctx.OperationalView().AddEdge(constraint.Target.Source, constraint.Target.Target)

case constraints.MustNotExistConstraintOperator:
return removePath()
return reconciler.RemovePath(constraint.Target.Source, constraint.Target.Target, ctx)
}
return nil
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/engine2/operational_eval/dot.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,10 @@ func keyAttributes(eval *Evaluator, key Key) map[string]string {
if key.PathSatisfication.Classification != "" {
extra = append(extra, fmt.Sprintf("<%s>", key.PathSatisfication.Classification))
}
if propertyReferenceChangesBoundary(key.PathSatisfication.Target) {
if key.PathSatisfication.Target.PropertyReferenceChangesBoundary() {
extra = append(extra, fmt.Sprintf("target#%s", key.PathSatisfication.Target.PropertyReference))
}
if propertyReferenceChangesBoundary(key.PathSatisfication.Source) {
if key.PathSatisfication.Source.PropertyReferenceChangesBoundary() {
extra = append(extra, fmt.Sprintf("source#%s", key.PathSatisfication.Target.PropertyReference))
}
if len(extra) > 0 {
Expand Down
4 changes: 2 additions & 2 deletions pkg/engine2/operational_eval/evaluator.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,10 +125,10 @@ func (key Key) String() string {
if key.PathSatisfication.Classification != "" {
args = append(args, fmt.Sprintf("<%s>", key.PathSatisfication.Classification))
}
if propertyReferenceChangesBoundary(key.PathSatisfication.Target) {
if key.PathSatisfication.Target.PropertyReferenceChangesBoundary() {
args = append(args, fmt.Sprintf("target#%s", key.PathSatisfication.Target.PropertyReference))
}
if propertyReferenceChangesBoundary(key.PathSatisfication.Source) {
if key.PathSatisfication.Source.PropertyReferenceChangesBoundary() {
args = append(args, fmt.Sprintf("source#%s", key.PathSatisfication.Target.PropertyReference))
}
return fmt.Sprintf("Expand(%s)", strings.Join(args, ", "))
Expand Down
2 changes: 1 addition & 1 deletion pkg/engine2/operational_eval/graph.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ func (eval *Evaluator) pathVertices(source, target construct.ResourceId) (graphC
// We are checking to see if either of the source or target nodes will change due to property references,
// if there are property references we want to ensure the correct dependency ordering is in place so
// we cannot yet split the expansion vertex up or build the temp graph
if propertyReferenceChangesBoundary(satisfication.Source) || propertyReferenceChangesBoundary(satisfication.Target) {
if satisfication.Source.PropertyReferenceChangesBoundary() || satisfication.Target.PropertyReferenceChangesBoundary() {
buildTempGraph = false
}

Expand Down
73 changes: 1 addition & 72 deletions pkg/engine2/operational_eval/vertex_path_expand.go
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,7 @@ func (v *pathExpandVertex) getExpansionsToRun(eval *Evaluator) ([]path_selection
return nil, fmt.Errorf("could not find target resource %s: %w", v.Edge.Target, err)
}
edge := construct.ResourceEdge{Source: sourceRes, Target: targetRes}
expansions, err := DeterminePathSatisfactionInputs(eval.Solution, v.Satisfication, edge)
expansions, err := path_selection.DeterminePathSatisfactionInputs(eval.Solution, v.Satisfication, edge)
if err != nil {
errs = errors.Join(errs, err)
}
Expand Down Expand Up @@ -510,68 +510,6 @@ func (v *pathExpandVertex) Dependencies(eval *Evaluator) (graphChanges, error) {
return changes, errs
}

func DeterminePathSatisfactionInputs(
sol solution_context.SolutionContext,
satisfaction knowledgebase.EdgePathSatisfaction,
edge construct.ResourceEdge,
) (expansions []path_selection.ExpansionInput, errs error) {
srcIds := []construct.ResourceId{edge.Source.ID}
targetIds := []construct.ResourceId{edge.Target.ID}
var err error
if propertyReferenceChangesBoundary(satisfaction.Source) {
srcIds, err = solution_context.GetResourcesFromPropertyReference(sol, edge.Source.ID, satisfaction.Source.PropertyReference)
if err != nil {
errs = errors.Join(errs, fmt.Errorf(
"failed to determine path satisfaction inputs. could not find resource %s: %w",
edge.Source.ID, err,
))
}
}
if propertyReferenceChangesBoundary(satisfaction.Target) {
targetIds, err = solution_context.GetResourcesFromPropertyReference(sol, edge.Target.ID, satisfaction.Target.PropertyReference)
if err != nil {
errs = errors.Join(errs, fmt.Errorf(
"failed to determine path satisfaction inputs. could not find resource %s: %w",
edge.Target.ID, err,
))

}
}

for _, srcId := range srcIds {
for _, targetId := range targetIds {
if srcId == targetId {
continue
}
src, err := sol.RawView().Vertex(srcId)
if err != nil {
errs = errors.Join(errs, fmt.Errorf(
"failed to determine path satisfaction inputs. could not find resource %s: %w",
srcId, err,
))
continue
}

target, err := sol.RawView().Vertex(targetId)
if err != nil {
errs = errors.Join(errs, fmt.Errorf(
"failed to determine path satisfaction inputs. could not find resource %s: %w",
targetId, err,
))
continue
}

e := construct.ResourceEdge{Source: src, Target: target}
exp := path_selection.ExpansionInput{
Dep: e,
Classification: satisfaction.Classification,
}
expansions = append(expansions, exp)
}
}
return
}

// getDepsForPropertyRef takes a property reference and recurses down until the property is not filled in on the resource
// When we reach resources with missing property references, we know they are the property vertex keys we must depend on
func getDepsForPropertyRef(
Expand Down Expand Up @@ -605,12 +543,3 @@ func getDepsForPropertyRef(
}
return keys
}

// propertyReferenceChangesBoundary returns whether the [PathSatisfactionRoute] changes the boundary of the path
// via satisfaction rules. Note: validity checks do not change the boundary, so those return false.
func propertyReferenceChangesBoundary(v knowledgebase.PathSatisfactionRoute) bool {
if v.Validity != "" {
return false
}
return v.PropertyReference != ""
}
34 changes: 32 additions & 2 deletions pkg/engine2/path_selection/candidate_weight.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@ package path_selection
import (
"errors"

"github.com/dominikbraun/graph"
"github.com/klothoplatform/klotho/pkg/collectionutil"
construct "github.com/klothoplatform/klotho/pkg/construct2"
"github.com/klothoplatform/klotho/pkg/engine2/operational_rule"
"github.com/klothoplatform/klotho/pkg/engine2/solution_context"
"github.com/klothoplatform/klotho/pkg/graph_addons"
knowledgebase "github.com/klothoplatform/klotho/pkg/knowledge_base2"
)

Expand Down Expand Up @@ -52,7 +53,7 @@ func determineCandidateWeight(
weight += 9
}

undirected, err := operational_rule.BuildUndirectedGraph(ctx)
undirected, err := BuildUndirectedGraph(ctx)
if err != nil {
errs = errors.Join(errs, err)
return
Expand Down Expand Up @@ -97,3 +98,32 @@ func determineCandidateWeight(
weight += availableWeight
return
}

func BuildUndirectedGraph(ctx solution_context.SolutionContext) (construct.Graph, error) {
undirected := graph.NewWithStore(
construct.ResourceHasher,
graph_addons.NewMemoryStore[construct.ResourceId, *construct.Resource](),
)
err := undirected.AddVerticesFrom(ctx.RawView())
if err != nil {
return nil, err
}
edges, err := ctx.RawView().Edges()
if err != nil {
return nil, err
}
for _, e := range edges {
weight := 1
// increase weights for edges that are connected to a functional resource
if knowledgebase.GetFunctionality(ctx.KnowledgeBase(), e.Source) != knowledgebase.Unknown {
weight = 1000
} else if knowledgebase.GetFunctionality(ctx.KnowledgeBase(), e.Target) != knowledgebase.Unknown {
weight = 1000
}
err := undirected.AddEdge(e.Source, e.Target, graph.EdgeWeight(weight))
if err != nil {
return nil, err
}
}
return undirected, nil
}
159 changes: 159 additions & 0 deletions pkg/engine2/path_selection/paths.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
package path_selection

import (
"errors"
"fmt"

"github.com/dominikbraun/graph"
construct "github.com/klothoplatform/klotho/pkg/construct2"
"github.com/klothoplatform/klotho/pkg/engine2/solution_context"
knowledgebase "github.com/klothoplatform/klotho/pkg/knowledge_base2"
)

func GetPaths(
sol solution_context.SolutionContext,
source, target construct.ResourceId,
pathValidityChecks func(source, target construct.ResourceId, path []construct.ResourceId) bool,
hasPathCheck bool,
) ([]construct.Path, error) {
var errs error
var result []construct.Path
pathsCache := map[construct.SimpleEdge][][]construct.ResourceId{}
pathSatisfactions, err := sol.KnowledgeBase().GetPathSatisfactionsFromEdge(source, target)
if err != nil {
return result, err
}
sourceRes, err := sol.RawView().Vertex(source)
if err != nil {
return result, fmt.Errorf("has path could not find source resource %s: %w", source, err)
}
targetRes, err := sol.RawView().Vertex(target)
if err != nil {
return result, fmt.Errorf("has path could not find target resource %s: %w", target, err)
}
edge := construct.ResourceEdge{Source: sourceRes, Target: targetRes}
for _, satisfaction := range pathSatisfactions {
expansions, err := DeterminePathSatisfactionInputs(sol, satisfaction, edge)
if err != nil {
return result, err
}
for _, expansion := range expansions {
simple := construct.SimpleEdge{Source: expansion.Dep.Source.ID, Target: expansion.Dep.Target.ID}
paths, found := pathsCache[simple]
if !found {
var err error
paths, err = graph.AllPathsBetween(sol.RawView(), expansion.Dep.Source.ID, expansion.Dep.Target.ID)
if err != nil {
errs = errors.Join(errs, err)
continue
}
pathsCache[simple] = paths
}
if len(paths) == 0 {
return result, nil
}
// we have to track the result of each expansion because if we cant find a path for a single expansion
// we denote that we dont have an actual path from src -> target
var expansionResult []construct.ResourceId
if expansion.Classification != "" {
PATHS:
for _, path := range paths {
for i, res := range path {
if i == 0 {
continue
}
if et := sol.KnowledgeBase().GetEdgeTemplate(path[i-1], res); et != nil && et.DirectEdgeOnly {
continue PATHS
}

}
if !PathSatisfiesClassification(sol.KnowledgeBase(), path, expansion.Classification) {
continue PATHS
}
if !pathValidityChecks(source, target, path) {
continue PATHS
}
result = append(result, path)
expansionResult = path
if hasPathCheck {
break
}
}
} else {
expansionResult = paths[0]
for _, path := range paths {
result = append(result, path)
}
if hasPathCheck {
break
}
}
if expansionResult == nil {
return nil, nil
}
}
}
return result, nil
}

func DeterminePathSatisfactionInputs(
sol solution_context.SolutionContext,
satisfaction knowledgebase.EdgePathSatisfaction,
edge construct.ResourceEdge,
) (expansions []ExpansionInput, errs error) {
srcIds := []construct.ResourceId{edge.Source.ID}
targetIds := []construct.ResourceId{edge.Target.ID}
var err error
if satisfaction.Source.PropertyReferenceChangesBoundary() {
srcIds, err = solution_context.GetResourcesFromPropertyReference(sol, edge.Source.ID, satisfaction.Source.PropertyReference)
if err != nil {
errs = errors.Join(errs, fmt.Errorf(
"failed to determine path satisfaction inputs. could not find resource %s: %w",
edge.Source.ID, err,
))
}
}
if satisfaction.Target.PropertyReferenceChangesBoundary() {
targetIds, err = solution_context.GetResourcesFromPropertyReference(sol, edge.Target.ID, satisfaction.Target.PropertyReference)
if err != nil {
errs = errors.Join(errs, fmt.Errorf(
"failed to determine path satisfaction inputs. could not find resource %s: %w",
edge.Target.ID, err,
))

}
}

for _, srcId := range srcIds {
for _, targetId := range targetIds {
if srcId == targetId {
continue
}
src, err := sol.RawView().Vertex(srcId)
if err != nil {
errs = errors.Join(errs, fmt.Errorf(
"failed to determine path satisfaction inputs. could not find resource %s: %w",
srcId, err,
))
continue
}

target, err := sol.RawView().Vertex(targetId)
if err != nil {
errs = errors.Join(errs, fmt.Errorf(
"failed to determine path satisfaction inputs. could not find resource %s: %w",
targetId, err,
))
continue
}

e := construct.ResourceEdge{Source: src, Target: target}
exp := ExpansionInput{
Dep: e,
Classification: satisfaction.Classification,
}
expansions = append(expansions, exp)
}
}
return
}
Loading

0 comments on commit 90e0de6

Please sign in to comment.