Fix Local layer to only look at operational resource paths

pkg/engine2/cli.go

@@ -412,14 +412,14 @@ func writeDebugGraphs(sol solution_context.SolutionContext) {
 	wg.Add(2)
 	go func() {
 		defer wg.Done()
-		err := construct.GraphToSVG(sol.DataflowGraph(), "dataflow")
+		err := GraphToSVG(sol.KnowledgeBase(), sol.DataflowGraph(), "dataflow")
 		if err != nil {
 			zap.S().Errorf("failed to write dataflow graph: %s", err.Error())
 		}
 	}()
 	go func() {
 		defer wg.Done()
-		err := construct.GraphToSVG(sol.DeploymentGraph(), "iac")
+		err := GraphToSVG(sol.KnowledgeBase(), sol.DeploymentGraph(), "iac")
 		if err != nil {
 			zap.S().Errorf("failed to write iac graph: %s", err.Error())
 		}

pkg/engine2/dot.go

@@ -0,0 +1,129 @@
+package engine2
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+
+	construct "github.com/klothoplatform/klotho/pkg/construct2"
+	"github.com/klothoplatform/klotho/pkg/dot"
+	knowledgebase2 "github.com/klothoplatform/klotho/pkg/knowledge_base2"
+)
+
+func dotAttributes(kb knowledgebase2.TemplateKB, r *construct.Resource) map[string]string {
+	a := make(map[string]string)
+	a["label"] = r.ID.String()
+	a["shape"] = "box"
+	tmpl, _ := kb.GetResourceTemplate(r.ID)
+	if tmpl != nil {
+		a["label"] += fmt.Sprintf("\n%v", tmpl.Classification.Is)
+	}
+	return a
+}
+
+func dotEdgeAttributes(e construct.ResourceEdge) map[string]string {
+	a := make(map[string]string)
+	_ = e.Source.WalkProperties(func(path construct.PropertyPath, nerr error) error {
+		v := path.Get()
+		if v == e.Target.ID {
+			a["label"] = path.String()
+			return construct.StopWalk
+		}
+		return nil
+	})
+	if e.Properties.Weight > 0 {
+		if a["label"] == "" {
+			a["label"] = fmt.Sprintf("%d", e.Properties.Weight)
+		} else {
+			a["label"] = fmt.Sprintf("%s\n%d", a["label"], e.Properties.Weight)
+		}
+	}
+	return a
+}
+
+func GraphToDOT(kb knowledgebase2.TemplateKB, g construct.Graph, out io.Writer) error {
+	ids, err := construct.TopologicalSort(g)
+	if err != nil {
+		return err
+	}
+	nodes, err := construct.ResolveIds(g, ids)
+	if err != nil {
+		return err
+	}
+	var errs error
+	printf := func(s string, args ...any) {
+		_, err := fmt.Fprintf(out, s, args...)
+		errs = errors.Join(errs, err)
+	}
+	printf(`digraph {
+  rankdir = TB
+`)
+	for _, n := range nodes {
+		printf("  %q%s\n", n.ID, dot.AttributesToString(dotAttributes(kb, n)))
+	}
+
+	topoIndex := func(id construct.ResourceId) int {
+		for i, id2 := range ids {
+			if id2 == id {
+				return i
+			}
+		}
+		return -1
+	}
+	edges, err := g.Edges()
+	if err != nil {
+		return err
+	}
+	sort.Slice(edges, func(i, j int) bool {
+		ti, tj := topoIndex(edges[i].Source), topoIndex(edges[j].Source)
+		if ti != tj {
+			return ti < tj
+		}
+		ti, tj = topoIndex(edges[i].Target), topoIndex(edges[j].Target)
+		return ti < tj
+	})
+	for _, e := range edges {
+		edge, err := g.Edge(e.Source, e.Target)
+		if err != nil {
+			errs = errors.Join(errs, err)
+			continue
+		}
+		printf("  %q -> %q%s\n", e.Source, e.Target, dot.AttributesToString(dotEdgeAttributes(edge)))
+	}
+	printf("}\n")
+	return errs
+}
+
+func GraphToSVG(kb knowledgebase2.TemplateKB, g construct.Graph, prefix string) error {
+	if debugDir := os.Getenv("KLOTHO_DEBUG_DIR"); debugDir != "" {
+		prefix = filepath.Join(debugDir, prefix)
+	}
+	f, err := os.Create(prefix + ".gv")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	dotContent := new(bytes.Buffer)
+	err = GraphToDOT(kb, g, io.MultiWriter(f, dotContent))
+	if err != nil {
+		return fmt.Errorf("could not render graph to file %s: %v", prefix+".gv", err)
+	}
+
+	svgContent, err := dot.ExecPan(bytes.NewReader(dotContent.Bytes()))
+	if err != nil {
+		return fmt.Errorf("could not run 'dot' for %s: %v", prefix+".gv", err)
+	}
+
+	svgFile, err := os.Create(prefix + ".gv.svg")
+	if err != nil {
+		return fmt.Errorf("could not create file %s: %v", prefix+".gv.svg", err)
+	}
+	defer svgFile.Close()
+	_, err = fmt.Fprint(svgFile, svgContent)
+	return err
+}

pkg/engine2/path_selection/paths.go

@@ -13,7 +13,7 @@ import (
 func GetPaths(
 	sol solution_context.SolutionContext,
 	source, target construct.ResourceId,
-	pathValidityChecks func(source, target construct.ResourceId, path []construct.ResourceId) bool,
+	pathValidityChecks func(source, target construct.ResourceId, path construct.Path) bool,
 	hasPathCheck bool,
 ) ([]construct.Path, error) {
 	var errs error

pkg/engine2/reconciler/remove_edge.go

@@ -155,7 +155,7 @@ func findEdgesUsedInOtherPathSelection(
 			continue
 		}
 		paths, err := path_selection.GetPaths(ctx, upstream, target,
-			func(source, target construct.ResourceId, path []construct.ResourceId) bool { return true }, false)
+			func(source, target construct.ResourceId, path construct.Path) bool { return true }, false)
 		if err != nil {
 			errs = errors.Join(errs, err)
 			continue
@@ -190,7 +190,7 @@ func findEdgesUsedInOtherPathSelection(
 		}
 
 		paths, err := path_selection.GetPaths(ctx, source, downstream,
-			func(source, target construct.ResourceId, path []construct.ResourceId) bool { return true }, false)
+			func(source, target construct.ResourceId, path construct.Path) bool { return true }, false)
 		if err != nil {
 			errs = errors.Join(errs, err)
 			continue

pkg/engine2/testdata/2_routes.dataflow-viz.yaml

@@ -3,56 +3,44 @@ resources:
   lambda_function/lambda_function_1:
     children:
         - aws:ecr_image:lambda_function_1-image
+        - aws:ecr_repo:ecr_repo-0
         - aws:iam_role:lambda_function_1-ExecutionRole
-    parent: rest_api/rest_api_1
     tag: big
 
   lambda_function/lambda_function_0:
     children:
         - aws:ecr_image:lambda_function_0-image
         - aws:ecr_repo:ecr_repo-0
         - aws:iam_role:lambda_function_0-ExecutionRole
-        - aws:log_group:lambda_function_0-log-group
-        - aws:log_group:lambda_function_1-log-group
-    parent: rest_api/rest_api_1
     tag: big
 
   rest_api/rest_api_1:
-    children:
-        - aws:api_resource:rest_api_1:lambda0
-        - aws:api_resource:rest_api_1:lambda1
-        - aws:api_stage:rest_api_1:api_stage-0
-    tag: parent
-
-  aws:api_integration:rest_api_1/integ1:
     children:
         - aws:api_deployment:rest_api_1:api_deployment-0
+        - aws:api_integration:rest_api_1:integ0
+        - aws:api_integration:rest_api_1:integ1
         - aws:api_method:rest_api_1:integ0-api_method
         - aws:api_method:rest_api_1:integ1-api_method
         - aws:api_resource:rest_api_1:api_resource-0
         - aws:api_resource:rest_api_1:api_resource-1
         - aws:api_resource:rest_api_1:lambda0
         - aws:api_resource:rest_api_1:lambda1
         - aws:api_stage:rest_api_1:api_stage-0
-        - aws:lambda_permission:integ0-lambda_function_0
-        - aws:lambda_permission:integ1-lambda_function_1
+    tag: parent
+
+  aws:api_integration:rest_api_1/integ1:
     parent: rest_api/rest_api_1
     tag: big
 
   aws:api_integration:rest_api_1/integ1 -> lambda_function/lambda_function_1:
-  aws:api_integration:rest_api_1/integ0:
-    children:
-        - aws:api_deployment:rest_api_1:api_deployment-0
-        - aws:api_method:rest_api_1:integ0-api_method
-        - aws:api_method:rest_api_1:integ1-api_method
-        - aws:api_resource:rest_api_1:api_resource-0
-        - aws:api_resource:rest_api_1:api_resource-1
-        - aws:api_resource:rest_api_1:lambda0
-        - aws:api_resource:rest_api_1:lambda1
-        - aws:api_stage:rest_api_1:api_stage-0
-        - aws:lambda_permission:integ0-lambda_function_0
+    path:
         - aws:lambda_permission:integ1-lambda_function_1
+
+  aws:api_integration:rest_api_1/integ0:
     parent: rest_api/rest_api_1
     tag: big
 
   aws:api_integration:rest_api_1/integ0 -> lambda_function/lambda_function_0:
+    path:
+        - aws:lambda_permission:integ0-lambda_function_0
+

pkg/engine2/testdata/delete_namespace_resource.dataflow-viz.yaml

@@ -5,6 +5,5 @@ resources:
         - aws:ecr_image:lambda_function_2-image
         - aws:ecr_repo:ecr_repo-0
         - aws:iam_role:lambda_function_2-ExecutionRole
-        - aws:log_group:lambda_function_2-log-group
     tag: big
 

pkg/engine2/testdata/delete_resource_and_iacdeps.dataflow-viz.yaml

@@ -1,16 +1,15 @@
 provider: aws
 resources:
   rest_api/rest_api_0:
-    children:
-        - aws:api_stage:rest_api_0:api_stage-0
-    tag: parent
-
-  aws:api_integration:rest_api_0/rest_api_0_integration_0:
     children:
         - aws:api_deployment:rest_api_0:api_deployment-0
+        - aws:api_integration:rest_api_0:rest_api_0_integration_0
         - aws:api_method:rest_api_0:rest_api_0_integration_0_method
         - aws:api_resource:rest_api_0:api_resource-0
         - aws:api_stage:rest_api_0:api_stage-0
+    tag: parent
+
+  aws:api_integration:rest_api_0/rest_api_0_integration_0:
     parent: rest_api/rest_api_0
     tag: big
 

pkg/engine2/testdata/ecs_rds.dataflow-viz.yaml

@@ -2,41 +2,38 @@ provider: aws
 resources:
   rds_instance/rds-instance-2:
     children:
-        - aws:iam_role:ecs_service_0-execution-role
         - aws:rds_subnet_group:rds_subnet_group-0
-        - aws:security_group:vpc-0:rds-instance-2-security_group
     parent: vpc/vpc-0
     tag: big
 
   vpc/vpc-0:
     children:
-        - aws:ecr_image:ecs_service_0-image
-        - aws:ecr_repo:ecr_repo-0
-        - aws:ecs_task_definition:ecs_service_0
-        - aws:elastic_ip:subnet-0-route_table-nat_gateway-elastic_ip
-        - aws:elastic_ip:subnet-1-route_table-nat_gateway-elastic_ip
-        - aws:iam_role:ecs_service_0-execution-role
         - aws:internet_gateway:vpc-0:internet_gateway-0
-        - aws:log_group:ecs_service_0-log-group
-        - aws:nat_gateway:subnet-2:subnet-0-route_table-nat_gateway
-        - aws:nat_gateway:subnet-3:subnet-1-route_table-nat_gateway
-        - aws:rds_subnet_group:rds_subnet_group-0
         - aws:route_table:vpc-0:subnet-0-route_table
         - aws:route_table:vpc-0:subnet-1-route_table
         - aws:route_table:vpc-0:subnet-2-route_table
         - aws:route_table:vpc-0:subnet-3-route_table
         - aws:security_group:vpc-0:ecs_service_0-security_group
         - aws:security_group:vpc-0:rds-instance-2-security_group
+        - aws:subnet:vpc-0:subnet-0
+        - aws:subnet:vpc-0:subnet-1
+        - aws:subnet:vpc-0:subnet-2
+        - aws:subnet:vpc-0:subnet-3
     tag: parent
 
   ecs_service/ecs_service_0:
     children:
         - aws:ecr_image:ecs_service_0-image
         - aws:ecr_repo:ecr_repo-0
         - aws:ecs_task_definition:ecs_service_0
+        - aws:iam_role:ecs_service_0-execution-role
         - aws:log_group:ecs_service_0-log-group
-        - aws:security_group:vpc-0:ecs_service_0-security_group
     parent: vpc/vpc-0
     tag: big
 
   ecs_service/ecs_service_0 -> rds_instance/rds-instance-2:
+    path:
+        - aws:ecs_task_definition:ecs_service_0
+        - aws:iam_role:ecs_service_0-execution-role
+        - aws:security_group:vpc-0:rds-instance-2-security_group
+

pkg/engine2/testdata/extend_graph.dataflow-viz.yaml

@@ -5,26 +5,25 @@ resources:
         - aws:ecr_image:lambda_function_0-image
         - aws:ecr_repo:ecr_repo-0
         - aws:iam_role:lambda_function_0-ExecutionRole
-        - aws:log_group:lambda_function_0-log-group
-    parent: rest_api/rest_api_1
     tag: big
 
   rest_api/rest_api_1:
     children:
+        - aws:api_deployment:rest_api_1:api_deployment-0
+        - aws:api_integration:rest_api_1:rest_api_1_integration_0
+        - aws:api_method:rest_api_1:rest_api_1_integration_0_method
+        - aws:api_resource:rest_api_1:api_resource-0
         - aws:api_stage:rest_api_1:api_stage-0
     tag: parent
 
   dynamodb_table/dynamodb_table_3:
     tag: big
 
   aws:api_integration:rest_api_1/rest_api_1_integration_0:
-    children:
-        - aws:api_deployment:rest_api_1:api_deployment-0
-        - aws:api_method:rest_api_1:rest_api_1_integration_0_method
-        - aws:api_resource:rest_api_1:api_resource-0
-        - aws:api_stage:rest_api_1:api_stage-0
-        - aws:lambda_permission:rest_api_1_integration_0_lambda_function_0
     parent: rest_api/rest_api_1
     tag: big
 
   aws:api_integration:rest_api_1/rest_api_1_integration_0 -> lambda_function/lambda_function_0:
+    path:
+        - aws:lambda_permission:rest_api_1_integration_0_lambda_function_0
+