Further Integrate Semantic Models

.changes/unreleased/Features-20230616-163045.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Further integrate semantic models into the DAG and partial parsing module
+time: 2023-06-16T16:30:45.513314-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "7800"

core/dbt/compilation.py

@@ -179,6 +179,9 @@ def link_node(self, node: GraphMemberNode, manifest: Manifest):
     def link_graph(self, manifest: Manifest):
         for source in manifest.sources.values():
             self.add_node(source.unique_id)
+        for semantic_node in manifest.semantic_nodes.values():
+            self.add_node(semantic_node.unique_id)
+
         for node in manifest.nodes.values():
             self.link_node(node, manifest)
         for exposure in manifest.exposures.values():

core/dbt/contracts/graph/manifest.py

@@ -762,6 +762,9 @@ def build_flat_graph(self):
             "nodes": {k: v.to_dict(omit_none=False) for k, v in self.nodes.items()},
             "sources": {k: v.to_dict(omit_none=False) for k, v in self.sources.items()},
             "public_nodes": {k: v.to_dict(omit_none=False) for k, v in self.public_nodes.items()},
+            "semantic_nodes": {
+                k: v.to_dict(omit_none=False) for k, v in self.semantic_nodes.items()
+            },
         }
 
     def build_disabled_by_file_id(self):
@@ -822,6 +825,7 @@ def get_resource_fqns(self) -> Mapping[str, PathSet]:
             self.nodes.values(),
             self.sources.values(),
             self.metrics.values(),
+            self.semantic_nodes.values(),
         )
         for resource in all_resources:
             resource_type_plural = resource.resource_type.pluralize()
@@ -857,6 +861,8 @@ def deepcopy(self):
             public_nodes={k: _deepcopy(v) for k, v in self.public_nodes.items()},
             files={k: _deepcopy(v) for k, v in self.files.items()},
             state_check=_deepcopy(self.state_check),
+            publications={k: _deepcopy(v) for k, v in self.publications.items()},
+            semantic_nodes={k: _deepcopy(v) for k, v in self.semantic_nodes.items()},
         )
         copy.build_flat_graph()
         return copy
@@ -869,6 +875,7 @@ def build_parent_and_child_maps(self):
                 self.exposures.values(),
                 self.metrics.values(),
                 self.public_nodes.values(),
+                self.semantic_nodes.values(),
             )
         )
         forward_edges, backward_edges = build_node_edges(edge_members)
@@ -933,6 +940,8 @@ def expect(self, unique_id: str) -> GraphMemberNode:
             return self.exposures[unique_id]
         elif unique_id in self.metrics:
             return self.metrics[unique_id]
+        elif unique_id in self.semantic_nodes:
+            return self.semantic_nodes[unique_id]
         else:
             # something terrible has happened
             raise dbt.exceptions.DbtInternalError(
@@ -1002,7 +1011,9 @@ def pydantic_semantic_manifest(self) -> PydanticSemanticManifest:
         return pydantic_semantic_manifest
 
     def resolve_refs(
-        self, source_node: GraphMemberNode, current_project: str
+        self,
+        source_node: ModelNode,
+        current_project: str,  # TODO: ModelNode is overly restrictive typing
     ) -> List[MaybeNonSource]:
         resolved_refs: List[MaybeNonSource] = []
         for ref in source_node.refs:
@@ -1297,6 +1308,8 @@ def __reduce_ex__(self, protocol):
             self.disabled,
             self.env_vars,
             self.public_nodes,
+            self.publications,
+            self.semantic_nodes,
             self._doc_lookup,
             self._source_lookup,
             self._ref_lookup,

core/dbt/contracts/graph/nodes.py

@@ -1465,12 +1465,14 @@ class NodeRelation(dbtClassMixin):
 class SemanticModel(GraphNode):
     model: str
     node_relation: Optional[NodeRelation]
+    depends_on: DependsOn = field(default_factory=DependsOn)
     description: Optional[str] = None
     defaults: Optional[Defaults] = None
     entities: Sequence[Entity] = field(default_factory=list)
     measures: Sequence[Measure] = field(default_factory=list)
     dimensions: Sequence[Dimension] = field(default_factory=list)
     metadata: Optional[SourceFileMetadata] = None
+    created_at: float = field(default_factory=lambda: time.time())  # REVIEW: Needed?
 
     @property
     def entity_references(self) -> List[LinkableElementReference]:
@@ -1521,6 +1523,18 @@ def partition(self) -> Optional[Dimension]:
     def reference(self) -> SemanticModelReference:
         return SemanticModelReference(semantic_model_name=self.name)
 
+    @property
+    def depends_on_nodes(self):
+        return self.depends_on.nodes
+
+    @property
+    def depends_on_public_nodes(self):
+        return self.depends_on.public_nodes
+
+    @property
+    def depends_on_macros(self):
+        return self.depends_on.macros
+
 
 # ====================================
 # Patches
@@ -1628,6 +1642,7 @@ def alias(self):
     ResultNode,
     Exposure,
     Metric,
+    SemanticModel,
 ]
 
 # All "nodes" (or node-like objects) in this file

core/dbt/parser/manifest.py

@@ -586,6 +586,7 @@ def check_for_model_deprecations(self):
 
                 resolved_refs = self.manifest.resolve_refs(node, self.root_project.project_name)
                 resolved_model_refs = [r for r in resolved_refs if isinstance(r, ModelNode)]
+                node.depends_on
                 for resolved_ref in resolved_model_refs:
                     if resolved_ref.deprecation_date:
 
@@ -1198,6 +1199,7 @@ def process_semantic_models(self) -> None:
                             schema_name=refd_node.schema,
                             database=refd_node.database,
                         )
+                        semantic_model.depends_on.add_node(refd_node.unique_id)
 
     # nodes: node and column descriptions
     # sources: source and table descriptions, column descriptions

core/dbt/parser/partial.py

@@ -609,7 +609,7 @@ def handle_schema_file_changes(self, schema_file, saved_yaml_dict, new_yaml_dict
             env_var_changes = self.env_vars_changed_schema_files[schema_file.file_id]
 
         # models, seeds, snapshots, analyses
-        for dict_key in ["models", "seeds", "snapshots", "analyses"]:
+        for dict_key in ["models", "seeds", "snapshots", "analyses", "semantic_models"]:
             key_diff = self.get_diff_for(dict_key, saved_yaml_dict, new_yaml_dict)
             if key_diff["changed"]:
                 for elem in key_diff["changed"]:

tests/functional/semantic_models/test_semantic_model_parsing.py

@@ -1,7 +1,11 @@
 import pytest
 
+from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
+
 from dbt.cli.main import dbtRunner
 from dbt.contracts.graph.manifest import Manifest
+from dbt.tests.util import write_file
+
 
 schema_yml = """models:
   - name: fct_revenue
@@ -57,3 +61,22 @@ def test_semantic_model_parsing(self, project):
         assert len(manifest.semantic_nodes) == 1
         semantic_model = manifest.semantic_nodes["semanticmodel.test.revenue"]
         assert semantic_model.node_relation.alias == "fct_revenue"
+
+    def test_semantic_model_partial_parsing(self, project):
+        # First, use the default schema.yml to define our semantic model, and
+        # run the dbt parse command
+        runner = dbtRunner()
+        result = runner.invoke(["parse"])
+
+        # Next, modify the default schema.yml to change a detail of the semantic
+        # model.
+        modified_schema_yml = schema_yml.replace("time_granularity: day", "time_granularity: week")
+        write_file(modified_schema_yml, project.project_root, "models", "schema.yml")
+
+        # Now, run the dbt parse command again.
+        result = runner.invoke(["parse"])
+
+        # Finally, verify that the manifest reflects the partially parsed change
+        manifest = result.result
+        semantic_model = manifest.semantic_nodes["semanticmodel.test.revenue"]
+        assert semantic_model.dimensions[0].type_params.time_granularity == TimeGranularity.WEEK

tests/unit/test_manifest.py

@@ -335,6 +335,9 @@ def setUp(self):
                 original_file_path="schema.yml",
             ),
         }
+
+        self.semantic_nodes = {}
+
         for exposure in self.exposures.values():
             exposure.validate(exposure.to_dict(omit_none=True))
         for metric in self.metrics.values():
@@ -475,15 +478,27 @@ def test_build_flat_graph(self):
         flat_metrics = flat_graph["metrics"]
         flat_nodes = flat_graph["nodes"]
         flat_sources = flat_graph["sources"]
+        flat_semantic_nodes = flat_graph["semantic_nodes"]
         self.assertEqual(
             set(flat_graph),
-            set(["exposures", "groups", "nodes", "sources", "metrics", "public_nodes"]),
+            set(
+                [
+                    "exposures",
+                    "groups",
+                    "nodes",
+                    "sources",
+                    "metrics",
+                    "public_nodes",
+                    "semantic_nodes",
+                ]
+            ),
         )
         self.assertEqual(set(flat_exposures), set(self.exposures))
         self.assertEqual(set(flat_groups), set(self.groups))
         self.assertEqual(set(flat_metrics), set(self.metrics))
         self.assertEqual(set(flat_nodes), set(self.nested_nodes))
         self.assertEqual(set(flat_sources), set(self.sources))
+        self.assertEqual(set(flat_semantic_nodes), set(self.semantic_nodes))
         for node in flat_nodes.values():
             self.assertEqual(frozenset(node), REQUIRED_PARSED_NODE_KEYS)
 
@@ -1001,13 +1016,24 @@ def test_build_flat_graph(self):
             selectors={},
             files={},
             exposures={},
+            semantic_nodes={},
         )
         manifest.build_flat_graph()
         flat_graph = manifest.flat_graph
         flat_nodes = flat_graph["nodes"]
         self.assertEqual(
             set(flat_graph),
-            set(["exposures", "groups", "metrics", "nodes", "sources", "public_nodes"]),
+            set(
+                [
+                    "exposures",
+                    "groups",
+                    "metrics",
+                    "nodes",
+                    "sources",
+                    "public_nodes",
+                    "semantic_nodes",
+                ]
+            ),
         )
         self.assertEqual(set(flat_nodes), set(self.nested_nodes))
         compiled_count = 0