Merge branch 'dev/kiyoshi-kuromiya' into unlock-google-api

CHANGELOG.md

@@ -4,13 +4,20 @@
 - Added macro get_partitions_metadata(table) to return partition metadata for partitioned table [#2596](https://github.com/fishtown-analytics/dbt/pull/2596)
 - Added native python 're' module for regex in jinja templates [#2851](https://github.com/fishtown-analytics/dbt/pull/2851)
 - Store resolved node names in manifest ([#2647](https://github.com/fishtown-analytics/dbt/issues/2647), [#2837](https://github.com/fishtown-analytics/dbt/pull/2837))
+- Save selectors dictionary to manifest, allow descriptions ([#2693](https://github.com/fishtown-analytics/dbt/issues/2693), [#2866](https://github.com/fishtown-analytics/dbt/pull/2866))
+- Normalize cli-style-strings in manifest selectors dictionary ([#2879](https://github.com/fishtown-anaytics/dbt/issues/2879), [#2895](https://github.com/fishtown-analytics/dbt/pull/2895))
 
 ### Fixes
 - Respect --project-dir in dbt clean command ([#2840](https://github.com/fishtown-analytics/dbt/issues/2840), [#2841](https://github.com/fishtown-analytics/dbt/pull/2841))
 - Fix Redshift adapter `get_columns_in_relation` macro to push schema filter down to the `svv_external_columns` view ([#2855](https://github.com/fishtown-analytics/dbt/issues/2854))
 - Add `unixodbc-dev` package to testing docker image ([#2859](https://github.com/fishtown-analytics/dbt/pull/2859))
 - Increased the supported relation name length in postgres from 29 to 51 ([#2850](https://github.com/fishtown-analytics/dbt/pull/2850))
-- Widen supported Google Cloud libraries dependencies ([#2877](https://github.com/fishtown-analytics/dbt/pull/2877#issuecomment-731434482)).
+- Widen supported Google Cloud libraries dependencies ([#2794](https://github.com/fishtown-analytics/dbt/pull/2794), [#2877](https://github.com/fishtown-analytics/dbt/pull/2877)).
+- dbt list command always return 0 as exit code ([#2886](https://github.com/fishtown-analytics/dbt/issues/2886), [#2892](https://github.com/fishtown-analytics/dbt/issues/2892))
+
+### Under the hood
+- Bump hologram version to 0.0.11. Add scripts/dtr.py ([#2888](https://github.com/fishtown-analytics/dbt/issues/2840),[#2889](https://github.com/fishtown-analytics/dbt/pull/2889))
+- Add event tracking for project parser/load times ([#2823](https://github.com/fishtown-analytics/dbt/issues/2823),[#2893](https://github.com/fishtown-analytics/dbt/pull/2893))
 
 Contributors:
 - [@feluelle](https://github.com/feluelle) ([#2841](https://github.com/fishtown-analytics/dbt/pull/2841))
@@ -20,6 +27,7 @@ Contributors:
 - [@elexisvenator](https://github.com/elexisvenator) ([#2850](https://github.com/fishtown-analytics/dbt/pull/2850))
 - [@franloza](https://github.com/franloza) ([#2837](https://github.com/fishtown-analytics/dbt/pull/2837))
 - [@max-sixty](https://github.com/max-sixty) ([#2877](https://github.com/fishtown-analytics/dbt/pull/2877))
+- [@rsella](https://github.com/rsella) ([#2892](https://github.com/fishtown-analytics/dbt/issues/2892))
 
 ## dbt 0.19.0b1 (October 21, 2020)
 

core/dbt/config/profile.py

@@ -81,6 +81,8 @@ def read_user_config(directory: str) -> UserConfig:
     return UserConfig()
 
 
+# The Profile class is included in RuntimeConfig, so any attribute
+# additions must also be set where the RuntimeConfig class is created
 @dataclass
 class Profile(HasCredentials):
     profile_name: str

core/dbt/config/project.py

@@ -25,6 +25,7 @@
 from dbt.version import get_installed_version
 from dbt.utils import MultiDict
 from dbt.node_types import NodeType
+from dbt.config.selectors import SelectorDict
 
 from dbt.contracts.project import (
     Project as ProjectContract,
@@ -370,6 +371,12 @@ def create_project(self, rendered: RenderComponents) -> 'Project':
 
         packages = package_config_from_data(rendered.packages_dict)
         selectors = selector_config_from_data(rendered.selectors_dict)
+        manifest_selectors: Dict[str, Any] = {}
+        if rendered.selectors_dict and rendered.selectors_dict['selectors']:
+            # this is a dict with a single key 'selectors' pointing to a list
+            # of dicts.
+            manifest_selectors = SelectorDict.parse_from_selectors_list(
+                rendered.selectors_dict['selectors'])
 
         project = Project(
             project_name=name,
@@ -396,6 +403,7 @@ def create_project(self, rendered: RenderComponents) -> 'Project':
             snapshots=snapshots,
             dbt_version=dbt_version,
             packages=packages,
+            manifest_selectors=manifest_selectors,
             selectors=selectors,
             query_comment=query_comment,
             sources=sources,
@@ -458,6 +466,7 @@ def from_project_root(
 
 class VarProvider:
     """Var providers are tied to a particular Project."""
+
     def __init__(
         self,
         vars: Dict[str, Dict[str, Any]]
@@ -476,6 +485,8 @@ def to_dict(self):
         return self.vars
 
 
+# The Project class is included in RuntimeConfig, so any attribute
+# additions must also be set where the RuntimeConfig class is created
 @dataclass
 class Project:
     project_name: str
@@ -504,6 +515,7 @@ class Project:
     vars: VarProvider
     dbt_version: List[VersionSpecifier]
     packages: Dict[str, Any]
+    manifest_selectors: Dict[str, Any]
     selectors: SelectorConfig
     query_comment: QueryComment
     config_version: int

core/dbt/config/runtime.py

@@ -106,6 +106,7 @@ def from_parts(
             snapshots=project.snapshots,
             dbt_version=project.dbt_version,
             packages=project.packages,
+            manifest_selectors=project.manifest_selectors,
             selectors=project.selectors,
             query_comment=project.query_comment,
             sources=project.sources,
@@ -483,6 +484,7 @@ def from_parts(
             snapshots=project.snapshots,
             dbt_version=project.dbt_version,
             packages=project.packages,
+            manifest_selectors=project.manifest_selectors,
             selectors=project.selectors,
             query_comment=project.query_comment,
             sources=project.sources,

core/dbt/config/selectors.py

@@ -15,6 +15,7 @@
 from dbt.contracts.selection import SelectorFile
 from dbt.exceptions import DbtSelectorsError, RuntimeException
 from dbt.graph import parse_from_selectors_definition, SelectionSpec
+from dbt.graph.selector_spec import SelectionCriteria
 
 MALFORMED_SELECTOR_ERROR = """\
 The selectors.yml file in this project is malformed. Please double check
@@ -113,3 +114,67 @@ def selector_config_from_data(
             result_type='invalid_selector',
         ) from e
     return selectors
+
+
+# These are utilities to clean up the dictionary created from
+# selectors.yml by turning the cli-string format entries into
+# normalized dictionary entries. It parallels the flow in
+# dbt/graph/cli.py. If changes are made there, it might
+# be necessary to make changes here. Ideally it would be
+# good to combine the two flows into one at some point.
+class SelectorDict:
+
+    @classmethod
+    def parse_dict_definition(cls, definition):
+        key = list(definition)[0]
+        value = definition[key]
+        if isinstance(value, list):
+            new_values = []
+            for sel_def in value:
+                new_value = cls.parse_from_definition(sel_def)
+                new_values.append(new_value)
+            value = new_values
+        if key == 'exclude':
+            definition = {key: value}
+        elif len(definition) == 1:
+            definition = {'method': key, 'value': value}
+        return definition
+
+    @classmethod
+    def parse_a_definition(cls, def_type, definition):
+        # this definition must be a list
+        new_dict = {def_type: []}
+        for sel_def in definition[def_type]:
+            if isinstance(sel_def, dict):
+                sel_def = cls.parse_from_definition(sel_def)
+                new_dict[def_type].append(sel_def)
+            elif isinstance(sel_def, str):
+                sel_def = SelectionCriteria.dict_from_single_spec(sel_def)
+                new_dict[def_type].append(sel_def)
+            else:
+                new_dict[def_type].append(sel_def)
+        return new_dict
+
+    @classmethod
+    def parse_from_definition(cls, definition):
+        if isinstance(definition, str):
+            definition = SelectionCriteria.dict_from_single_spec(definition)
+        elif 'union' in definition:
+            definition = cls.parse_a_definition('union', definition)
+        elif 'intersection' in definition:
+            definition = cls.parse_a_definition('intersection', definition)
+        elif isinstance(definition, dict):
+            definition = cls.parse_dict_definition(definition)
+        return definition
+
+    # This is the normal entrypoint of this code. Give it the
+    # list of selectors generated from the selectors.yml file.
+    @classmethod
+    def parse_from_selectors_list(cls, selectors):
+        selector_dict = {}
+        for selector in selectors:
+            sel_name = selector['name']
+            selector_dict[sel_name] = selector
+            definition = cls.parse_from_definition(selector['definition'])
+            selector_dict[sel_name]['definition'] = definition
+        return selector_dict

core/dbt/contracts/graph/manifest.py

@@ -431,11 +431,15 @@ def _update_into(dest: MutableMapping[str, T], new_item: T):
 class Manifest:
     """The manifest for the full graph, after parsing and during compilation.
     """
+    # These attributes are both positional and by keyword. If an attribute
+    # is added it must all be added in the __reduce_ex__ method in the
+    # args tuple in the right position.
     nodes: MutableMapping[str, ManifestNode]
     sources: MutableMapping[str, ParsedSourceDefinition]
     macros: MutableMapping[str, ParsedMacro]
     docs: MutableMapping[str, ParsedDocumentation]
     exposures: MutableMapping[str, ParsedExposure]
+    selectors: MutableMapping[str, Any]
     disabled: List[CompileResultNode]
     files: MutableMapping[str, SourceFile]
     metadata: ManifestMetadata = field(default_factory=ManifestMetadata)
@@ -461,6 +465,7 @@ def from_macros(
             macros=macros,
             docs={},
             exposures={},
+            selectors={},
             disabled=[],
             files=files,
         )
@@ -730,8 +735,9 @@ def deepcopy(self):
             macros={k: _deepcopy(v) for k, v in self.macros.items()},
             docs={k: _deepcopy(v) for k, v in self.docs.items()},
             exposures={k: _deepcopy(v) for k, v in self.exposures.items()},
-            disabled=[_deepcopy(n) for n in self.disabled],
+            selectors=self.root_project.manifest_selectors,
             metadata=self.metadata,
+            disabled=[_deepcopy(n) for n in self.disabled],
             files={k: _deepcopy(v) for k, v in self.files.items()},
         )
 
@@ -749,6 +755,7 @@ def writable_manifest(self):
             macros=self.macros,
             docs=self.docs,
             exposures=self.exposures,
+            selectors=self.selectors,
             metadata=self.metadata,
             disabled=self.disabled,
             child_map=forward_edges,
@@ -905,14 +912,21 @@ def merge_from_artifact(
             f'Merged {len(merged)} items from state (sample: {sample})'
         )
 
-    # provide support for copy.deepcopy() - we jsut need to avoid the lock!
+    # Provide support for copy.deepcopy() - we just need to avoid the lock!
+    # pickle and deepcopy use this. It returns a callable object used to
+    # create the initial version of the object and a tuple of arguments
+    # for the object, i.e. the Manifest.
+    # The order of the arguments must match the order of the attributes
+    # in the Manifest class declaration, because they are used as
+    # positional arguments to construct a Manifest.
     def __reduce_ex__(self, protocol):
         args = (
             self.nodes,
             self.sources,
             self.macros,
             self.docs,
             self.exposures,
+            self.selectors,
             self.disabled,
             self.files,
             self.metadata,
@@ -952,6 +966,11 @@ class WritableManifest(ArtifactMixin):
             'The exposures defined in the dbt project and its dependencies'
         ))
     )
+    selectors: Mapping[UniqueID, Any] = field(
+        metadata=dict(description=(
+            'The selectors defined in selectors.yml'
+        ))
+    )
     disabled: Optional[List[CompileResultNode]] = field(metadata=dict(
         description='A list of the disabled nodes in the target'
     ))

core/dbt/contracts/selection.py

@@ -8,6 +8,7 @@
 class SelectorDefinition(JsonSchemaMixin):
     name: str
     definition: Union[str, Dict[str, Any]]
+    description: str = ''
 
 
 @dataclass

core/dbt/graph/selector_spec.py

@@ -123,6 +123,26 @@ def from_dict(cls, raw: Any, dct: Dict[str, Any]) -> 'SelectionCriteria':
             children_depth=children_depth,
         )
 
+    @classmethod
+    def dict_from_single_spec(cls, raw: str):
+        result = RAW_SELECTOR_PATTERN.match(raw)
+        if result is None:
+            return {'error': 'Invalid selector spec'}
+        dct: Dict[str, Any] = result.groupdict()
+        method_name, method_arguments = cls.parse_method(dct)
+        meth_name = str(method_name)
+        if method_arguments:
+            meth_name = meth_name + '.' + '.'.join(method_arguments)
+        dct['method'] = meth_name
+        dct = {k: v for k, v in dct.items() if (v is not None and v != '')}
+        if 'childrens_parents' in dct:
+            dct['childrens_parents'] = bool(dct.get('childrens_parents'))
+        if 'parents' in dct:
+            dct['parents'] = bool(dct.get('parents'))
+        if 'children' in dct:
+            dct['children'] = bool(dct.get('children'))
+        return dct
+
     @classmethod
     def from_single_spec(cls, raw: str) -> 'SelectionCriteria':
         result = RAW_SELECTOR_PATTERN.match(raw)

core/dbt/main.py

@@ -23,6 +23,7 @@
 import dbt.task.serve as serve_task
 import dbt.task.freshness as freshness_task
 import dbt.task.run_operation as run_operation_task
+import dbt.task.parse as parse_task
 from dbt.profiler import profiler
 from dbt.task.list import ListTask
 from dbt.task.rpc.server import RPCServerTask
@@ -494,6 +495,21 @@ def _build_compile_subparser(subparsers, base_subparser):
     return sub
 
 
+def _build_parse_subparser(subparsers, base_subparser):
+    sub = subparsers.add_parser(
+        'parse',
+        parents=[base_subparser],
+        help='''
+        Parsed the project and provides information on performance
+        '''
+    )
+    sub.set_defaults(cls=parse_task.ParseTask, which='parse',
+                     rpc_method='parse')
+    sub.add_argument('--write-manifest', action='store_true')
+    sub.add_argument('--compile', action='store_true')
+    return sub
+
+
 def _build_docs_generate_subparser(subparsers, base_subparser):
     # it might look like docs_sub is the correct parents entry, but that
     # will cause weird errors about 'conflicting option strings'.
@@ -1006,12 +1022,13 @@ def parse_args(args, cls=DBTArgumentParser):
     rpc_sub = _build_rpc_subparser(subs, base_subparser)
     run_sub = _build_run_subparser(subs, base_subparser)
     compile_sub = _build_compile_subparser(subs, base_subparser)
+    parse_sub = _build_parse_subparser(subs, base_subparser)
     generate_sub = _build_docs_generate_subparser(docs_subs, base_subparser)
     test_sub = _build_test_subparser(subs, base_subparser)
     seed_sub = _build_seed_subparser(subs, base_subparser)
     # --threads, --no-version-check
     _add_common_arguments(run_sub, compile_sub, generate_sub, test_sub,
-                          rpc_sub, seed_sub)
+                          rpc_sub, seed_sub, parse_sub)
     # --models, --exclude
     # list_sub sets up its own arguments.
     _add_selection_arguments(run_sub, compile_sub, generate_sub, test_sub)