Merge branch 'dev/0.18.1' into dev/kiyoshi-kuromiya

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2.1
 jobs:
   unit:
     docker: &test_only
-      - image: fishtownanalytics/test-container:7
+      - image: fishtownanalytics/test-container:9
         environment:
           DBT_INVOCATION_ENV: circle
     steps:
@@ -30,7 +30,7 @@ jobs:
           destination: dist
   integration-postgres-py36:
     docker: &test_and_postgres
-      - image: fishtownanalytics/test-container:7
+      - image: fishtownanalytics/test-container:9
         environment:
           DBT_INVOCATION_ENV: circle
       - image: postgres

.gitignore

@@ -43,6 +43,7 @@ htmlcov/
 .coverage
 .coverage.*
 .cache
+.env
 nosetests.xml
 coverage.xml
 *,cover

CHANGELOG.md

@@ -4,12 +4,30 @@
 - dbt will compare configurations using the un-rendered form of the config block in dbt_project.yml ([#2713](https://github.com/fishtown-analytics/dbt/issues/2713), [#2735](https://github.com/fishtown-analytics/dbt/pull/2735))
 - Added state and defer arguments to the RPC client, matching the CLI ([#2678](https://github.com/fishtown-analytics/dbt/issues/2678), [#2736](https://github.com/fishtown-analytics/dbt/pull/2736))
 
+## dbt 0.18.1 (Release TBD)
+
+### Under the hood
+- If column config says quote, use quoting in SQL for adding a comment. ([#2539](https://github.com/fishtown-analytics/dbt/issues/2539), [#2733](https://github.com/fishtown-analytics/dbt/pull/2733))
+- Added support for running docker-based tests under Linux. ([#2739](https://github.com/fishtown-analytics/dbt/issues/2739))
+
+### Features
+- Specify all three logging levels (`INFO`, `WARNING`, `ERROR`) in result logs for commands `test`, `seed`, `run`, `snapshot` and `source snapshot-freshness` ([#2680](https://github.com/fishtown-analytics/dbt/pull/2680), [#2723](https://github.com/fishtown-analytics/dbt/pull/2723))
+- Added "reports" ([#2730](https://github.com/fishtown-analytics/dbt/issues/2730), [#2752](https://github.com/fishtown-analytics/dbt/pull/2752))
+
+### Docs
+- Add Report nodes ([docs#135](https://github.com/fishtown-analytics/dbt-docs/issues/135), [docs#136](https://github.com/fishtown-analytics/dbt-docs/pull/136))
+
+Contributors:
+- [@tpilewicz](https://github.com/tpilewicz) ([#2723](https://github.com/fishtown-analytics/dbt/pull/2723))
+- [@heisencoder](https://github.com/heisencoder) ([#2739](https://github.com/fishtown-analytics/dbt/issues/2739))
+
+
 ## dbt 0.18.0 (September 03, 2020)
 
 ### Under the hood
 - Added 3 more adapter methods that the new dbt-adapter-test suite can use for testing. ([#2492](https://github.com/fishtown-analytics/dbt/issues/2492), [#2721](https://github.com/fishtown-analytics/dbt/pull/2721))
 - It is now an error to attempt installing `dbt` with a Python version less than 3.6. (resolves [#2347](https://github.com/fishtown-analytics/dbt/issues/2347))
-- Check for Postgres relation names longer than 63 and throw exception. ([#2197](https://github.com/fishtown-analytics/dbt/issues/2197))
+- Check for Postgres relation names longer than 63 and throw exception. ([#2197](https://github.com/fishtown-analytics/dbt/issues/2197), [#2727](https://github.com/fishtown-analytics/dbt/pull/2727))
 
 
 ### Fixes
@@ -49,7 +67,6 @@ Contributors:
 - Add state:modified and state:new selectors ([#2641](https://github.com/fishtown-analytics/dbt/issues/2641), [#2695](https://github.com/fishtown-analytics/dbt/pull/2695))
 - Add two new flags `--use-colors` and `--no-use-colors` to `dbt run` command to enable or disable log colorization from the command line ([#2708](https://github.com/fishtown-analytics/dbt/pull/2708))
 
-
 ### Fixes
 - Fix Redshift table size estimation; e.g. 44 GB tables are no longer reported as 44 KB. [#2702](https://github.com/fishtown-analytics/dbt/issues/2702)
 - Fix issue where jinja that only contained jinja comments wasn't rendered. ([#2707](https://github.com/fishtown-analytics/dbt/issues/2707), [#2178](https://github.com/fishtown-analytics/dbt/pull/2178))

Dockerfile

@@ -1,7 +1,6 @@
 FROM ubuntu:18.04
 
 ENV DEBIAN_FRONTEND noninteractive
-ARG DOCKERIZE_VERSION=v0.6.1
 
 RUN apt-get update && \
     apt-get dist-upgrade -y && \
@@ -19,20 +18,33 @@ RUN apt-get update && \
         python3.9 python3.9-dev python3.9-venv && \
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-RUN useradd -mU dbt_test_user
-RUN mkdir /usr/app && chown dbt_test_user /usr/app
-RUN mkdir /home/tox && chown dbt_test_user /home/tox
+ARG DOCKERIZE_VERSION=v0.6.1
 RUN curl -LO https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \
     tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \
     rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
 
-WORKDIR /usr/app
-VOLUME /usr/app
-
 RUN pip3 install -U "tox==3.14.4" wheel "six>=1.14.0,<1.15.0" "virtualenv==20.0.3" setuptools
 # tox fails if the 'python' interpreter (python2) doesn't have `tox` installed
 RUN pip install -U "tox==3.14.4" "six>=1.14.0,<1.15.0" "virtualenv==20.0.3" setuptools
 
+# These args are passed in via docker-compose, which reads then from the .env file.
+# On Linux, run `make .env` to create the .env file for the current user.
+# On MacOS and Windows, these can stay unset.
+ARG USER_ID
+ARG GROUP_ID
+
+RUN if [ ${USER_ID:-0} -ne 0 ] && [ ${GROUP_ID:-0} -ne 0 ]; then \
+        groupadd -g ${GROUP_ID} dbt_test_user && \
+        useradd -m -l -u ${USER_ID} -g ${GROUP_ID} dbt_test_user; \
+    else \
+        useradd -mU -l dbt_test_user; \
+    fi
+RUN mkdir /usr/app && chown dbt_test_user /usr/app
+RUN mkdir /home/tox && chown dbt_test_user /home/tox
+
+WORKDIR /usr/app
+VOLUME /usr/app
+
 USER dbt_test_user
 
 ENV PYTHONIOENCODING=utf-8

Makefile

@@ -5,25 +5,33 @@ changed_tests := `git status --porcelain | grep '^\(M\| M\|A\| A\)' | awk '{ pri
 install:
 	pip install -e .
 
-test:
+test: .env
 	@echo "Full test run starting..."
 	@time docker-compose run test tox
 
-test-unit:
+test-unit: .env
 	@echo "Unit test run starting..."
 	@time docker-compose run test tox -e unit-py36,flake8
 
-test-integration:
+test-integration: .env
 	@echo "Integration test run starting..."
 	@time docker-compose run test tox -e integration-postgres-py36,integration-redshift-py36,integration-snowflake-py36,integration-bigquery-py36
 
-test-quick:
+test-quick: .env
 	@echo "Integration test run starting..."
 	@time docker-compose run test tox -e integration-postgres-py36 -- -x
 
+# This rule creates a file named .env that is used by docker-compose for passing
+# the USER_ID and GROUP_ID arguments to the Docker image.
+.env:
+	@echo USER_ID=$(shell id -u) > .env
+	@echo GROUP_ID=$(shell id -g) >> .env
+	@time docker-compose build
+
 clean:
 	rm -f .coverage
 	rm -rf .eggs/
+	rm -f .env
 	rm -rf .tox/
 	rm -rf build/
 	rm -rf dbt.egg-info/

core/dbt/adapters/protocol.py

@@ -9,7 +9,7 @@
 
 from dbt.contracts.connection import Connection, AdapterRequiredConfig
 from dbt.contracts.graph.compiled import (
-    CompiledNode, NonSourceNode, NonSourceCompiledNode
+    CompiledNode, ManifestNode, NonSourceCompiledNode
 )
 from dbt.contracts.graph.parsed import ParsedNode, ParsedSourceDefinition
 from dbt.contracts.graph.model_config import BaseConfig
@@ -55,7 +55,7 @@ def compile(self, manifest: Manifest, write=True) -> Graph:
 
     def compile_node(
         self,
-        node: NonSourceNode,
+        node: ManifestNode,
         manifest: Manifest,
         extra_context: Optional[Dict[str, Any]] = None,
     ) -> NonSourceCompiledNode:

core/dbt/compilation.py

@@ -12,12 +12,13 @@
 from dbt.context.providers import generate_runtime_model
 from dbt.contracts.graph.manifest import Manifest
 from dbt.contracts.graph.compiled import (
-    InjectedCTE,
-    COMPILED_TYPES,
-    NonSourceNode,
-    NonSourceCompiledNode,
     CompiledDataTestNode,
     CompiledSchemaTestNode,
+    COMPILED_TYPES,
+    GraphMemberNode,
+    InjectedCTE,
+    ManifestNode,
+    NonSourceCompiledNode,
 )
 from dbt.contracts.graph.parsed import ParsedNode
 from dbt.exceptions import (
@@ -64,7 +65,7 @@ def print_compile_stats(stats):
     logger.info("Found {}".format(stat_line))
 
 
-def _node_enabled(node: NonSourceNode):
+def _node_enabled(node: ManifestNode):
     # Disabled models are already excluded from the manifest
     if node.resource_type == NodeType.Test and not node.config.enabled:
         return False
@@ -358,7 +359,7 @@ def _insert_ctes(
 
     def _compile_node(
         self,
-        node: NonSourceNode,
+        node: ManifestNode,
         manifest: Manifest,
         extra_context: Optional[Dict[str, Any]] = None,
     ) -> NonSourceCompiledNode:
@@ -402,7 +403,7 @@ def write_graph_file(self, linker: Linker, manifest: Manifest):
             linker.write_graph(graph_path, manifest)
 
     def link_node(
-        self, linker: Linker, node: NonSourceNode, manifest: Manifest
+        self, linker: Linker, node: GraphMemberNode, manifest: Manifest
     ):
         linker.add_node(node.unique_id)
 
@@ -425,6 +426,9 @@ def link_graph(self, linker: Linker, manifest: Manifest):
             linker.add_node(source.unique_id)
         for node in manifest.nodes.values():
             self.link_node(linker, node, manifest)
+        for report in manifest.reports.values():
+            self.link_node(linker, report, manifest)
+            # linker.add_node(report.unique_id)
 
         cycle = linker.find_cycles()
 
@@ -445,7 +449,7 @@ def compile(self, manifest: Manifest, write=True) -> Graph:
 
         return Graph(linker.graph)
 
-    def _write_node(self, node: NonSourceCompiledNode) -> NonSourceNode:
+    def _write_node(self, node: NonSourceCompiledNode) -> ManifestNode:
         if not _is_writable(node):
             return node
         logger.debug(f'Writing injected SQL for node "{node.unique_id}"')
@@ -467,7 +471,7 @@ def _write_node(self, node: NonSourceCompiledNode) -> NonSourceNode:
 
     def compile_node(
         self,
-        node: NonSourceNode,
+        node: ManifestNode,
         manifest: Manifest,
         extra_context: Optional[Dict[str, Any]] = None,
         write: bool = True,

core/dbt/context/providers.py

@@ -21,10 +21,11 @@
 from dbt.contracts.graph.compiled import (
     CompiledResource,
     CompiledSeedNode,
-    NonSourceNode,
+    ManifestNode,
 )
 from dbt.contracts.graph.parsed import (
     ParsedMacro,
+    ParsedReport,
     ParsedSeedNode,
     ParsedSourceDefinition,
 )
@@ -419,7 +420,7 @@ def resolve(
         return self.Relation.create_from(self.config, self.model)
 
 
-ResolveRef = Union[Disabled, NonSourceNode]
+ResolveRef = Union[Disabled, ManifestNode]
 
 
 class RuntimeRefResolver(BaseRefResolver):
@@ -444,7 +445,7 @@ def resolve(
         return self.create_relation(target_model, target_name)
 
     def create_relation(
-        self, target_model: NonSourceNode, name: str
+        self, target_model: ManifestNode, name: str
     ) -> RelationProxy:
         if target_model.is_ephemeral_model:
             self.model.set_cte(target_model.unique_id, None)
@@ -456,7 +457,7 @@ def create_relation(
 
     def validate(
         self,
-        resolved: NonSourceNode,
+        resolved: ManifestNode,
         target_name: str,
         target_package: Optional[str]
     ) -> None:
@@ -468,14 +469,14 @@ def validate(
 class OperationRefResolver(RuntimeRefResolver):
     def validate(
         self,
-        resolved: NonSourceNode,
+        resolved: ManifestNode,
         target_name: str,
         target_package: Optional[str],
     ) -> None:
         pass
 
     def create_relation(
-        self, target_model: NonSourceNode, name: str
+        self, target_model: ManifestNode, name: str
     ) -> RelationProxy:
         if target_model.is_ephemeral_model:
             # In operations, we can't ref() ephemeral nodes, because
@@ -627,7 +628,7 @@ def __init__(
             )
         # mypy appeasement - we know it'll be a RuntimeConfig
         self.config: RuntimeConfig
-        self.model: Union[ParsedMacro, NonSourceNode] = model
+        self.model: Union[ParsedMacro, ManifestNode] = model
         super().__init__(config, manifest, model.package_name)
         self.sql_results: Dict[str, AttrDict] = {}
         self.context_config: Optional[ContextConfig] = context_config
@@ -1196,7 +1197,7 @@ def __init__(
 
 
 class ModelContext(ProviderContext):
-    model: NonSourceNode
+    model: ManifestNode
 
     @contextproperty
     def pre_hooks(self) -> List[Dict[str, Any]]:
@@ -1267,7 +1268,7 @@ def this(self) -> Optional[RelationProxy]:
 
 
 def generate_parser_model(
-    model: NonSourceNode,
+    model: ManifestNode,
     config: RuntimeConfig,
     manifest: Manifest,
     context_config: ContextConfig,
@@ -1302,7 +1303,7 @@ def generate_generate_component_name_macro(
 
 
 def generate_runtime_model(
-    model: NonSourceNode,
+    model: ManifestNode,
     config: RuntimeConfig,
     manifest: Manifest,
 ) -> Dict[str, Any]:
@@ -1322,3 +1323,45 @@ def generate_runtime_macro(
         macro, config, manifest, OperationProvider(), package_name
     )
     return ctx.to_dict()
+
+
+class ReportRefResolver(BaseResolver):
+    def __call__(self, *args) -> str:
+        if len(args) not in (1, 2):
+            ref_invalid_args(self.model, args)
+        self.model.refs.append(list(args))
+        return ''
+
+
+class ReportSourceResolver(BaseResolver):
+    def __call__(self, *args) -> str:
+        if len(args) != 2:
+            raise_compiler_error(
+                f"source() takes exactly two arguments ({len(args)} given)",
+                self.model
+            )
+        self.model.sources.append(list(args))
+        return ''
+
+
+def generate_parse_report(
+    report: ParsedReport,
+    config: RuntimeConfig,
+    manifest: Manifest,
+    package_name: str,
+) -> Dict[str, Any]:
+    project = config.load_dependencies()[package_name]
+    return {
+        'ref': ReportRefResolver(
+            None,
+            report,
+            project,
+            manifest,
+        ),
+        'source': ReportSourceResolver(
+            None,
+            report,
+            project,
+            manifest,
+        )
+    }

core/dbt/contracts/files.py

@@ -121,6 +121,7 @@ class SourceFile(JsonSchemaMixin):
     docs: List[str] = field(default_factory=list)
     macros: List[str] = field(default_factory=list)
     sources: List[str] = field(default_factory=list)
+    reports: List[str] = field(default_factory=list)
     # any node patches in this file. The entries are names, not unique ids!
     patches: List[str] = field(default_factory=list)
     # any macro patches in this file. The entries are package, name pairs.