From 614f0b43505049ad0349365d63174d3fb584059c Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 16 Jun 2020 13:08:37 -0600 Subject: [PATCH 001/282] deepcopy args in method calls --- core/dbt/rpc/method.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/dbt/rpc/method.py b/core/dbt/rpc/method.py index b2bd1b36456..74563305107 100644 --- a/core/dbt/rpc/method.py +++ b/core/dbt/rpc/method.py @@ -1,5 +1,6 @@ import inspect from abc import abstractmethod +from copy import deepcopy from typing import List, Optional, Type, TypeVar, Generic, Dict, Any from hologram import JsonSchemaMixin, ValidationError @@ -20,7 +21,7 @@ class RemoteMethod(Generic[Parameters, Result]): METHOD_NAME: Optional[str] = None def __init__(self, args, config): - self.args = args + self.args = deepcopy(args) self.config = config @classmethod From 12d3c52de2e6e6c06da7c0b9fc2c1b6e73b8b387 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 16 Jun 2020 13:28:15 -0600 Subject: [PATCH 002/282] add a test For the test to work, we have to switch to spawn so that we don't deadlock --- core/dbt/flags.py | 9 ++------- test/rpc/test_concurrency.py | 39 ++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 7 deletions(-) create mode 100644 test/rpc/test_concurrency.py diff --git a/core/dbt/flags.py b/core/dbt/flags.py index 7f3e4cb864c..ffcb3958081 100644 --- a/core/dbt/flags.py +++ b/core/dbt/flags.py @@ -28,13 +28,8 @@ def env_set_truthy(key: str) -> Optional[str]: def _get_context(): - if os.name == 'posix' and os.uname().sysname.lower() != 'darwin': - # on linux fork is available and it's fast - return multiprocessing.get_context('fork') - else: - # on windows, spawn is the only choice. - # On osx, fork is buggy: https://bugs.python.org/issue33725 - return multiprocessing.get_context('spawn') + # TODO: change this back to use fork() on linux when we have made that safe + return multiprocessing.get_context('spawn') MP_CONTEXT = _get_context() diff --git a/test/rpc/test_concurrency.py b/test/rpc/test_concurrency.py new file mode 100644 index 00000000000..31be49540ef --- /dev/null +++ b/test/rpc/test_concurrency.py @@ -0,0 +1,39 @@ +from concurrent.futures import ThreadPoolExecutor, as_completed + +from .util import ( + get_querier, + ProjectDefinition, +) + + +def _compile_poll_for_result(querier, id: int): + sql = f'select {id} as id' + resp = querier.compile_sql( + request_id=id, sql=sql, name=f'query_{id}' + ) + compile_sql_result = querier.async_wait_for_result(resp) + assert compile_sql_result['results'][0]['compiled_sql'] == sql + + +def test_rpc_compile_sql_concurrency( + project_root, profiles_root, postgres_profile, unique_schema +): + project = ProjectDefinition( + models={'my_model.sql': 'select 1 as id'} + ) + querier_ctx = get_querier( + project_def=project, + project_dir=project_root, + profiles_dir=profiles_root, + schema=unique_schema, + test_kwargs={}, + ) + + with querier_ctx as querier: + values = {} + with ThreadPoolExecutor(max_workers=10) as tpe: + for id in range(20): + fut = tpe.submit(_compile_poll_for_result, querier, id) + values[fut] = id + for fut in as_completed(values): + fut.result() From 12a9c2b4417c52cc6445890d965c3f8473780320 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 16 Jun 2020 14:46:07 -0600 Subject: [PATCH 003/282] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 384ed38274d..b6e7ddcaec9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - dbt compile and ls no longer create schemas if they don't already exist ([#2525](https://github.com/fishtown-analytics/dbt/issues/2525), [#2528](https://github.com/fishtown-analytics/dbt/pull/2528)) - `dbt deps` now respects the `--project-dir` flag, so using `dbt deps --project-dir=/some/path` and then `dbt run --project-dir=/some/path` will properly find dependencies ([#2519](https://github.com/fishtown-analytics/dbt/issues/2519), [#2534](https://github.com/fishtown-analytics/dbt/pull/2534)) - `packages.yml` revision/version fields can be float-like again (`revision: '1.0'` is valid). ([#2518](https://github.com/fishtown-analytics/dbt/issues/2518), [#2535](https://github.com/fishtown-analytics/dbt/pull/2535)) +- Parallel RPC requests no longer step on each others' arguments ([[#2484](https://github.com/fishtown-analytics/dbt/issues/2484), [#2554](https://github.com/fishtown-analytics/dbt/pull/2554)]) ## dbt 0.17.0 (June 08, 2020) From dbf7e690b8c6d9af8ff2ab273b16f638c7ce0897 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 16 Jun 2020 16:00:07 -0600 Subject: [PATCH 004/282] this test difference appears to be spawn vs fork --- test/integration/048_rpc_test/test_rpc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/integration/048_rpc_test/test_rpc.py b/test/integration/048_rpc_test/test_rpc.py index 866dcb750b3..70320681750 100644 --- a/test/integration/048_rpc_test/test_rpc.py +++ b/test/integration/048_rpc_test/test_rpc.py @@ -726,8 +726,9 @@ def test_invalid_requests_postgres(self): 'hi this is not sql', name='foo' ).json() - # neat mystery: Why is this "1" on macos and "2" on linux? - lineno = '1' if sys.platform == 'darwin' else '2' + # this is "1" if the multiprocessing context is "spawn" and "2" if + # it's fork. + lineno = '1' error_data = self.assertIsErrorWith(data, 10003, 'Database Error', { 'type': 'DatabaseException', 'message': f'Database Error in rpc foo (from remote system)\n syntax error at or near "hi"\n LINE {lineno}: hi this is not sql\n ^', From 5299079f010fda3e7c5111976ccdca66914d3fea Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Wed, 17 Jun 2020 16:30:30 -0400 Subject: [PATCH 005/282] bump docs site --- core/dbt/include/index.html | 92 ++++++++++++++++++++++++++----------- 1 file changed, 66 insertions(+), 26 deletions(-) diff --git a/core/dbt/include/index.html b/core/dbt/include/index.html index 0695b123744..bc92d4f5520 100644 --- a/core/dbt/include/index.html +++ b/core/dbt/include/index.html @@ -8,7 +8,7 @@ - + @@ -24,7 +24,7 @@
icons
- From 9c7d519ab2704dc61592e358ff233f378312ddd6 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Wed, 17 Jun 2020 16:32:57 -0400 Subject: [PATCH 006/282] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 384ed38274d..53ec2b9dc75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ - dbt compile and ls no longer create schemas if they don't already exist ([#2525](https://github.com/fishtown-analytics/dbt/issues/2525), [#2528](https://github.com/fishtown-analytics/dbt/pull/2528)) - `dbt deps` now respects the `--project-dir` flag, so using `dbt deps --project-dir=/some/path` and then `dbt run --project-dir=/some/path` will properly find dependencies ([#2519](https://github.com/fishtown-analytics/dbt/issues/2519), [#2534](https://github.com/fishtown-analytics/dbt/pull/2534)) - `packages.yml` revision/version fields can be float-like again (`revision: '1.0'` is valid). ([#2518](https://github.com/fishtown-analytics/dbt/issues/2518), [#2535](https://github.com/fishtown-analytics/dbt/pull/2535)) +- fix unclickable nodes in the dbt Docs DAG viz ([#101](https://github.com/fishtown-analytics/dbt-docs/pull/101)) +- fix null database names for Spark projects in dbt Docs site ([#96](https://github.com/fishtown-analytics/dbt-docs/pull/96)) ## dbt 0.17.0 (June 08, 2020) From f4b93c8a5a5efc30421849eb065514bf9b9bffe1 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 17 Jun 2020 12:03:01 -0600 Subject: [PATCH 007/282] handle long paths in windows, add a test --- CHANGELOG.md | 1 + core/dbt/clients/system.py | 47 +++++++++++++- .../test_docs_generate.py | 64 +++++++++++++++++++ test/integration/base.py | 5 +- 4 files changed, 115 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6e7ddcaec9..ba8aeff50b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - `dbt deps` now respects the `--project-dir` flag, so using `dbt deps --project-dir=/some/path` and then `dbt run --project-dir=/some/path` will properly find dependencies ([#2519](https://github.com/fishtown-analytics/dbt/issues/2519), [#2534](https://github.com/fishtown-analytics/dbt/pull/2534)) - `packages.yml` revision/version fields can be float-like again (`revision: '1.0'` is valid). ([#2518](https://github.com/fishtown-analytics/dbt/issues/2518), [#2535](https://github.com/fishtown-analytics/dbt/pull/2535)) - Parallel RPC requests no longer step on each others' arguments ([[#2484](https://github.com/fishtown-analytics/dbt/issues/2484), [#2554](https://github.com/fishtown-analytics/dbt/pull/2554)]) +- On windows (depending upon OS support), dbt no longer fails with errors when writing artifacts ([#2558](https://github.com/fishtown-analytics/dbt/issues/2558), [#2566](https://github.com/fishtown-analytics/dbt/pull/2566)) ## dbt 0.17.0 (June 08, 2020) diff --git a/core/dbt/clients/system.py b/core/dbt/clients/system.py index b167d192e4e..eebe6fe8945 100644 --- a/core/dbt/clients/system.py +++ b/core/dbt/clients/system.py @@ -66,6 +66,7 @@ def find_matching( def load_file_contents(path: str, strip: bool = True) -> str: + path = convert_path(path) with open(path, 'rb') as handle: to_return = handle.read().decode('utf-8') @@ -81,6 +82,7 @@ def make_directory(path: str) -> None: exist. This function handles the case where two threads try to create a directory at once. """ + path = convert_path(path) if not os.path.exists(path): # concurrent writes that try to create the same dir can fail try: @@ -99,6 +101,7 @@ def make_file(path: str, contents: str = '', overwrite: bool = False) -> bool: exists. The file is saved with contents `contents` """ if overwrite or not os.path.exists(path): + path = convert_path(path) with open(path, 'w') as fh: fh.write(contents) return True @@ -121,6 +124,7 @@ def supports_symlinks() -> bool: def write_file(path: str, contents: str = '') -> bool: + path = convert_path(path) make_directory(os.path.dirname(path)) with open(path, 'w', encoding='utf-8') as f: f.write(str(contents)) @@ -163,7 +167,7 @@ def rmdir(path: str) -> None: different permissions on Windows. Otherwise, removing directories (eg. cloned via git) can cause rmtree to throw a PermissionError exception """ - logger.debug("DEBUG** Window rmdir sys.platform: {}".format(sys.platform)) + path = convert_path(path) if sys.platform == 'win32': onerror = _windows_rmdir_readonly else: @@ -172,15 +176,49 @@ def rmdir(path: str) -> None: shutil.rmtree(path, onerror=onerror) +def convert_path(path: str) -> str: + """Convert a path that dbt has, which might be >260 characters long, to one + that will be writable/readable on Windows. + + On other platforms, this is a no-op. + """ + if sys.platform != 'win32': + return path + prefix = '\\\\?\\' + # Nothing to do + if path.startswith(prefix): + return path + path = os.path.normpath(path) + if path.startswith('\\'): + # if a path starts with '\\', splitdrive() on it will return '' for the + # drive, but the prefix requires a drive letter. So let's add the drive + # letter back in. + curdrive = os.path.splitdrive(os.getcwd())[0] + path = curdrive + path + + # now our path is either an absolute UNC path or relative to the current + # directory. If it's relative, we need to make it absolute or the prefix + # won't work. + if not os.path.splitdrive(path)[0]: + path = os.path.join(os.getcwd(), path) + + if not path.startswith(prefix): + path = prefix + path + return path + + def remove_file(path: str) -> None: + path = convert_path(path) os.remove(path) def path_exists(path: str) -> bool: + path = convert_path(path) return os.path.lexists(path) def path_is_symlink(path: str) -> bool: + path = convert_path(path) return os.path.islink(path) @@ -326,6 +364,7 @@ def run_cmd( def download(url: str, path: str, timeout: Union[float, tuple] = None) -> None: + path = convert_path(path) connection_timeout = timeout or float(os.getenv('DBT_HTTP_TIMEOUT', 10)) response = requests.get(url, timeout=connection_timeout) with open(path, 'wb') as handle: @@ -334,6 +373,8 @@ def download(url: str, path: str, timeout: Union[float, tuple] = None) -> None: def rename(from_path: str, to_path: str, force: bool = False) -> None: + from_path = convert_path(from_path) + to_path = convert_path(to_path) is_symlink = path_is_symlink(to_path) if os.path.exists(to_path) and force: @@ -348,6 +389,7 @@ def rename(from_path: str, to_path: str, force: bool = False) -> None: def untar_package( tar_path: str, dest_dir: str, rename_to: Optional[str] = None ) -> None: + tar_path = convert_path(tar_path) tar_dir_name = None with tarfile.open(tar_path, 'r') as tarball: tarball.extractall(dest_dir) @@ -384,6 +426,8 @@ def move(src, dst): This is almost identical to the real shutil.move, except it uses our rmtree and skips handling non-windows OSes since the existing one works ok there. """ + src = convert_path(src) + dst = convert_path(dst) if os.name != 'nt': return shutil.move(src, dst) @@ -418,4 +462,5 @@ def rmtree(path): """Recursively remove path. On permissions errors on windows, try to remove the read-only flag and try again. """ + path = convert_path(path) return shutil.rmtree(path, onerror=chmod_and_retry) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 213ace98b67..de0ba561eaa 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -2,15 +2,19 @@ import json import os import random +import shutil +import tempfile import time from datetime import datetime from unittest.mock import ANY, patch +from pytest import mark from test.integration.base import DBTIntegrationTest, use_profile, AnyFloat, \ AnyString, AnyStringWith, normalize, Normalized from dbt.exceptions import CompilationException + def _read_file(path): with open(path, 'r') as fp: return fp.read().replace('\r', '').replace('\\r', '') @@ -3180,3 +3184,63 @@ def test_postgres_override_used(self): self.run_dbt(['docs', 'generate']) self.assertIn('rejected: no catalogs for you', str(exc.exception)) + + +@mark.skipif(os.name != 'nt', reason='This is only relevant on windows') +class TestDocsGenerateLongWindowsPaths(DBTIntegrationTest): + def _generate_test_root_dir(self): + assert os.name == 'nt' + magic_prefix = '\\\\?\\' + + # tempfile.mkdtemp doesn't use `\\?\` by default so we have to + # get a tiny bit creative. + temp_dir = tempfile.gettempdir() + if not temp_dir.startswith(magic_prefix): + temp_dir = magic_prefix + temp_dir + outer = tempfile.mkdtemp(prefix='dbt-int-test-', dir=temp_dir) + # then inside _that_ directory make a new one that gets us to just + # barely 260 total. I picked 250 to account for the '\' and anything + # else. The key is that len(inner) + len('target\\compiled\\...') will + # be >260 chars + new_length = 250 - len(outer) + inner = os.path.join(outer, 'a'*new_length) + os.mkdir(inner) + return normalize(inner) + + def _symlink_test_folders(self): + # dbt's normal symlink behavior breaks this test, so special-case it + for entry in os.listdir(self.test_original_source_path): + src = os.path.join(self.test_original_source_path, entry) + tst = os.path.join(self.test_root_dir, entry) + if entry == 'trivial_models': + shutil.copytree(src, tst) + elif entry == 'local_dependency': + continue + elif os.path.isdir(entry) or entry.endswith('.sql'): + os.symlink(src, tst) + + @property + def schema(self): + return 'docs_generate_029' + + @staticmethod + def dir(path): + return normalize(path) + + @property + def models(self): + return self.dir("trivial_models") + + def run_and_generate(self): + self.assertEqual(len(self.run_dbt(['run'])), 1) + os.remove(normalize('target/manifest.json')) + os.remove(normalize('target/run_results.json')) + self.run_dbt(['docs', 'generate']) + + @use_profile('postgres') + def test_postgres_long_paths(self): + self.run_and_generate() + # this doesn't use abspath, so all should be well here + manifest = _read_json('./target/manifest.json') + self.assertIn('nodes', manifest) + assert os.path.exists('./target/run/test/trivial_models/model.sql') diff --git a/test/integration/base.py b/test/integration/base.py index 7122d1a6c86..0ca24bf8b94 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -348,6 +348,9 @@ def test_root_realpath(self): else: return self.test_root_dir + def _generate_test_root_dir(self): + return normalize(tempfile.mkdtemp(prefix='dbt-int-test-')) + def setUp(self): self.dbt_core_install_root = os.path.dirname(dbt.__file__) log_manager.reset_handlers() @@ -357,7 +360,7 @@ def setUp(self): self._logs_dir = os.path.join(self.initial_dir, 'logs', self.prefix) _really_makedirs(self._logs_dir) self.test_original_source_path = _pytest_get_test_root() - self.test_root_dir = normalize(tempfile.mkdtemp(prefix='dbt-int-test-')) + self.test_root_dir = self._generate_test_root_dir() os.chdir(self.test_root_dir) try: From ff272c797a5314116caedc30ad97cb58069c0940 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Thu, 18 Jun 2020 15:17:27 -0600 Subject: [PATCH 008/282] make sure dbt clean still works... --- test/integration/029_docs_generate_tests/test_docs_generate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index de0ba561eaa..18066105620 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -3244,3 +3244,5 @@ def test_postgres_long_paths(self): manifest = _read_json('./target/manifest.json') self.assertIn('nodes', manifest) assert os.path.exists('./target/run/test/trivial_models/model.sql') + self.run_dbt(['clean']) + assert not os.path.exists('./target/run') From 8e63da2f04e59cc8203099fd11a4cd62141de5b0 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Thu, 18 Jun 2020 17:11:15 -0600 Subject: [PATCH 009/282] Be pickier about when to mangle paths, handle the case where we still hit path length limits --- core/dbt/clients/system.py | 95 +++++++++++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 17 deletions(-) diff --git a/core/dbt/clients/system.py b/core/dbt/clients/system.py index eebe6fe8945..2193e6e7bc9 100644 --- a/core/dbt/clients/system.py +++ b/core/dbt/clients/system.py @@ -19,6 +19,12 @@ from dbt.logger import GLOBAL_LOGGER as logger +if sys.platform == 'win32': + from ctypes import WinDLL, c_bool +else: + WinDLL = None + c_bool = None + def find_matching( root_path: str, @@ -125,10 +131,24 @@ def supports_symlinks() -> bool: def write_file(path: str, contents: str = '') -> bool: path = convert_path(path) - make_directory(os.path.dirname(path)) - with open(path, 'w', encoding='utf-8') as f: - f.write(str(contents)) - + try: + make_directory(os.path.dirname(path)) + with open(path, 'w', encoding='utf-8') as f: + f.write(str(contents)) + except FileNotFoundError as exc: + if ( + os.name == 'nt' and + getattr(exc, 'winerror', 0) == 3 and + len(path) >= 260 + ): + # all our hard work and the path was still too long. Log and + # continue. + logger.debug( + f'Could not write to path {path}: Path was too long ' + f'({len(path)} characters)' + ) + else: + raise return True @@ -176,32 +196,73 @@ def rmdir(path: str) -> None: shutil.rmtree(path, onerror=onerror) +def _win_prepare_path(path: str) -> str: + """Given a windows path, prepare it for use by making sure it is absolute + and normalized. + """ + path = os.path.normpath(path) + + # if a path starts with '\', splitdrive() on it will return '' for the + # drive, but the prefix requires a drive letter. So let's add the drive + # letter back in. + # Unless it starts with '\\'. In that case, the path is a UNC mount point + # and splitdrive will be fine. + if not path.startswith('\\\\') and path.startswith('\\'): + curdrive = os.path.splitdrive(os.getcwd())[0] + path = curdrive + path + + # now our path is either an absolute UNC path or relative to the current + # directory. If it's relative, we need to make it absolute or the prefix + # won't work. `ntpath.abspath` allegedly doesn't always play nice with long + # paths, so do this instead. + if not os.path.splitdrive(path)[0]: + path = os.path.join(os.getcwd(), path) + + return path + + +def _supports_long_paths() -> bool: + if sys.platform != 'win32': + return True + # Eryk Sun says to use `WinDLL('ntdll')` instead of `windll.ntdll` because + # of pointer caching in a comment here: + # https://stackoverflow.com/a/35097999/11262881 + # I don't know exaclty what he means, but I am inclined to believe him as + # he's pretty active on Python windows bugs! + try: + dll = WinDLL('ntdll') + except OSError: # I don't think this happens? you need ntdll to run python + return False + # not all windows versions have it at all + if not hasattr(dll, 'RtlAreLongPathsEnabled'): + return False + # tell windows we want to get back a single unsigned byte (a bool). + dll.RtlAreLongPathsEnabled.restype = c_bool + return dll.RtlAreLongPathsEnabled() + + def convert_path(path: str) -> str: """Convert a path that dbt has, which might be >260 characters long, to one that will be writable/readable on Windows. On other platforms, this is a no-op. """ - if sys.platform != 'win32': + # some parts of python seem to append '\*.*' to strings, better safe than + # sorry. + if len(path) < 250: + return path + if _supports_long_paths(): return path + prefix = '\\\\?\\' # Nothing to do if path.startswith(prefix): return path - path = os.path.normpath(path) - if path.startswith('\\'): - # if a path starts with '\\', splitdrive() on it will return '' for the - # drive, but the prefix requires a drive letter. So let's add the drive - # letter back in. - curdrive = os.path.splitdrive(os.getcwd())[0] - path = curdrive + path - # now our path is either an absolute UNC path or relative to the current - # directory. If it's relative, we need to make it absolute or the prefix - # won't work. - if not os.path.splitdrive(path)[0]: - path = os.path.join(os.getcwd(), path) + path = _win_prepare_path(path) + # add the prefix. The check is just in case os.getcwd() does something + # unexpected - I believe this if-state should always be True though! if not path.startswith(prefix): path = prefix + path return path From db50880399b743792e3a60a2085243b55de68372 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 19 Jun 2020 07:26:40 -0600 Subject: [PATCH 010/282] fix this test for spawn() vs fork() --- test/integration/048_rpc_test/test_rpc.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/integration/048_rpc_test/test_rpc.py b/test/integration/048_rpc_test/test_rpc.py index 70320681750..7c29dc006b0 100644 --- a/test/integration/048_rpc_test/test_rpc.py +++ b/test/integration/048_rpc_test/test_rpc.py @@ -768,11 +768,9 @@ def test_timeout_postgres(self): self.assertIn('message', error_data) self.assertEqual(error_data['message'], 'RPC timed out after 1.0s') self.assertIn('logs', error_data) - if sys.platform == 'darwin': - # because fork() without exec() is broken on macos, we use 'spawn' - # so on macos we don't get logs back because we didn't fork() - return - self.assertTrue(len(error_data['logs']) > 0) + # because fork() without exec() is broken, we use 'spawn' so we don't + # get logs back because we didn't fork() + return @mark.flaky(rerun_filter=addr_in_use) From 51ce3e10933ed2f2a7fddb1b8274faeb1e867854 Mon Sep 17 00:00:00 2001 From: Bob De Schutter Date: Fri, 19 Jun 2020 16:24:01 +0200 Subject: [PATCH 011/282] Fix persist docs functionality for bigquery to include descriptions for nested columns --- .../bigquery/dbt/adapters/bigquery/impl.py | 44 +++++++++-- .../dbt/include/bigquery/macros/catalog.sql | 7 +- .../models-bigquery-nested/schema.yml | 19 +++++ .../table_model_nested.sql | 8 ++ .../view_model_nested.sql | 8 ++ .../test_persist_docs.py | 79 ++++++++++++++++++- 6 files changed, 153 insertions(+), 12 deletions(-) create mode 100644 test/integration/060_persist_docs_tests/models-bigquery-nested/schema.yml create mode 100644 test/integration/060_persist_docs_tests/models-bigquery-nested/table_model_nested.sql create mode 100644 test/integration/060_persist_docs_tests/models-bigquery-nested/view_model_nested.sql diff --git a/plugins/bigquery/dbt/adapters/bigquery/impl.py b/plugins/bigquery/dbt/adapters/bigquery/impl.py index caafcb0fbfb..0b07eb728bf 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/impl.py +++ b/plugins/bigquery/dbt/adapters/bigquery/impl.py @@ -564,6 +564,36 @@ def get_table_ref_from_relation(self, conn, relation): relation.identifier, conn) + def _update_column_dict(self, bq_column_dict, dbt_columns, parent=''): + """ + Helper function to recursively traverse the schema of a table in the + update_column_descriptions function below. + + bq_column_dict should be a dict as obtained by the to_api_repr() + function of a SchemaField object. + """ + if parent: + dotted_column_name = '{}.{}'.format(parent, bq_column_dict['name']) + else: + dotted_column_name = bq_column_dict['name'] + + if dotted_column_name in dbt_columns: + column_config = dbt_columns[dotted_column_name] + bq_column_dict['description'] = column_config.get('description') + + new_fields = [] + for child_col_dict in bq_column_dict.get('fields', list()): + new_child_column_dict = self._update_column_dict( + child_col_dict, + dbt_columns, + parent=dotted_column_name + ) + new_fields.append(new_child_column_dict) + + bq_column_dict['fields'] = new_fields + + return bq_column_dict + @available.parse_none def update_column_descriptions(self, relation, columns): if len(columns) == 0: @@ -574,13 +604,13 @@ def update_column_descriptions(self, relation, columns): table = conn.handle.get_table(table_ref) new_schema = [] - for column in table.schema: - if column.name in columns: - column_config = columns[column.name] - column_dict = column.to_api_repr() - column_dict['description'] = column_config.get('description') - column = SchemaField.from_api_repr(column_dict) - new_schema.append(column) + for bq_column in table.schema: + bq_column_dict = bq_column.to_api_repr() + new_bq_column_dict = self._update_column_dict( + bq_column_dict, + columns + ) + new_schema.append(SchemaField.from_api_repr(new_bq_column_dict)) new_table = google.cloud.bigquery.Table(table_ref, schema=new_schema) conn.handle.update_table(new_table, ['schema']) diff --git a/plugins/bigquery/dbt/include/bigquery/macros/catalog.sql b/plugins/bigquery/dbt/include/bigquery/macros/catalog.sql index ed64af88173..343a1ee04e3 100644 --- a/plugins/bigquery/dbt/include/bigquery/macros/catalog.sql +++ b/plugins/bigquery/dbt/include/bigquery/macros/catalog.sql @@ -83,7 +83,7 @@ -- use the "real" column name from the paths query below column_name as base_column_name, ordinal_position as column_index, - cast(null as string) as column_comment, + -- cast(null as string) as column_comment, is_partitioning_column, clustering_ordinal_position @@ -99,10 +99,11 @@ concat(table_catalog, '.', table_schema, '.', table_name) as relation_id, field_path as column_name, data_type as column_type, - column_name as base_column_name + column_name as base_column_name, + description as column_comment from {{ information_schema.replace(information_schema_view='COLUMN_FIELD_PATHS') }} - where data_type not like 'STRUCT%' + -- where data_type not like 'STRUCT%' ), diff --git a/test/integration/060_persist_docs_tests/models-bigquery-nested/schema.yml b/test/integration/060_persist_docs_tests/models-bigquery-nested/schema.yml new file mode 100644 index 00000000000..0311dcb1449 --- /dev/null +++ b/test/integration/060_persist_docs_tests/models-bigquery-nested/schema.yml @@ -0,0 +1,19 @@ +version: 2 + +models: + - name: table_model_nested + columns: + - name: level_1 + description: level_1 column description + - name: level_1.level_2 + description: level_2 column description + - name: level_1.level_2.level_3_a + description: level_3 column description + - name: view_model_nested + columns: + - name: level_1 + description: level_1 column description + - name: level_1.level_2 + description: level_2 column description + - name: level_1.level_2.level_3_a + description: level_3 column description \ No newline at end of file diff --git a/test/integration/060_persist_docs_tests/models-bigquery-nested/table_model_nested.sql b/test/integration/060_persist_docs_tests/models-bigquery-nested/table_model_nested.sql new file mode 100644 index 00000000000..c2936d4f186 --- /dev/null +++ b/test/integration/060_persist_docs_tests/models-bigquery-nested/table_model_nested.sql @@ -0,0 +1,8 @@ +{{ config(materialized='table') }} +SELECT + STRUCT( + STRUCT( + 1 AS level_3_a, + 2 AS level_3_b + ) AS level_2 + ) AS level_1 \ No newline at end of file diff --git a/test/integration/060_persist_docs_tests/models-bigquery-nested/view_model_nested.sql b/test/integration/060_persist_docs_tests/models-bigquery-nested/view_model_nested.sql new file mode 100644 index 00000000000..e3323ddf4e6 --- /dev/null +++ b/test/integration/060_persist_docs_tests/models-bigquery-nested/view_model_nested.sql @@ -0,0 +1,8 @@ +{{ config(materialized='view') }} +SELECT + STRUCT( + STRUCT( + 1 AS level_3_a, + 2 AS level_3_b + ) AS level_2 + ) AS level_1 \ No newline at end of file diff --git a/test/integration/060_persist_docs_tests/test_persist_docs.py b/test/integration/060_persist_docs_tests/test_persist_docs.py index c5a6cc1689e..44e651701b6 100644 --- a/test/integration/060_persist_docs_tests/test_persist_docs.py +++ b/test/integration/060_persist_docs_tests/test_persist_docs.py @@ -33,13 +33,15 @@ def _assert_has_table_comments(self, table_node): assert table_id_comment.startswith('id Column description') table_name_comment = table_node['columns']['name']['comment'] - assert table_name_comment.startswith('Some stuff here and then a call to') + assert table_name_comment.startswith( + 'Some stuff here and then a call to') self._assert_common_comments( table_comment, table_id_comment, table_name_comment ) - def _assert_has_view_comments(self, view_node, has_node_comments=True, has_column_comments=True): + def _assert_has_view_comments(self, view_node, has_node_comments=True, + has_column_comments=True): view_comment = view_node['metadata']['comment'] if has_node_comments: assert view_comment.startswith('View model description') @@ -156,3 +158,76 @@ def test_snowflake_persist_docs(self): @use_profile('bigquery') def test_bigquery_persist_docs(self): self.run_dbt() + + +class TestPersistDocsNested(BasePersistDocsTest): + @property + def project_config(self): + return { + 'config-version': 2, + 'models': { + 'test': { + '+persist_docs': { + "relation": True, + "columns": True, + }, + } + } + } + + @property + def models(self): + return 'models-bigquery-nested' + + @use_profile('bigquery') + def test_bigquery_persist_docs(self): + """ + run dbt and use the bigquery client from the adapter to check if the + colunmn descriptions are persisted on the test model table and view. + + Next, generate the catalog and check if the comments are also included. + """ + self.run_dbt() + + self.run_dbt(['docs', 'generate']) + with open('target/catalog.json') as fp: + catalog_data = json.load(fp) + assert 'nodes' in catalog_data + assert len(catalog_data['nodes']) == 2 # table and view model + + for node_id in ['table_model_nested', 'view_model_nested']: + # check the descriptions using the api + with self.adapter.connection_named('_test'): + client = self.adapter.connections \ + .get_thread_connection().handle + + table_id = "{}.{}.{}".format( + self.default_database, + self.unique_schema(), + node_id + ) + bq_schema = client.get_table(table_id).schema + + level_1_field = bq_schema[0] + assert level_1_field.description == \ + "level_1 column description" + + level_2_field = level_1_field.fields[0] + assert level_2_field.description == \ + "level_2 column description" + + level_3_field = level_2_field.fields[0] + assert level_3_field.description == \ + "level_3 column description" + + # check the descriptions in the catalog + node = catalog_data['nodes']['model.test.{}'.format(node_id)] + + level_1_column = node['columns']['level_1'] + assert level_1_column['comment'] == "level_1 column description" + + level_2_column = node['columns']['level_1.level_2'] + assert level_2_column['comment'] == "level_2 column description" + + level_3_column = node['columns']['level_1.level_2.level_3_a'] + assert level_3_column['comment'] == "level_3 column description" From 3a5db3c16d4bbcc39f4981902f89f49a082afef2 Mon Sep 17 00:00:00 2001 From: Bob De Schutter Date: Fri, 19 Jun 2020 16:37:53 +0200 Subject: [PATCH 012/282] Updated changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6e7ddcaec9..b0b08af69e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,11 @@ - `dbt deps` now respects the `--project-dir` flag, so using `dbt deps --project-dir=/some/path` and then `dbt run --project-dir=/some/path` will properly find dependencies ([#2519](https://github.com/fishtown-analytics/dbt/issues/2519), [#2534](https://github.com/fishtown-analytics/dbt/pull/2534)) - `packages.yml` revision/version fields can be float-like again (`revision: '1.0'` is valid). ([#2518](https://github.com/fishtown-analytics/dbt/issues/2518), [#2535](https://github.com/fishtown-analytics/dbt/pull/2535)) - Parallel RPC requests no longer step on each others' arguments ([[#2484](https://github.com/fishtown-analytics/dbt/issues/2484), [#2554](https://github.com/fishtown-analytics/dbt/pull/2554)]) +- `persist_docs` now takes into account descriptions for nested columns in bigquery ([#2549](https://github.com/fishtown-analytics/dbt/issues/2549), [#2550](https://github.com/fishtown-analytics/dbt/pull/2550)) +Contributors: + - [@bodschut](https://github.com/bodschut) ([#2550](https://github.com/fishtown-analytics/dbt/pull/2550)) + ## dbt 0.17.0 (June 08, 2020) ### Fixes From e99f0d12b815ca8ace8bcac9a08386207256b162 Mon Sep 17 00:00:00 2001 From: Bob De Schutter Date: Fri, 19 Jun 2020 16:56:17 +0200 Subject: [PATCH 013/282] Remove commented out code Co-authored-by: Jacob Beck --- plugins/bigquery/dbt/include/bigquery/macros/catalog.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/bigquery/dbt/include/bigquery/macros/catalog.sql b/plugins/bigquery/dbt/include/bigquery/macros/catalog.sql index 343a1ee04e3..6822d88a6a8 100644 --- a/plugins/bigquery/dbt/include/bigquery/macros/catalog.sql +++ b/plugins/bigquery/dbt/include/bigquery/macros/catalog.sql @@ -83,7 +83,6 @@ -- use the "real" column name from the paths query below column_name as base_column_name, ordinal_position as column_index, - -- cast(null as string) as column_comment, is_partitioning_column, clustering_ordinal_position @@ -103,7 +102,6 @@ description as column_comment from {{ information_schema.replace(information_schema_view='COLUMN_FIELD_PATHS') }} - -- where data_type not like 'STRUCT%' ), From 0d2593e0e3a26dcb6c4eda9d88ab165bfdbbb324 Mon Sep 17 00:00:00 2001 From: Bob De Schutter Date: Fri, 19 Jun 2020 18:19:50 +0200 Subject: [PATCH 014/282] Fix docs_generate test --- .../bq_models/nested_table.sql | 4 ++-- .../029_docs_generate_tests/test_docs_generate.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/test/integration/029_docs_generate_tests/bq_models/nested_table.sql b/test/integration/029_docs_generate_tests/bq_models/nested_table.sql index 61309296267..22f9048d629 100644 --- a/test/integration/029_docs_generate_tests/bq_models/nested_table.sql +++ b/test/integration/029_docs_generate_tests/bq_models/nested_table.sql @@ -10,6 +10,6 @@ select 3 as field_3, struct( - 4 as field_4, - 5 as field_5 + 5 as field_5, + 6 as field_6 ) as nested_field diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 18066105620..cb899806eac 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -619,10 +619,10 @@ def expected_bigquery_complex_catalog(self): 'type': 'INT64', 'comment': None }, - 'nested_field.field_4': { - 'name': 'nested_field.field_4', + 'nested_field': { + 'name': 'nested_field', 'index': 4, - 'type': 'INT64', + 'type': 'STRUCT', 'comment': None }, 'nested_field.field_5': { @@ -630,6 +630,12 @@ def expected_bigquery_complex_catalog(self): 'index': 5, 'type': 'INT64', 'comment': None + }, + 'nested_field.field_6': { + 'name': 'nested_field.field_6', + 'index': 6, + 'type': 'INT64', + 'comment': None } } From 3829d1f964c25bb852565fae45dac593b09659dd Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 17 Jun 2020 07:39:41 -0600 Subject: [PATCH 015/282] Fix a regression where dbt ignored aliases in config() calls --- CHANGELOG.md | 1 + core/dbt/context/context_config.py | 3 ++- .../040_override_database_test/models/view_2.sql | 8 +++++--- test/integration/base.py | 2 ++ 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba8aeff50b2..cfc2389c426 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - `packages.yml` revision/version fields can be float-like again (`revision: '1.0'` is valid). ([#2518](https://github.com/fishtown-analytics/dbt/issues/2518), [#2535](https://github.com/fishtown-analytics/dbt/pull/2535)) - Parallel RPC requests no longer step on each others' arguments ([[#2484](https://github.com/fishtown-analytics/dbt/issues/2484), [#2554](https://github.com/fishtown-analytics/dbt/pull/2554)]) - On windows (depending upon OS support), dbt no longer fails with errors when writing artifacts ([#2558](https://github.com/fishtown-analytics/dbt/issues/2558), [#2566](https://github.com/fishtown-analytics/dbt/pull/2566)) +- dbt again respects config aliases in config() calls ([#2557](https://github.com/fishtown-analytics/dbt/issues/2557), [#2559](https://github.com/fishtown-analytics/dbt/pull/2559)) ## dbt 0.17.0 (June 08, 2020) diff --git a/core/dbt/context/context_config.py b/core/dbt/context/context_config.py index a33be3357e4..3cdf9bf92e1 100644 --- a/core/dbt/context/context_config.py +++ b/core/dbt/context/context_config.py @@ -131,8 +131,9 @@ def active_project_configs( def _update_from_config( self, result: T, partial: Dict[str, Any], validate: bool = False ) -> T: + translated = self.active_project.credentials.translate_aliases(partial) return result.update_from( - partial, + translated, self.active_project.credentials.type, validate=validate ) diff --git a/test/integration/040_override_database_test/models/view_2.sql b/test/integration/040_override_database_test/models/view_2.sql index bdfa5369fa7..9ac6bdad6a7 100644 --- a/test/integration/040_override_database_test/models/view_2.sql +++ b/test/integration/040_override_database_test/models/view_2.sql @@ -1,4 +1,6 @@ -{{ - config(database=var('alternate_db')) -}} +{%- if target.type == 'bigquery' -%} + {{ config(project=var('alternate_db')) }} +{%- else -%} + {{ config(database=var('alternate_db')) }} +{%- endif -%} select * from {{ ref('seed') }} diff --git a/test/integration/base.py b/test/integration/base.py index 0ca24bf8b94..6ae5a62fe2e 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -1024,6 +1024,8 @@ def assertManyRelationsEqual(self, relations, default_schema=None, default_datab first_columns = None for relation in specs: key = (relation.database, relation.schema, relation.identifier) + # get a good error here instead of a hard-to-diagnose KeyError + self.assertIn(key, column_specs, f'No columns found for {key}') columns = column_specs[key] if first_columns is None: first_columns = columns From e300c62d763cedb69e78ce6fc0d56c996eb4bbcb Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 19 Jun 2020 11:24:47 -0600 Subject: [PATCH 016/282] add a test --- .../test_override_database.py | 66 +++++++++++++++---- 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/test/integration/040_override_database_test/test_override_database.py b/test/integration/040_override_database_test/test_override_database.py index 37ea59b8eb5..085750a8b15 100644 --- a/test/integration/040_override_database_test/test_override_database.py +++ b/test/integration/040_override_database_test/test_override_database.py @@ -99,13 +99,44 @@ def test_snowflake_database_override(self): self.run_database_override() -class TestProjectModelOverride(BaseOverrideDatabase): +class BaseTestProjectModelOverride(BaseOverrideDatabase): + # this is janky, but I really want to access self.default_database in + # project_config + @property + def default_database(self): + target = self._profile_config['test']['target'] + profile = self._profile_config['test']['outputs'][target] + for key in ['database', 'project', 'dbname']: + if key in profile: + database = profile[key] + if self.adapter_type == 'snowflake': + return database.upper() + return database + assert False, 'No profile database found!' + def run_database_override(self): + self.run_dbt_notstrict(['seed']) + self.assertEqual(len(self.run_dbt_notstrict(['run'])), 4) + self.assertExpectedRelations() + + def assertExpectedRelations(self): if self.adapter_type == 'snowflake': func = lambda x: x.upper() else: func = lambda x: x + self.assertManyRelationsEqual([ + (func('seed'), self.unique_schema(), self.default_database), + (func('view_2'), self.unique_schema(), self.alternative_database), + (func('view_1'), self.unique_schema(), self.alternative_database), + (func('view_3'), self.unique_schema(), self.default_database), + (func('view_4'), self.unique_schema(), self.alternative_database), + ]) + + +class TestProjectModelOverride(BaseTestProjectModelOverride): + @property + def project_config(self): self.use_default_project({ 'config-version': 2, 'vars': { @@ -120,16 +151,6 @@ def run_database_override(self): } }, }) - self.run_dbt_notstrict(['seed']) - - self.assertEqual(len(self.run_dbt_notstrict(['run'])), 4) - self.assertManyRelationsEqual([ - (func('seed'), self.unique_schema(), self.default_database), - (func('view_2'), self.unique_schema(), self.alternative_database), - (func('view_1'), self.unique_schema(), self.alternative_database), - (func('view_3'), self.unique_schema(), self.default_database), - (func('view_4'), self.unique_schema(), self.alternative_database), - ]) @use_profile('bigquery') def test_bigquery_database_override(self): @@ -140,6 +161,29 @@ def test_snowflake_database_override(self): self.run_database_override() +class TestProjectModelAliasOverride(BaseTestProjectModelOverride): + @property + def project_config(self): + return { + 'config-version': 2, + 'vars': { + 'alternate_db': self.alternative_database, + }, + 'models': { + 'project': self.alternative_database, + 'test': { + 'subfolder': { + 'project': self.default_database, + } + } + }, + } + + @use_profile('bigquery') + def test_bigquery_project_override(self): + self.run_database_override() + + class TestProjectSeedOverride(BaseOverrideDatabase): def run_database_override(self): if self.adapter_type == 'snowflake': From 05917c6d2ffcbca346dfa9e45f9c7c81c7ebf5f1 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 19 Jun 2020 11:29:35 -0600 Subject: [PATCH 017/282] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfc2389c426..712835010c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - `packages.yml` revision/version fields can be float-like again (`revision: '1.0'` is valid). ([#2518](https://github.com/fishtown-analytics/dbt/issues/2518), [#2535](https://github.com/fishtown-analytics/dbt/pull/2535)) - Parallel RPC requests no longer step on each others' arguments ([[#2484](https://github.com/fishtown-analytics/dbt/issues/2484), [#2554](https://github.com/fishtown-analytics/dbt/pull/2554)]) - On windows (depending upon OS support), dbt no longer fails with errors when writing artifacts ([#2558](https://github.com/fishtown-analytics/dbt/issues/2558), [#2566](https://github.com/fishtown-analytics/dbt/pull/2566)) -- dbt again respects config aliases in config() calls ([#2557](https://github.com/fishtown-analytics/dbt/issues/2557), [#2559](https://github.com/fishtown-analytics/dbt/pull/2559)) +- dbt again respects config aliases in config() calls and dbt_project.yml ([#2557](https://github.com/fishtown-analytics/dbt/issues/2557), [#2559](https://github.com/fishtown-analytics/dbt/pull/2559), [#2575](https://github.com/fishtown-analytics/dbt/pull/2575)) ## dbt 0.17.0 (June 08, 2020) From 098d05c27fd157595ab8f511c7eae6085713f7ad Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 19 Jun 2020 11:52:43 -0600 Subject: [PATCH 018/282] Fix tests --- .../test_override_database.py | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/test/integration/040_override_database_test/test_override_database.py b/test/integration/040_override_database_test/test_override_database.py index 085750a8b15..1686ca1214e 100644 --- a/test/integration/040_override_database_test/test_override_database.py +++ b/test/integration/040_override_database_test/test_override_database.py @@ -137,7 +137,7 @@ def assertExpectedRelations(self): class TestProjectModelOverride(BaseTestProjectModelOverride): @property def project_config(self): - self.use_default_project({ + return { 'config-version': 2, 'vars': { 'alternate_db': self.alternative_database, @@ -150,7 +150,17 @@ def project_config(self): } } }, - }) + 'data-paths': ['data'], + 'vars': { + 'alternate_db': self.alternative_database, + }, + 'quoting': { + 'database': True, + }, + 'seeds': { + 'quote_columns': False, + } + } @use_profile('bigquery') def test_bigquery_database_override(self): @@ -177,6 +187,16 @@ def project_config(self): } } }, + 'data-paths': ['data'], + 'vars': { + 'alternate_db': self.alternative_database, + }, + 'quoting': { + 'database': True, + }, + 'seeds': { + 'quote_columns': False, + } } @use_profile('bigquery') From aa35dc9437d420799acbc0ea2d257b789a4adbc0 Mon Sep 17 00:00:00 2001 From: Claire Carroll Date: Fri, 19 Jun 2020 15:25:28 -0400 Subject: [PATCH 019/282] Update logo --- README.md | 2 +- etc/dbt-horizontal.png | Bin 8968 -> 0 bytes etc/dbt-logo-full.svg | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 etc/dbt-horizontal.png create mode 100644 etc/dbt-logo-full.svg diff --git a/README.md b/README.md index 34e6e647ffa..fd78588ad09 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- dbt logo + dbt logo

diff --git a/etc/dbt-horizontal.png b/etc/dbt-horizontal.png deleted file mode 100644 index 2d2549089222496ec86b65423fe0122f29a5d4d7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8968 zcmb7q^7_;b(jb_Tj zkd~bE_xSmI|AEhbfW2Pl+&lNV;(fiZ>wPCaG0KQu^VUw^jy?P35t@LZ!)pxiR{OeW6Wwe>b!n5%(+D@|)OW`$j;_*KN3edd% z!#@hGNyLdS-xmQunI>~tM`lOCM`Xg|_F9#mvhjDF@!VoV%G$4vkkR*#<=!0InoK<_ zP8C<#{4~@_21KXCU2Dm74ZO$lN+HX#>V8&$1|To&n4PQJ_C$Wy?se5OkAHMEsB}zWMSa11<9=XpIk`}-*RLaDyAPoR6Z$_|FPL^W#Ax=mM6wu^T z!w$2DOu~O1&rV5K-HCWXoUrWK0Dx$La4NK8_JU5fX=I0!mjk(l8z+Sm69Q2Yn(J>; zN?8p6l!ID?^?NP|t3Iyb)?3jGNs7Tpk*E=dL;@QS?bFH#=gy?@FzI!$an2*vl02}X z=p)8eDUIKBfbt7CHM+@Z7qj;?pd#&xso%)ywC)jMs__f01>QW}+JuDxP`15KjgFc` z^f*m3#FO`bad-=^`xvl`mV*NL0u+>}w+jnCY%d zp+&fQVX6M-Var{euX8T)c#Dn?>Xwnz1HPrv^4d$%7nee))yHp`^r~+ffR5zjnrgvR zhjcBatOTTemrn~YDwzoziz&F)lS__1Pz|e3j+i_r99TQGv4ams_20zr*{r=Tu9m{x zztcI#Kj&yz(*EerA)kcZy`UO%Z=}Kp134&|gJ&MoqZdQ3wyp9eS130%)y zR$s7*m@{>0cqa>I!j_`$6{xLuvsr-+;h?7-fr`&pWL3f3|!gcaG!ep_(X?> z{ArO1wsW4Us{RwoJM7dlxiQ6Uo(2Gd%>{jug%%?ZjrBJem2e$42o~+70m=4;#}>bdl-h?noqr90~xTWy3!zKY5=7N#WA&XzZ^cP)^95#cy%N_2NTEA1GoN z#>E^M6_0;EA&o~!=CeCyE55L0J++q&Jpa6>Q}q4g*F%7yp-;`cxxTX{b6nMttUp<^ zt^9j|$AII#_z{CKRVwCsrudgsreoM+I1>bPi z@#gLCdIR|0^#$nO?bvqGlD6$kJ5__WyPQa%LodDOyupl9H7Paas}4SWLoIk9D7CZt zMP2ciGNIK3i~iZ>c;hJ2Hzx;-JG)OTE$mX;CLw9qZf(*iqL;b`BU885_qif*X<@T? zd!%jnd@tJ_yfby35^du)?`*v6a8dQ8+@)3{DW1c9ml zhwxvo?EMg)`>hD=)WaiDPTu?xuZR=*)GX8E(JZLCSS4yi4vw3a#{q-0gwoIK%&Cu~ z;*BQ?J6Az#n>ANp1*agt-t@xRt=D$bJzp$P%=+AU6VsWA z;yc1mM0YZBRtlsqe&Ri>F#9gP)NFPB{)Ztfc(Z#c3IGUNRamKPLubcqM44s0H+T5f zGo*Qdz1B2QyzvWD`b$qmy8%i0^kZw@KD`JXxjEP1baq~MT-K2zxE88^(qi4mRiKJY zlqMJF9Mlm=L*Q+?Il4byrooRvm<7tJk9wh2QN3%Mf!u2k%aS9!9HM)h$bnJ+MhfVL z8W;?1jn_uZK#Nox5kPosbs?twjwq%AbM+iAxqSb1pX#)Ex>xvLKTF~t`7{dtbRKFZ z5BNzjV-8F|9q|3I@&`WP`!h_)1+5K@(T8$4L5vQp##VmQ0Lsdl%N9gi|J!RBgE6bY z%k0n(9knFb75J89mCzLD=PLnmM^NB_`oQjM9*PN>qbO3uxhZh|On3c?Ey+&%>0AEEp|DEpnhTgh=)=7-H1o|a$ zVnBRoF-ef)tlzUj8c;6N6F98md1i*5wy%($wNun6p~vK&fVQ!%P#*;{ls2LNu0`0V7Czli!n(gF3ji4M8+;iba4;ruHs3pD}0?`hwy1@PW#*+%LHEii^xu0z) z#?K~-Q{|^g_u|h5aiqLh?NBJ%@$Z(yzG)NOUf-n07wgqmr<-$2-GAHvj+?@fqeliD2|&8E^@*CV6x#$uLyPSzp>*>gA96vx;x^>HDTvXs}Kvc&V# zFKDA)%GxOHeks`VV6&^go^QOWaHvDuC{`vdoL}qEft>7NgeHVv%TBU|O>_SJ4M$mn zJDpiq9z3NH>P{Cp8qpcp1(^efh#!r6mxpX#*)0CDBq#aIsdMGe_-?klg!syq4^NBc z{+b%fNdjf_==sJB!Au z^!(YT`^gs)I6^g;-t9tFfAhO zfRuWve{OG)xy*tpIJBM9SJb(~4@>a|c}kvIrkDq80| z)>pR{kE|1&`(2;`1C$kl;*Bqr#9e7Zw~b3=U+nX(*`I&{^_&Te_EP)SAf|32XswA0*=||P29M)BQxSqqo;$_Sn|rFF~2j*@gNlXW+CDt z*^zvm9#z3xxaAEw5OYB!@h{As2X!*$k(zfHt|m5P&Fon{m=Ia6M{_bmE=mY6tZQc# zk4Idf4c@YOF0vP*ELTswcH`3&aScZ^RnG`)BLflMwvRig(d?*($YXfP3{E#AaL?T8 zU(1_YFOP(K1f^{cr^Xo~>|=3-SBG00tJ(?dAIU}bc)&Dh3bNk{rX9HONS*R<5H3QD zrK~){`Gt#_Byfi6>oq!K{>vW>`5jJ&qfpuHMNz7SwG2z~fJMHcOHol~2rEHXi>*(o}F2 z+R(8au731xDuzIgk4cH`1abbWbSY$}{Aq&t3N?n`tR6j{>UGU9fj-Ss1Njv~#TZbY z+~<4xu4*yQlE zR{5WnK)zn=6y2>G?)KabM1+_F#nYvyM$6(u0?2Kbp0C8cOR0j?%w_+F7J$po7izF7 zE7yj*;okNyVj19`0_fiz$!$25ih4tXn6u z{OE!RHbpJoov)zqHjVm4-}S8Pr0!gx?B4?{vgCpa4OQBZ{7cvMB=W^lKu)pnKmo{= zcvE|AX%5TBCVsQt`)EhCNJGAl)<5mHdI?(6b>-pQYsr?Kw(oXkJ1b2&U-iZXwcOcK za5M5MG$1Cm-3Os<>wR!Q;jeW*bB)q;Oc*RoF*)uZ%rDrI(&TM;oL)kvSo)qpPIpdE zo_6+(ah7}BE>BRw$|JN{mMf;DWA8k3xp*%k``D^TV>?*CLHHMT>O|M>%y{QZP;&X5 zi5;-$2ptREd?Yrp`L9&PuU9uEnZp*p>+(3P&&!Ch5yj@ELx^3TDZ>iavh3P3-5bi@ zhu$Q-Tgk-|$6*ugPxxXKsP7e1 zSZ?Sos3m(0Z#PE`8X%JLP+!Gi|G`ph1}=MNpgef$z-!?Iw*e|1)(d*%22uiSEFXso z?caM8NnGOhPZY{JfJ)|H+|aBsa(d_*W+R_HT`43eG$zV|{hk+8%kx&;AGhbk%_`P2 znaVWgQakP!F1%K8rr4zxl~47>GvQEuJB-4ra+b>kB%+Qer^{YRv;1EqG5gxip;hrA z3}o%&dKmdlsTaRucbwLO(yiSaT8a(-g*m!uL|B*K_R;u#a;+s^X{R}(w&H&}x`?dS ztFYB;QNn9YV9|HX=AQ8=9of{YGOEEQ4JK|T_Hc)fy6mPzv*kWSA#I8}m^OEAFjrzI z(Q#n3jkVZVp`2F55A}VQ7v32T*?1mJ;pqt+me7X2Rh1!=WOn2!oZRfsdoFDpo63eR zfg?26q3ArrHL${qyu<1^>L0mJ@^tLM-ByMh!;-K;g{tI;r=i^^p-cPcd1PniV&wSI zoEaI}Xw#CJk8Pa?H@_r3c`S$>gQh0WA;Op=5`W}XXdW9H-H$FfyzRq%-|DiJNd@6S z5m1o$ZK91=&hs&o!Uz;~U8I&-k{nNbDHjArq4N1J!?{d}Qz;7M@m-d84_#@j-%&)z zC1ix;YL3#v&t21r_l6HCBN7v&B6!lt{FvOkd2p10nuJ+6d_U6J)9ZuaU*wGN zCRcQDYU@dX4!MR=jf6MEzEZ5bviS--MG@8?X(QINsirSBn-y#^f&X#CR^%W>F5177 z7OqyETsh{j1l7%F*6Wv~FPtDdSxHWi@@H}C?_Fx3M#F|2|BXZ)Nbrvj<$XxU-oCeMmnwW+ z^|Sv!)68H9OQ0fpjs0L4IgImk-03(+JR=|1G9lD zbUhnnK;UEow$ZSvNHnkvxl%VWa)T<#6App=?={u{f*VCP#sSVhaZYGXNcnIazz^Bd zSN(q1!ehD3^aWEEtu`3h%8z3i;Zq~gPIYR1%G831M5GeR7TSH^=6C=*xP(zb0FL|nd3NJRM+(ykUTvb>^tW44L!c6 z`T1u#Xd?jlK`ZLhNVbUDiD=kVA18?qMRG~kFHjkrd)~8%7-^sYr%II9Tly(Q)iyUm z_}1F=>UTg&nCZ%{weKpr*1wNYeeZBOC*#8po|uJ%_W zR6s+LP&6$JtmsE#b1#>kvI+w&S|}iv9pnS`Rex_&coj=kvttc*am?v_c52@RpID44 zmN(oel5<0lZF1sc+zZEIjmap7oV%w;E_izF(w`HKHV^37(H&r9Vf_}2qDg9$BOp%n zy5Lz(N%BUo7Oy)XNKD?v)6R*@hZtFFrm{h3x72gsKjrZdaLkdOqVao{=2h$hL@82? z1xz4_Rm#&=2t_)W17(9D1P~=(f4R98YF;v);x0qg(~_=iHpQ#Gm@P>E+3?LM7T3tt-lG)?U83L7`zon5qAt%l?gu+9k&FO zjvvv~F>T$tDJl*OlS4VV;!lc?%f0;m4PH_AZvh_*2|wcz0D-1>Wx7W;EkiRYBDf0# z8DOY&uqA#}o4{Y2H>vMMS>2j&ZYu-YMeT><)XE=+)8uH}@eSWjrWteXms>Xw2k+is z!D6Tzcd5Oa)z3ER}|HuGDmA`|g{?W>rz zG=?^NKc`gR1-6Cq+6f;5_$0MaPT{dU()o=+4c2NTkmDTR`zP(FRL@pq%C z@w&?9O<&!Bh$nI{Ra@pORNDl@Pals%MSAeLlr_ZF-0A#~`IF_HT8LcurjiW?;cU6+ z*={G9PfYjA*=>LFXMrW&O_ABY?UNAF2OdFQM&!We6w3HWV z+aDA@`0e6chuC;8WH{j0PR;&xqB?BL&>AL7#u%@CPC~5B$WfXsPQTlih?4qYQ3&Aws@=seaEs8r` zeRs_BzW1EwKcQsytYLGBtkF{oC!9!Z$eId z*F{iU*BM=g@-zChcHf;Y6pyi|6+075$IRh079<+eazeL5S2zHt_bENYAB5>(k;!qb{YPNG@s5Gv4LVO7Auo#|h9u%Ji zcTYHRI2g8}tEgU&86=xbyyGe>>aaeI>4kgF9L2w%?ovZP`}TXDGL!5K?2owr_IoZc zutr2U&l8_g+k->BEElERsn&W1+f>i3ow(p<>$70cCMrHkD7+!!7Z!bc@?>}VLc3#| zJlOJw)FlN}5S7l@LN91oDF z2YFAHk_U`!lEmZAqA^5`1&g#kceabuh@> zuIev?vW1mka?rS#mfOSNj6pI{dDWvQ4sU0{EfwAr@!MM6=`GhxddhHGcYpSpt z;pRGxv64$uW%8>HPx{m`tty)8@l>!ZIjO@JiBEUR5ARxB9eA?zpLJbGc~Ebb*;pQp zcF$+-?$E@L$OPCkE1MSN_s*%iK8>ZDZ_ES74Ra~0uW{4D0yS|+TRj|-7;+u0&^eiN zI!hb+v}Eij(mNKXb&Ek$xggsK?kYQJ_xt|&WK1*#{E(s3Yz{MAfu zueb_=JIuyjDUET>O$Q_rYNqm_1+CL@f#cpAgvNTK<|6Mi_sgFA-Vb96rw^FHFBD$sx~Ni<0=?#Sx9InaEOS8S&W?2joE)3v!@lgeF@UK zTPH=t^F`yE2E(&@&$;J<4LQa^(f1>{77ZJd6>> zl?ms&?Iwflg-=m4&s7CX9__a#iVul8YW_ajVp-3q`mvCBE+>Sq(%g4lSyc>mkj?9b z$NZYxzib${7+(3KJPyg-B93KDP(TKDKU!aOiaH!gRQ!3j;c>=!C?wIv<9p_+d{}WP z(vvvIwv@P(tl-YSzkhQ3Uh^So`nw-|jff=2>zXmKdwsH&ZvIGNGnHBwlL6|9HqOlX z+gqd^uwzJT*tW6)!GrW^apC7oC4|}moKD6U-HDW`t9Mh`nDsj|TcEK+Q{WTx#I#OL zH^L6~i%-NectT0zml>aE1hsx%8?HeO zl=Z+;3?1sKYw_nD9{g)+&!RWQ4h6$nIQKi>$yx|T9d+ewZpI9f4wc9AL(h14%VtQY zDnw^Y`hkAx=XDAdH!5P7)Wk{1-aDfx$IFzdvmmABsO#bhm2}!vDAUVz2Y7{3%QhH{ z=G7WGQUSjADpOE;SXl64kAX)H+C2^xbAK}}4%RPuigpt-&d|B`01!yJKKpqDF5ps? zU;fa0+GcKclz!d`9@F%0a1+fd`LeT|lQH?(D|fS{^~EtK1TU+(<}JJg;vd!P!8EZY zSr&eA`9|@E8^U>4Y?nS=kWG}(q0|I;twP>#iSRUVRZn(>S{H=rp*Zh=%9BYUTtljYJppo0g*sSJ6U0iXpzHKUsPlD7z4`A7m)Xc1)$aX6JJ~#cp zugQNab|;tLXq!_4Bs?wPE`(t|73?lI=EoMribD|qF#1|9#(gUE%QNwzb*H^yx5a8S zXyy|;UUFP1Xnc@$nV_zyw=S=29OYE`cXbe90Dvt5*H|wrU2o!^81~q5G=5qo=Z7je zD2;ISp5m8R-8?L^0uKj#{d*SR$U-1mgiA!eJln3hIe*e3NplPU4xSv2WfjHsOA5JH zHl|-;Y^G*7`qrNGnIevQ^j?8Gqdaq^LnM!tv+-P$b-vm8lNmM1d|y=-6#C)_`=$!ofj*u#W96_djTZHMm4+OPA$)gr2;|jgOgH0!fZCVb&$~XkWtN%& z3#`HQQL9$Ss9q;A(2nJC*xHx!=Vx3YABntfjQ42B06u0>a%eg3o&J*b;Lk$29PX|s zT>|D+?m=XLGA*caBN?5m5K=AGx)NKU&WsM;R(=At$*TuNuA7Y#Xyf0Oc(}mV?6Aci z>zag*UToX@KSaTqel3M7_lz=?#G)4tDwdLaAeR`f?P}}Dk0z(EA!nR5G(z3 z<|IH%uY9V&n&ideQFj;-RKs>ri>0(MGJqCUjWtW}mz2-C@}wB_mPop3fFVVL#y@bSx`)md})~Sld{bL;N)`31cqYueriXZ{Z0ZY&^LembCpr)GAs zGc`f_{5kRBJSybl9Mrndi2xotiC)OK3`16LGf9TAou6Lb|NbGFn)&Z(5HxcOxUAqh zEgTePN9ig59!mK|^;g&i4#;Cb@d*fx!Lot>{;!{^( \ No newline at end of file From 32702559f6620398f79c0d5d55a238fc27abb235 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Fri, 19 Jun 2020 21:13:16 +0000 Subject: [PATCH 020/282] Release dbt v0.17.1rc1 --- .bumpversion.cfg | 2 +- CHANGELOG.md | 3 + core/dbt/version.py | 2 +- core/setup.py | 2 +- .../requirements/requirements.0.17.1rc1.txt | 64 +++++++++++++++++++ .../dbt/adapters/bigquery/__version__.py | 2 +- plugins/bigquery/setup.py | 2 +- .../dbt/adapters/postgres/__version__.py | 2 +- plugins/postgres/setup.py | 2 +- .../dbt/adapters/redshift/__version__.py | 2 +- plugins/redshift/setup.py | 2 +- .../dbt/adapters/snowflake/__version__.py | 2 +- plugins/snowflake/setup.py | 2 +- setup.py | 2 +- 14 files changed, 79 insertions(+), 12 deletions(-) create mode 100644 docker/requirements/requirements.0.17.1rc1.txt diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 76045d8e247..d4f4de13e87 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.17.1a1 +current_version = 0.17.1rc1 parse = (?P\d+) \.(?P\d+) \.(?P\d+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 190977a861e..771054f4449 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## dbt 0.17.1 (Release TBD) +## dbt 0.17.1rc1 (June 19, 2020) + + ### Fixes - dbt compile and ls no longer create schemas if they don't already exist ([#2525](https://github.com/fishtown-analytics/dbt/issues/2525), [#2528](https://github.com/fishtown-analytics/dbt/pull/2528)) - `dbt deps` now respects the `--project-dir` flag, so using `dbt deps --project-dir=/some/path` and then `dbt run --project-dir=/some/path` will properly find dependencies ([#2519](https://github.com/fishtown-analytics/dbt/issues/2519), [#2534](https://github.com/fishtown-analytics/dbt/pull/2534)) diff --git a/core/dbt/version.py b/core/dbt/version.py index c3bf7765592..73cb71bb6da 100644 --- a/core/dbt/version.py +++ b/core/dbt/version.py @@ -96,5 +96,5 @@ def _get_dbt_plugins_info(): yield plugin_name, mod.version -__version__ = '0.17.1a1' +__version__ = '0.17.1rc1' installed = get_installed_version() diff --git a/core/setup.py b/core/setup.py index 82fc31ae8ba..cabfbb39217 100644 --- a/core/setup.py +++ b/core/setup.py @@ -18,7 +18,7 @@ def read(fname): package_name = "dbt-core" -package_version = "0.17.1a1" +package_version = "0.17.1rc1" description = """dbt (data build tool) is a command line tool that helps \ analysts and engineers transform data in their warehouse more effectively""" diff --git a/docker/requirements/requirements.0.17.1rc1.txt b/docker/requirements/requirements.0.17.1rc1.txt new file mode 100644 index 00000000000..4d2113811de --- /dev/null +++ b/docker/requirements/requirements.0.17.1rc1.txt @@ -0,0 +1,64 @@ +agate==1.6.1 +asn1crypto==1.3.0 +attrs==19.3.0 +azure-common==1.1.25 +azure-storage-blob==2.1.0 +azure-storage-common==2.1.0 +Babel==2.8.0 +boto3==1.11.17 +botocore==1.14.17 +cachetools==4.1.0 +certifi==2020.4.5.2 +cffi==1.13.2 +chardet==3.0.4 +colorama==0.4.3 +cryptography==2.9.2 +decorator==4.4.2 +docutils==0.15.2 +google-api-core==1.16.0 +google-auth==1.18.0 +google-cloud-bigquery==1.24.0 +google-cloud-core==1.3.0 +google-resumable-media==0.5.1 +googleapis-common-protos==1.6.0 +hologram==0.0.7 +idna==2.8 +ijson==2.6.1 +importlib-metadata==1.6.1 +isodate==0.6.0 +Jinja2==2.11.2 +jmespath==0.10.0 +json-rpc==1.13.0 +jsonschema==3.1.1 +leather==0.3.3 +Logbook==1.5.3 +MarkupSafe==1.1.1 +minimal-snowplow-tracker==0.0.2 +networkx==2.4 +oscrypto==1.2.0 +parsedatetime==2.6 +protobuf==3.11.3 +psycopg2-binary==2.8.5 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycparser==2.20 +pycryptodomex==3.9.7 +PyJWT==1.7.1 +pyOpenSSL==19.1.0 +pyrsistent==0.16.0 +python-dateutil==2.8.1 +python-slugify==4.0.0 +pytimeparse==1.1.8 +pytz==2020.1 +PyYAML==5.3.1 +requests==2.22.0 +rsa==4.6 +s3transfer==0.3.3 +six==1.15.0 +snowflake-connector-python==2.2.1 +sqlparse==0.3.1 +text-unidecode==1.3 +typing-extensions==3.7.4.2 +urllib3==1.25.9 +Werkzeug==0.16.1 +zipp==3.1.0 diff --git a/plugins/bigquery/dbt/adapters/bigquery/__version__.py b/plugins/bigquery/dbt/adapters/bigquery/__version__.py index 45490b1a275..a2e726b395b 100644 --- a/plugins/bigquery/dbt/adapters/bigquery/__version__.py +++ b/plugins/bigquery/dbt/adapters/bigquery/__version__.py @@ -1 +1 @@ -version = '0.17.1a1' +version = '0.17.1rc1' diff --git a/plugins/bigquery/setup.py b/plugins/bigquery/setup.py index 409e2902f8e..35eff781dbf 100644 --- a/plugins/bigquery/setup.py +++ b/plugins/bigquery/setup.py @@ -14,7 +14,7 @@ package_name = "dbt-bigquery" -package_version = "0.17.1a1" +package_version = "0.17.1rc1" description = """The bigquery adapter plugin for dbt (data build tool)""" this_directory = os.path.abspath(os.path.dirname(__file__)) diff --git a/plugins/postgres/dbt/adapters/postgres/__version__.py b/plugins/postgres/dbt/adapters/postgres/__version__.py index 45490b1a275..a2e726b395b 100644 --- a/plugins/postgres/dbt/adapters/postgres/__version__.py +++ b/plugins/postgres/dbt/adapters/postgres/__version__.py @@ -1 +1 @@ -version = '0.17.1a1' +version = '0.17.1rc1' diff --git a/plugins/postgres/setup.py b/plugins/postgres/setup.py index 7903817b386..da7817d13b5 100644 --- a/plugins/postgres/setup.py +++ b/plugins/postgres/setup.py @@ -35,7 +35,7 @@ def _dbt_psycopg2_name(): package_name = "dbt-postgres" -package_version = "0.17.1a1" +package_version = "0.17.1rc1" description = """The postgres adpter plugin for dbt (data build tool)""" this_directory = os.path.abspath(os.path.dirname(__file__)) diff --git a/plugins/redshift/dbt/adapters/redshift/__version__.py b/plugins/redshift/dbt/adapters/redshift/__version__.py index 45490b1a275..a2e726b395b 100644 --- a/plugins/redshift/dbt/adapters/redshift/__version__.py +++ b/plugins/redshift/dbt/adapters/redshift/__version__.py @@ -1 +1 @@ -version = '0.17.1a1' +version = '0.17.1rc1' diff --git a/plugins/redshift/setup.py b/plugins/redshift/setup.py index 68caa022aac..282fe59c0c9 100644 --- a/plugins/redshift/setup.py +++ b/plugins/redshift/setup.py @@ -14,7 +14,7 @@ package_name = "dbt-redshift" -package_version = "0.17.1a1" +package_version = "0.17.1rc1" description = """The redshift adapter plugin for dbt (data build tool)""" this_directory = os.path.abspath(os.path.dirname(__file__)) diff --git a/plugins/snowflake/dbt/adapters/snowflake/__version__.py b/plugins/snowflake/dbt/adapters/snowflake/__version__.py index 45490b1a275..a2e726b395b 100644 --- a/plugins/snowflake/dbt/adapters/snowflake/__version__.py +++ b/plugins/snowflake/dbt/adapters/snowflake/__version__.py @@ -1 +1 @@ -version = '0.17.1a1' +version = '0.17.1rc1' diff --git a/plugins/snowflake/setup.py b/plugins/snowflake/setup.py index 357362ad4d2..5b8531b02d1 100644 --- a/plugins/snowflake/setup.py +++ b/plugins/snowflake/setup.py @@ -14,7 +14,7 @@ package_name = "dbt-snowflake" -package_version = "0.17.1a1" +package_version = "0.17.1rc1" description = """The snowflake adapter plugin for dbt (data build tool)""" this_directory = os.path.abspath(os.path.dirname(__file__)) diff --git a/setup.py b/setup.py index 96dd765d95b..837c839016b 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ package_name = "dbt" -package_version = "0.17.1a1" +package_version = "0.17.1rc1" description = """With dbt, data analysts and engineers can build analytics \ the way engineers build applications.""" From 3616641e93d9ccc1e604055e65c6796cee503c51 Mon Sep 17 00:00:00 2001 From: Claire Carroll Date: Mon, 22 Jun 2020 08:48:59 -0400 Subject: [PATCH 021/282] Update other logos --- plugins/bigquery/README.md | 2 +- plugins/postgres/README.md | 2 +- plugins/redshift/README.md | 2 +- plugins/snowflake/README.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/bigquery/README.md b/plugins/bigquery/README.md index ec3527497ed..b7e967abe72 100644 --- a/plugins/bigquery/README.md +++ b/plugins/bigquery/README.md @@ -1,5 +1,5 @@

- dbt logo + dbt logo

**[dbt](https://www.getdbt.com/)** (data build tool) enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications. diff --git a/plugins/postgres/README.md b/plugins/postgres/README.md index 1afb624f9f0..dc4214e2e1c 100644 --- a/plugins/postgres/README.md +++ b/plugins/postgres/README.md @@ -1,5 +1,5 @@

- dbt logo + dbt logo

**[dbt](https://www.getdbt.com/)** (data build tool) enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications. diff --git a/plugins/redshift/README.md b/plugins/redshift/README.md index e47f666cfc2..d3c3c06f7eb 100644 --- a/plugins/redshift/README.md +++ b/plugins/redshift/README.md @@ -1,5 +1,5 @@

- dbt logo + dbt logo

**[dbt](https://www.getdbt.com/)** (data build tool) enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications. diff --git a/plugins/snowflake/README.md b/plugins/snowflake/README.md index c499ccadccd..93636fd30ec 100644 --- a/plugins/snowflake/README.md +++ b/plugins/snowflake/README.md @@ -1,5 +1,5 @@

- dbt logo + dbt logo

**[dbt](https://www.getdbt.com/)** (data build tool) enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications. From 37b03ef5d2126820f5015d6e8782b761c9689cd2 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 23 Jun 2020 10:00:24 -0600 Subject: [PATCH 022/282] Fix prefixing support to not render "+pre-hook" Increased the startup timeout in the RPC tests to try to fix some local timeouts --- CHANGELOG.md | 7 ++++++- core/dbt/config/renderer.py | 9 +++++---- .../014_hook_tests/test_model_hooks.py | 17 +++++++++++++++++ test/integration/048_rpc_test/test_rpc.py | 2 +- 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 771054f4449..1f43b063498 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ ## dbt 0.17.1 (Release TBD) + +### Fixes +- dbt config-version: 2 now properly defers rendering `+pre-hook` and `+post-hook` fields. ([#2583](https://github.com/fishtown-analytics/dbt/issues/2583), [#2854](https://github.com/fishtown-analytics/dbt/pull/2854)) + + ## dbt 0.17.1rc1 (June 19, 2020) @@ -16,7 +21,7 @@ Contributors: - [@bodschut](https://github.com/bodschut) ([#2550](https://github.com/fishtown-analytics/dbt/pull/2550)) - + ## dbt 0.17.0 (June 08, 2020) ### Fixes diff --git a/core/dbt/config/renderer.py b/core/dbt/config/renderer.py index f65625c9e3b..c5bc3258f7b 100644 --- a/core/dbt/config/renderer.py +++ b/core/dbt/config/renderer.py @@ -102,11 +102,12 @@ def should_render_keypath_v2(self, keypath: Keypath) -> bool: return False if first in {'seeds', 'models', 'snapshots', 'seeds'}: + keypath_parts = { + (k.lstrip('+') if isinstance(k, str) else k) + for k in keypath + } # model-level hooks - if 'pre-hook' in keypath or 'post-hook' in keypath: - return False - # model-level 'vars' declarations - if 'vars' in keypath: + if 'pre-hook' in keypath_parts or 'post-hook' in keypath_parts: return False return True diff --git a/test/integration/014_hook_tests/test_model_hooks.py b/test/integration/014_hook_tests/test_model_hooks.py index 2efce14aca7..ad4c8ce3ef0 100644 --- a/test/integration/014_hook_tests/test_model_hooks.py +++ b/test/integration/014_hook_tests/test_model_hooks.py @@ -226,6 +226,23 @@ def test_postgres_hooks_on_seeds(self): self.assertEqual(len(res), 1, 'Expected exactly one item') +class TestPrePostModelHooksOnSeedsPlusPrefixed(TestPrePostModelHooksOnSeeds): + @property + def project_config(self): + return { + 'config-version': 2, + 'data-paths': ['data'], + 'models': {}, + 'seeds': { + '+post-hook': [ + 'alter table {{ this }} add column new_col int', + 'update {{ this }} set new_col = 1' + ], + 'quote_columns': False, + }, + } + + class TestPrePostModelHooksOnSnapshots(DBTIntegrationTest): @property def schema(self): diff --git a/test/integration/048_rpc_test/test_rpc.py b/test/integration/048_rpc_test/test_rpc.py index 7c29dc006b0..912876624df 100644 --- a/test/integration/048_rpc_test/test_rpc.py +++ b/test/integration/048_rpc_test/test_rpc.py @@ -66,7 +66,7 @@ def is_up(self): def start(self): super().start() - for _ in range(30): + for _ in range(60): if self.is_up(): break time.sleep(0.5) From d38fd80dff38a9a58f45eac9b13ce061840ff82c Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 23 Jun 2020 13:25:01 -0600 Subject: [PATCH 023/282] PR feedback: double seeds --- core/dbt/config/renderer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/config/renderer.py b/core/dbt/config/renderer.py index c5bc3258f7b..9e37b0e70a9 100644 --- a/core/dbt/config/renderer.py +++ b/core/dbt/config/renderer.py @@ -78,7 +78,7 @@ def should_render_keypath_v1(self, keypath: Keypath) -> bool: if first in {'on-run-start', 'on-run-end', 'query-comment'}: return False # models have two things to avoid - if first in {'seeds', 'models', 'snapshots', 'seeds'}: + if first in {'seeds', 'models', 'snapshots'}: # model-level hooks if 'pre-hook' in keypath or 'post-hook' in keypath: return False From 26735dac8dbd23061a0a57aa01283207f9aef9d5 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Thu, 25 Jun 2020 09:35:13 -0400 Subject: [PATCH 024/282] update docs logo --- core/dbt/include/index.html | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/core/dbt/include/index.html b/core/dbt/include/index.html index bc92d4f5520..9a20c3cc558 100644 --- a/core/dbt/include/index.html +++ b/core/dbt/include/index.html @@ -24,7 +24,7 @@
icons
- From 62a0bf87327f24486386315b776534409259aab6 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Fri, 19 Jun 2020 15:10:59 -0600 Subject: [PATCH 025/282] store package info on the factory instead of globally adapter_types -> plugins Use factory.packages instead of global PACKAGES --- core/dbt/adapters/factory.py | 76 ++++++++++++++++--- core/dbt/config/runtime.py | 6 +- core/dbt/context/configured.py | 13 +++- core/dbt/context/providers.py | 6 +- core/dbt/contracts/graph/manifest.py | 12 ++- core/dbt/parser/manifest.py | 18 ++--- .../test_concurrent_transaction.py | 3 +- test/unit/test_bigquery_adapter.py | 3 +- test/unit/test_context.py | 17 +++-- test/unit/test_graph.py | 9 ++- test/unit/test_postgres_adapter.py | 5 +- test/unit/test_redshift_adapter.py | 9 ++- test/unit/test_snowflake_adapter.py | 3 +- test/unit/utils.py | 10 ++- 14 files changed, 139 insertions(+), 51 deletions(-) diff --git a/core/dbt/adapters/factory.py b/core/dbt/adapters/factory.py index 8cc8d20ae39..307abba2bc9 100644 --- a/core/dbt/adapters/factory.py +++ b/core/dbt/adapters/factory.py @@ -1,9 +1,13 @@ import threading +from pathlib import Path from importlib import import_module -from typing import Type, Dict, Any +from typing import Type, Dict, Any, List, Set, Optional -from dbt.exceptions import RuntimeException -from dbt.include.global_project import PACKAGES +from dbt.exceptions import RuntimeException, InternalException +from dbt.include.global_project import ( + PACKAGE_PATH as GLOBAL_PROJECT_PATH, + PROJECT_NAME as GLOBAL_PROJECT_NAME, +) from dbt.logger import GLOBAL_LOGGER as logger from dbt.contracts.connection import Credentials, AdapterRequiredConfig @@ -24,18 +28,25 @@ class AdpaterContainer: def __init__(self): self.lock = threading.Lock() self.adapters: Dict[str, Adapter] = {} - self.adapter_types: Dict[str, Type[Adapter]] = {} + self.plugins: Dict[str, AdapterPlugin] = {} + # map package names to their include paths + self.packages: Dict[str, Path] = { + GLOBAL_PROJECT_NAME: Path(GLOBAL_PROJECT_PATH), + } - def get_adapter_class_by_name(self, name: str) -> Type[Adapter]: + def get_plugin_by_name(self, name: str) -> AdapterPlugin: with self.lock: - if name in self.adapter_types: - return self.adapter_types[name] - - names = ", ".join(self.adapter_types.keys()) + if name in self.plugins: + return self.plugins[name] + names = ", ".join(self.plugins.keys()) message = f"Invalid adapter type {name}! Must be one of {names}" raise RuntimeException(message) + def get_adapter_class_by_name(self, name: str) -> Type[Adapter]: + plugin = self.get_plugin_by_name(name) + return plugin.adapter + def get_relation_class_by_name(self, name: str) -> Type[BaseRelation]: adapter = self.get_adapter_class_by_name(name) return adapter.Relation @@ -47,7 +58,7 @@ def get_config_class_by_name( return adapter.AdapterSpecificConfigs def load_plugin(self, name: str) -> Type[Credentials]: - # this doesn't need a lock: in the worst case we'll overwrite PACKAGES + # this doesn't need a lock: in the worst case we'll overwrite packages # and adapter_type entries with the same value, as they're all # singletons try: @@ -74,9 +85,9 @@ def load_plugin(self, name: str) -> Type[Credentials]: with self.lock: # things do hold the lock to iterate over it so we need it to add - self.adapter_types[name] = plugin.adapter + self.plugins[name] = plugin - PACKAGES[plugin.project_name] = plugin.include_path + self.packages[plugin.project_name] = Path(plugin.include_path) for dep in plugin.dependencies: self.load_plugin(dep) @@ -114,6 +125,39 @@ def cleanup_connections(self): for adapter in self.adapters.values(): adapter.cleanup_connections() + def get_adapter_package_names(self, name: Optional[str]) -> Set[str]: + if name is None: + return list(self.packages) + package_names: Set[str] = {GLOBAL_PROJECT_NAME} + plugin_names: Set[str] = {name} + while plugin_names: + plugin_name = plugin_names.pop() + try: + plugin = self.plugins[plugin_name] + except KeyError: + raise InternalException( + f'No plugin found for {plugin_name}' + ) from None + package_names.add(plugin.adapter.type()) + if plugin.dependencies is None: + continue + for dep in plugin.dependencies: + if dep not in package_names: + plugin_names.add(dep) + return package_names + + def get_include_paths(self, name: Optional[str]) -> List[Path]: + paths = [] + for package_name in self.get_adapter_package_names(name): + try: + path = self.packages[package_name] + except KeyError: + raise InternalException( + f'No internal package listing found for {package_name}' + ) + paths.append(path) + return paths + FACTORY: AdpaterContainer = AdpaterContainer() @@ -153,3 +197,11 @@ def get_relation_class_by_name(name: str) -> Type[BaseRelation]: def load_plugin(name: str) -> Type[Credentials]: return FACTORY.load_plugin(name) + + +def get_include_paths(name: Optional[str]) -> List[Path]: + return FACTORY.get_include_paths(name) + + +def get_adapter_package_names(name: Optional[str]) -> Set[str]: + return FACTORY.get_adapter_package_names(name) diff --git a/core/dbt/config/runtime.py b/core/dbt/config/runtime.py index bc212878775..bad39fa8b6c 100644 --- a/core/dbt/config/runtime.py +++ b/core/dbt/config/runtime.py @@ -12,7 +12,7 @@ from .project import Project from .renderer import DbtProjectYamlRenderer, ProfileRenderer from dbt import tracking -from dbt.adapters.factory import get_relation_class_by_name +from dbt.adapters.factory import get_relation_class_by_name, get_include_paths from dbt.helper_types import FQNPath, PathSet from dbt.context.base import generate_base_context from dbt.context.target import generate_target_context @@ -31,7 +31,6 @@ warn_or_error, raise_compiler_error ) -from dbt.include.global_project import PACKAGES from dbt.legacy_config_updater import ConfigUpdater from hologram import ValidationError @@ -323,8 +322,9 @@ def warn_for_unused_resource_config_paths( def load_dependencies(self) -> Mapping[str, 'RuntimeConfig']: if self.dependencies is None: all_projects = {self.project_name: self} + internal_packages = get_include_paths(self.credentials.type) project_paths = itertools.chain( - map(Path, PACKAGES.values()), + internal_packages, self._get_project_directories() ) for project_name, project in self.load_projects(project_paths): diff --git a/core/dbt/context/configured.py b/core/dbt/context/configured.py index a0a1510fdf8..c658240eace 100644 --- a/core/dbt/context/configured.py +++ b/core/dbt/context/configured.py @@ -1,10 +1,9 @@ -from typing import Any, Dict, Iterable, Union, Optional +from typing import Any, Dict, Iterable, Union, Optional, Set from dbt.clients.jinja import MacroGenerator, MacroStack from dbt.contracts.connection import AdapterRequiredConfig from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.parsed import ParsedMacro -from dbt.include.global_project import PACKAGES from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME from dbt.node_types import NodeType from dbt.utils import MultiDict @@ -93,10 +92,12 @@ def __init__( root_package: str, search_package: str, thread_ctx: MacroStack, + internal_packages: Set[str], node: Optional[Any] = None, ) -> None: self.root_package = root_package self.search_package = search_package + self.internal_packages = internal_packages self.globals: FlatNamespace = {} self.locals: FlatNamespace = {} self.packages: Dict[str, FlatNamespace] = {} @@ -110,7 +111,7 @@ def add_macro(self, macro: ParsedMacro, ctx: Dict[str, Any]): ) # put plugin macros into the root namespace - if macro.package_name in PACKAGES: + if macro.package_name in self.internal_packages: namespace: str = GLOBAL_PROJECT_NAME else: namespace = macro.package_name @@ -161,10 +162,16 @@ def __init__( self.macro_stack = MacroStack() def _get_namespace(self): + # avoid an import loop + from dbt.adapters.factory import get_adapter_package_names + internal_packages = get_adapter_package_names( + self.config.credentials.type + ) return MacroNamespace( self.config.project_name, self.search_package, self.macro_stack, + internal_packages, None, ) diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 5c59591217b..e85be69d93b 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -8,7 +8,7 @@ from dbt.adapters.base.column import Column -from dbt.adapters.factory import get_adapter +from dbt.adapters.factory import get_adapter, get_adapter_package_names from dbt.clients import agate_helper from dbt.clients.jinja import get_rendered from dbt.config import RuntimeConfig, Project @@ -573,10 +573,14 @@ def __init__( self.db_wrapper = self.provider.DatabaseWrapper(self.adapter) def _get_namespace(self): + internal_packages = get_adapter_package_names( + self.config.credentials.type + ) return MacroNamespace( self.config.project_name, self.search_package, self.macro_stack, + internal_packages, self.model, ) diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index 2df6c0e8dec..42d9faa6030 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -25,7 +25,6 @@ warn_or_error, raise_invalid_patch ) from dbt.helper_types import PathSet -from dbt.include.global_project import PACKAGES from dbt.logger import GLOBAL_LOGGER as logger from dbt.node_types import NodeType from dbt import deprecations @@ -468,10 +467,12 @@ def last(self) -> Optional[ParsedMacro]: return self[-1].macro -def _get_locality(macro: ParsedMacro, root_project_name: str) -> Locality: +def _get_locality( + macro: ParsedMacro, root_project_name: str, internal_packages: Set[str] +) -> Locality: if macro.package_name == root_project_name: return Locality.Root - elif macro.package_name in PACKAGES: + elif macro.package_name in internal_packages: return Locality.Core else: return Locality.Imported @@ -647,12 +648,15 @@ def _find_macros_by_name( ) -> CandidateList: """Find macros by their name. """ + # avoid an import cycle + from dbt.adapters.factory import get_adapter_package_names candidates: CandidateList = CandidateList() + packages = get_adapter_package_names(self.metadata.adapter_type) for unique_id, macro in self.macros.items(): if macro.name != name: continue candidate = MacroCandidate( - locality=_get_locality(macro, root_project_name), + locality=_get_locality(macro, root_project_name, packages), macro=macro, ) if filter is None or filter(candidate): diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 60ef395fdbd..341e9de8a08 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -9,9 +9,11 @@ import dbt.flags from dbt import deprecations -from dbt.adapters.factory import get_relation_class_by_name +from dbt.adapters.factory import ( + get_relation_class_by_name, + get_adapter_package_names, +) from dbt.helper_types import PathSet -from dbt.include.global_project import PACKAGES from dbt.logger import GLOBAL_LOGGER as logger, DbtProcessState from dbt.node_types import NodeType from dbt.clients.jinja import get_rendered @@ -119,13 +121,14 @@ def _load_macros( ) -> None: projects = self.all_projects if internal_manifest is not None: + # skip internal packages + packages = get_adapter_package_names(self.root_project.credentials.type) projects = { - k: v for k, v in self.all_projects.items() if k not in PACKAGES + k: v for k, v in self.all_projects.items() if k not in packages } self.results.macros.update(internal_manifest.macros) self.results.files.update(internal_manifest.files) - # TODO: go back to skipping the internal manifest during macro parsing for project in projects.values(): parser = MacroParser(self.results, project) for path in parser.search(): @@ -416,10 +419,6 @@ def _check_manifest(manifest: Manifest, config: RuntimeConfig) -> None: _warn_for_unused_resource_config_paths(manifest, config) -def internal_project_names(): - return iter(PACKAGES.values()) - - def _load_projects(config, paths): for path in paths: try: @@ -626,7 +625,8 @@ def process_node( def load_internal_projects(config): - return dict(_load_projects(config, internal_project_names())) + project_names = get_adapter_package_names(config.credentials.type) + return dict(_load_projects(config, project_names)) def load_internal_manifest(config: RuntimeConfig) -> Manifest: diff --git a/test/integration/032_concurrent_transaction_test/test_concurrent_transaction.py b/test/integration/032_concurrent_transaction_test/test_concurrent_transaction.py index 69d2a2af903..c1051bb584b 100644 --- a/test/integration/032_concurrent_transaction_test/test_concurrent_transaction.py +++ b/test/integration/032_concurrent_transaction_test/test_concurrent_transaction.py @@ -4,7 +4,8 @@ def get_adapter_standalone(config): - cls = FACTORY.adapter_types[config.credentials.type] + plugin = FACTORY.plugins[config.credentials.type] + cls = plugin.adapter return cls(config) diff --git a/test/unit/test_bigquery_adapter.py b/test/unit/test_bigquery_adapter.py index 4a8ac7ae303..71c3c95d3b8 100644 --- a/test/unit/test_bigquery_adapter.py +++ b/test/unit/test_bigquery_adapter.py @@ -12,6 +12,7 @@ from dbt.adapters.bigquery import BigQueryCredentials from dbt.adapters.bigquery import BigQueryAdapter from dbt.adapters.bigquery import BigQueryRelation +from dbt.adapters.bigquery import Plugin as BigQueryPlugin from dbt.adapters.bigquery.relation import BigQueryInformationSchema from dbt.adapters.bigquery.connections import BigQueryConnectionManager from dbt.adapters.base.query_headers import MacroQueryStringSetter @@ -94,7 +95,7 @@ def get_adapter(self, target): self.mock_query_header_add = self.qh_patch.start() self.mock_query_header_add.side_effect = lambda q: '/* dbt */\n{}'.format(q) - inject_adapter(adapter) + inject_adapter(adapter, BigQueryPlugin) return adapter diff --git a/test/unit/test_context.py b/test/unit/test_context.py index d8d25c92d48..c8131fdf9d7 100644 --- a/test/unit/test_context.py +++ b/test/unit/test_context.py @@ -5,7 +5,7 @@ import pytest -# make sure 'postgres' is in PACKAGES +# make sure 'postgres' is available from dbt.adapters import postgres # noqa from dbt.adapters.base import AdapterConfig from dbt.clients.jinja import MacroStack @@ -349,6 +349,13 @@ def get_adapter(): yield patch +@pytest.fixture +def get_include_paths(): + with mock.patch.object(providers, 'get_include_paths') as patch: + patch.return_value = [] + yield patch + + @pytest.fixture def config(): return config_from_parts_or_dicts(PROJECT_DATA, PROFILE_DATA) @@ -367,7 +374,7 @@ def test_query_header_context(config, manifest): assert_has_keys(REQUIRED_QUERY_HEADER_KEYS, MAYBE_KEYS, ctx) -def test_macro_parse_context(config, manifest, get_adapter): +def test_macro_parse_context(config, manifest, get_adapter, get_include_paths): ctx = providers.generate_parser_macro( macro=manifest.macros['macro.root.macro_a'], config=config, @@ -377,7 +384,7 @@ def test_macro_parse_context(config, manifest, get_adapter): assert_has_keys(REQUIRED_MACRO_KEYS, MAYBE_KEYS, ctx) -def test_macro_runtime_context(config, manifest, get_adapter): +def test_macro_runtime_context(config, manifest, get_adapter, get_include_paths): ctx = providers.generate_runtime_macro( macro=manifest.macros['macro.root.macro_a'], config=config, @@ -387,7 +394,7 @@ def test_macro_runtime_context(config, manifest, get_adapter): assert_has_keys(REQUIRED_MACRO_KEYS, MAYBE_KEYS, ctx) -def test_model_parse_context(config, manifest, get_adapter): +def test_model_parse_context(config, manifest, get_adapter, get_include_paths): ctx = providers.generate_parser_model( model=mock_model(), config=config, @@ -397,7 +404,7 @@ def test_model_parse_context(config, manifest, get_adapter): assert_has_keys(REQUIRED_MODEL_KEYS, MAYBE_KEYS, ctx) -def test_model_runtime_context(config, manifest, get_adapter): +def test_model_runtime_context(config, manifest, get_adapter, get_include_paths): ctx = providers.generate_runtime_model( model=mock_model(), config=config, diff --git a/test/unit/test_graph.py b/test/unit/test_graph.py index 1707ea753b0..7c1e9ca3cd8 100644 --- a/test/unit/test_graph.py +++ b/test/unit/test_graph.py @@ -2,6 +2,8 @@ import unittest from unittest.mock import MagicMock, patch +from dbt.adapters.postgres import Plugin as PostgresPlugin +from dbt.adapters.factory import reset_adapters import dbt.clients.system import dbt.compilation import dbt.exceptions @@ -24,7 +26,7 @@ from dbt.logger import GLOBAL_LOGGER as logger # noqa -from .utils import config_from_parts_or_dicts, generate_name_macros, MockMacro +from .utils import config_from_parts_or_dicts, generate_name_macros, MockMacro, inject_plugin class GraphTest(unittest.TestCase): @@ -39,7 +41,7 @@ def tearDown(self): self.mock_hook_constructor.stop() self.load_patch.stop() self.load_source_file_patcher.stop() - # self.relation_update_patcher.stop() + reset_adapters() def setUp(self): dbt.flags.STRICT_MODE = True @@ -104,8 +106,6 @@ def _mock_parse_result(config, all_projects): self.mock_source_file = self.load_source_file_patcher.start() self.mock_source_file.side_effect = lambda path: [n for n in self.mock_models if n.path == path][0] - # self.relation_update_patcher = patch.object(RelationUpdate, '_relation_components', lambda: []) - # self.mock_relation_update = self.relation_update_patcher.start() self.internal_manifest = Manifest.from_macros(macros={ n.unique_id: n for n in generate_name_macros('test_models_compile') }) @@ -131,6 +131,7 @@ def create_hook_patcher(cls, results, project, relative_dirs, extension): self.mock_filesystem_constructor.side_effect = create_filesystem_searcher self.mock_hook_constructor = self.hook_patcher.start() self.mock_hook_constructor.side_effect = create_hook_patcher + inject_plugin(PostgresPlugin) def get_config(self, extra_cfg=None): if extra_cfg is None: diff --git a/test/unit/test_postgres_adapter.py b/test/unit/test_postgres_adapter.py index 3b1ef63567e..6c5b3b0eb12 100644 --- a/test/unit/test_postgres_adapter.py +++ b/test/unit/test_postgres_adapter.py @@ -8,6 +8,7 @@ from dbt.adapters.base.query_headers import MacroQueryStringSetter from dbt.adapters.postgres import PostgresAdapter +from dbt.adapters.postgres import Plugin as PostgresPlugin from dbt.clients import agate_helper from dbt.exceptions import ValidationException, DbtConfigError from dbt.logger import GLOBAL_LOGGER as logger # noqa @@ -50,7 +51,7 @@ def setUp(self): def adapter(self): if self._adapter is None: self._adapter = PostgresAdapter(self.config) - inject_adapter(self._adapter) + inject_adapter(self._adapter, PostgresPlugin) return self._adapter @mock.patch('dbt.adapters.postgres.connections.psycopg2') @@ -283,7 +284,7 @@ def setUp(self): self.mock_query_header_add = self.qh_patch.start() self.mock_query_header_add.side_effect = lambda q: '/* dbt */\n{}'.format(q) self.adapter.acquire_connection() - inject_adapter(self.adapter) + inject_adapter(self.adapter, PostgresPlugin) self.load_patch = mock.patch('dbt.parser.manifest.make_parse_result') self.mock_parse_result = self.load_patch.start() diff --git a/test/unit/test_redshift_adapter.py b/test/unit/test_redshift_adapter.py index f2801de8e3d..91fa30e8fd5 100644 --- a/test/unit/test_redshift_adapter.py +++ b/test/unit/test_redshift_adapter.py @@ -1,4 +1,3 @@ -import string import unittest from unittest import mock import agate @@ -6,12 +5,15 @@ import dbt.adapters # noqa import dbt.flags as flags -from dbt.adapters.redshift import RedshiftAdapter +from dbt.adapters.redshift import ( + RedshiftAdapter, + Plugin as RedshiftPlugin, +) from dbt.clients import agate_helper from dbt.exceptions import FailedToConnectException from dbt.logger import GLOBAL_LOGGER as logger # noqa -from .utils import config_from_parts_or_dicts, mock_connection, TestAdapterConversions +from .utils import config_from_parts_or_dicts, mock_connection, TestAdapterConversions, inject_adapter @classmethod @@ -60,6 +62,7 @@ def setUp(self): def adapter(self): if self._adapter is None: self._adapter = RedshiftAdapter(self.config) + inject_adapter(self._adapter, RedshiftPlugin) return self._adapter def test_implicit_database_conn(self): diff --git a/test/unit/test_snowflake_adapter.py b/test/unit/test_snowflake_adapter.py index 7e41feeb589..9d0c60705f3 100644 --- a/test/unit/test_snowflake_adapter.py +++ b/test/unit/test_snowflake_adapter.py @@ -6,6 +6,7 @@ import dbt.flags as flags from dbt.adapters.snowflake import SnowflakeAdapter +from dbt.adapters.snowflake import Plugin as SnowflakePlugin from dbt.adapters.snowflake.column import SnowflakeColumn from dbt.adapters.base.query_headers import MacroQueryStringSetter from dbt.clients import agate_helper @@ -72,7 +73,7 @@ def setUp(self): self.mock_query_header_add.side_effect = lambda q: '/* dbt */\n{}'.format(q) self.adapter.acquire_connection() - inject_adapter(self.adapter) + inject_adapter(self.adapter, SnowflakePlugin) def tearDown(self): # we want a unique self.handle every time. diff --git a/test/unit/utils.py b/test/unit/utils.py index d27793afd87..a3f198ab7c4 100644 --- a/test/unit/utils.py +++ b/test/unit/utils.py @@ -103,14 +103,20 @@ def config_from_parts_or_dicts(project, profile, packages=None, cli_vars='{}'): ) -def inject_adapter(value): +def inject_plugin(plugin): + from dbt.adapters.factory import FACTORY + key = plugin.adapter.type() + FACTORY.plugins[key] = plugin + + +def inject_adapter(value, plugin): """Inject the given adapter into the adapter factory, so your hand-crafted artisanal adapter will be available from get_adapter() as if dbt loaded it. """ + inject_plugin(plugin) from dbt.adapters.factory import FACTORY key = value.type() FACTORY.adapters[key] = value - FACTORY.adapter_types[key] = type(value) class ContractTestCase(TestCase): From 32c559838d692c572a6a33446c5d71b63b88d257 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 23 Jun 2020 10:14:24 -0600 Subject: [PATCH 026/282] Try to make imports a little more sane, ordering-wise consolidate dbt.ui, move non-rpc node_runners into their tasks move parse_cli_vars into config.utils get rid of logger/exceptions requirements in dbt.utils --- core/dbt/adapters/base/connections.py | 8 +- core/dbt/adapters/base/impl.py | 14 +- core/dbt/adapters/sql/connections.py | 7 +- core/dbt/adapters/sql/impl.py | 1 - core/dbt/clients/jinja.py | 6 +- core/dbt/compilation.py | 6 +- core/dbt/config/runtime.py | 6 +- core/dbt/config/utils.py | 23 + core/dbt/contracts/graph/model_config.py | 2 +- core/dbt/contracts/graph/parsed.py | 6 +- core/dbt/contracts/project.py | 6 +- core/dbt/deprecations.py | 6 +- core/dbt/deps/git.py | 4 +- core/dbt/exceptions.py | 8 +- core/dbt/logger.py | 17 +- core/dbt/main.py | 6 +- core/dbt/node_runners.py | 662 ------------------ core/dbt/parser/base.py | 6 +- core/dbt/parser/manifest.py | 65 +- core/dbt/parser/results.py | 4 +- core/dbt/parser/sources.py | 4 +- core/dbt/rpc/gc.py | 1 - core/dbt/rpc/node_runners.py | 2 +- core/dbt/rpc/task_handler.py | 6 +- core/dbt/rpc/task_handler_protocol.py | 1 - core/dbt/rpc/task_manager.py | 7 +- core/dbt/task/base.py | 296 +++++++- core/dbt/task/compile.py | 26 +- core/dbt/task/debug.py | 18 +- core/dbt/task/freshness.py | 101 ++- core/dbt/task/generate.py | 15 +- core/dbt/{ui => task}/printer.py | 144 +--- core/dbt/task/rpc/cli.py | 2 +- core/dbt/task/run.py | 194 ++++- core/dbt/task/run_operation.py | 13 +- core/dbt/task/runnable.py | 31 +- core/dbt/task/seed.py | 35 +- core/dbt/task/snapshot.py | 16 +- core/dbt/task/test.py | 91 ++- core/dbt/ui.py | 76 ++ core/dbt/ui/__init__.py | 0 core/dbt/ui/colors.py | 10 - core/dbt/utils.py | 75 -- .../bigquery/dbt/adapters/bigquery/impl.py | 9 +- test/unit/test_main.py | 2 +- test/unit/utils.py | 4 +- 46 files changed, 1033 insertions(+), 1009 deletions(-) create mode 100644 core/dbt/config/utils.py delete mode 100644 core/dbt/node_runners.py rename core/dbt/{ui => task}/printer.py (71%) create mode 100644 core/dbt/ui.py delete mode 100644 core/dbt/ui/__init__.py delete mode 100644 core/dbt/ui/colors.py diff --git a/core/dbt/adapters/base/connections.py b/core/dbt/adapters/base/connections.py index 07088dc0083..e39d3681403 100644 --- a/core/dbt/adapters/base/connections.py +++ b/core/dbt/adapters/base/connections.py @@ -10,7 +10,6 @@ import agate import dbt.exceptions -import dbt.flags from dbt.contracts.connection import ( Connection, Identifier, ConnectionState, AdapterRequiredConfig, LazyHandle ) @@ -19,6 +18,7 @@ MacroQueryStringSetter, ) from dbt.logger import GLOBAL_LOGGER as logger +from dbt import flags class BaseConnectionManager(metaclass=abc.ABCMeta): @@ -39,7 +39,7 @@ class BaseConnectionManager(metaclass=abc.ABCMeta): def __init__(self, profile: AdapterRequiredConfig): self.profile = profile self.thread_connections: Dict[Hashable, Connection] = {} - self.lock: RLock = dbt.flags.MP_CONTEXT.RLock() + self.lock: RLock = flags.MP_CONTEXT.RLock() self.query_header: Optional[MacroQueryStringSetter] = None def set_query_header(self, manifest: Manifest) -> None: @@ -235,7 +235,7 @@ def _close_handle(cls, connection: Connection) -> None: @classmethod def _rollback(cls, connection: Connection) -> None: """Roll back the given connection.""" - if dbt.flags.STRICT_MODE: + if flags.STRICT_MODE: if not isinstance(connection, Connection): raise dbt.exceptions.CompilerException( f'In _rollback, got {connection} - not a Connection!' @@ -253,7 +253,7 @@ def _rollback(cls, connection: Connection) -> None: @classmethod def close(cls, connection: Connection) -> Connection: - if dbt.flags.STRICT_MODE: + if flags.STRICT_MODE: if not isinstance(connection, Connection): raise dbt.exceptions.CompilerException( f'In close, got {connection} - not a Connection!' diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index 6af38b5264b..ab6064bb90b 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -17,7 +17,7 @@ get_relation_returned_multiple_results, InternalException, NotImplementedException, RuntimeException, ) -import dbt.flags +from dbt import flags from dbt import deprecations from dbt.clients.agate_helper import empty_table, merge_tables, table_from_rows @@ -267,7 +267,7 @@ def load_internal_manifest(self) -> Manifest: def _schema_is_cached(self, database: Optional[str], schema: str) -> bool: """Check if the schema is cached, and by default logs if it is not.""" - if dbt.flags.USE_CACHE is False: + if flags.USE_CACHE is False: return False elif (database, schema) not in self.cache: logger.debug( @@ -323,7 +323,7 @@ def _relations_cache_for_schemas(self, manifest: Manifest) -> None: """Populate the relations cache for the given schemas. Returns an iterable of the schemas populated, as strings. """ - if not dbt.flags.USE_CACHE: + if not flags.USE_CACHE: return cache_schemas = self._get_cache_schemas(manifest) @@ -352,7 +352,7 @@ def set_relations_cache( """Run a query that gets a populated cache of the relations in the database and set the cache on this adapter. """ - if not dbt.flags.USE_CACHE: + if not flags.USE_CACHE: return with self.cache.lock: @@ -368,7 +368,7 @@ def cache_added(self, relation: Optional[BaseRelation]) -> str: raise_compiler_error( 'Attempted to cache a null relation for {}'.format(name) ) - if dbt.flags.USE_CACHE: + if flags.USE_CACHE: self.cache.add(relation) # so jinja doesn't render things return '' @@ -383,7 +383,7 @@ def cache_dropped(self, relation: Optional[BaseRelation]) -> str: raise_compiler_error( 'Attempted to drop a null relation for {}'.format(name) ) - if dbt.flags.USE_CACHE: + if flags.USE_CACHE: self.cache.drop(relation) return '' @@ -405,7 +405,7 @@ def cache_renamed( .format(src_name, dst_name, name) ) - if dbt.flags.USE_CACHE: + if flags.USE_CACHE: self.cache.rename(from_relation, to_relation) return '' diff --git a/core/dbt/adapters/sql/connections.py b/core/dbt/adapters/sql/connections.py index 3fe1dad2ec7..91ebe4e0d85 100644 --- a/core/dbt/adapters/sql/connections.py +++ b/core/dbt/adapters/sql/connections.py @@ -6,9 +6,10 @@ import dbt.clients.agate_helper import dbt.exceptions -from dbt.contracts.connection import Connection from dbt.adapters.base import BaseConnectionManager +from dbt.contracts.connection import Connection from dbt.logger import GLOBAL_LOGGER as logger +from dbt import flags class SQLConnectionManager(BaseConnectionManager): @@ -133,7 +134,7 @@ def add_commit_query(self): def begin(self): connection = self.get_thread_connection() - if dbt.flags.STRICT_MODE: + if flags.STRICT_MODE: if not isinstance(connection, Connection): raise dbt.exceptions.CompilerException( f'In begin, got {connection} - not a Connection!' @@ -151,7 +152,7 @@ def begin(self): def commit(self): connection = self.get_thread_connection() - if dbt.flags.STRICT_MODE: + if flags.STRICT_MODE: if not isinstance(connection, Connection): raise dbt.exceptions.CompilerException( f'In commit, got {connection} - not a Connection!' diff --git a/core/dbt/adapters/sql/impl.py b/core/dbt/adapters/sql/impl.py index 84268b34d16..93b7aae0906 100644 --- a/core/dbt/adapters/sql/impl.py +++ b/core/dbt/adapters/sql/impl.py @@ -4,7 +4,6 @@ import dbt.clients.agate_helper from dbt.contracts.connection import Connection import dbt.exceptions -import dbt.flags from dbt.adapters.base import BaseAdapter, available from dbt.adapters.sql import SQLConnectionManager from dbt.logger import GLOBAL_LOGGER as logger diff --git a/core/dbt/clients/jinja.py b/core/dbt/clients/jinja.py index 6138e8902c8..07a344338b7 100644 --- a/core/dbt/clients/jinja.py +++ b/core/dbt/clients/jinja.py @@ -30,7 +30,7 @@ InternalException, raise_compiler_error, CompilationException, invalid_materialization_argument, MacroReturn ) -from dbt.flags import MACRO_DEBUGGING +from dbt import flags from dbt.logger import GLOBAL_LOGGER as logger # noqa @@ -93,8 +93,8 @@ def _compile(self, source, filename): If the value is 'write', also write the files to disk. WARNING: This can write a ton of data if you aren't careful. """ - if filename == '