Merge branch 'dev/kiyoshi-kuromiya' of https://github.com/fishtown-an…

…alytics/dbt into re-partition-metadata

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.18.0b1
+current_version = 0.19.0a1
 parse = (?P<major>\d+)
 	\.(?P<minor>\d+)
 	\.(?P<patch>\d+)

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2.1
 jobs:
   unit:
     docker: &test_only
-      - image: fishtownanalytics/test-container:7
+      - image: fishtownanalytics/test-container:9
         environment:
           DBT_INVOCATION_ENV: circle
     steps:
@@ -30,7 +30,7 @@ jobs:
           destination: dist
   integration-postgres-py36:
     docker: &test_and_postgres
-      - image: fishtownanalytics/test-container:7
+      - image: fishtownanalytics/test-container:9
         environment:
           DBT_INVOCATION_ENV: circle
       - image: postgres

.gitignore

@@ -9,6 +9,7 @@ __pycache__/
 # Distribution / packaging
 .Python
 env/
+dbt_env/
 build/
 develop-eggs/
 dist/
@@ -42,6 +43,7 @@ htmlcov/
 .coverage
 .coverage.*
 .cache
+.env
 nosetests.xml
 coverage.xml
 *,cover
@@ -84,5 +86,10 @@ target/
 # pycharm
 .idea/
 
-# Visual Studio Code
+# AWS credentials
+.aws/
+
+.DS_Store
+
+# vscode
 .vscode/

Dockerfile

@@ -1,7 +1,6 @@
 FROM ubuntu:18.04
 
 ENV DEBIAN_FRONTEND noninteractive
-ARG DOCKERIZE_VERSION=v0.6.1
 
 RUN apt-get update && \
     apt-get dist-upgrade -y && \
@@ -19,20 +18,33 @@ RUN apt-get update && \
         python3.9 python3.9-dev python3.9-venv && \
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-RUN useradd -mU dbt_test_user
-RUN mkdir /usr/app && chown dbt_test_user /usr/app
-RUN mkdir /home/tox && chown dbt_test_user /home/tox
+ARG DOCKERIZE_VERSION=v0.6.1
 RUN curl -LO https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \
     tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && \
     rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
 
-WORKDIR /usr/app
-VOLUME /usr/app
-
 RUN pip3 install -U "tox==3.14.4" wheel "six>=1.14.0,<1.15.0" "virtualenv==20.0.3" setuptools
 # tox fails if the 'python' interpreter (python2) doesn't have `tox` installed
 RUN pip install -U "tox==3.14.4" "six>=1.14.0,<1.15.0" "virtualenv==20.0.3" setuptools
 
+# These args are passed in via docker-compose, which reads then from the .env file.
+# On Linux, run `make .env` to create the .env file for the current user.
+# On MacOS and Windows, these can stay unset.
+ARG USER_ID
+ARG GROUP_ID
+
+RUN if [ ${USER_ID:-0} -ne 0 ] && [ ${GROUP_ID:-0} -ne 0 ]; then \
+        groupadd -g ${GROUP_ID} dbt_test_user && \
+        useradd -m -l -u ${USER_ID} -g ${GROUP_ID} dbt_test_user; \
+    else \
+        useradd -mU -l dbt_test_user; \
+    fi
+RUN mkdir /usr/app && chown dbt_test_user /usr/app
+RUN mkdir /home/tox && chown dbt_test_user /home/tox
+
+WORKDIR /usr/app
+VOLUME /usr/app
+
 USER dbt_test_user
 
 ENV PYTHONIOENCODING=utf-8

Makefile

@@ -5,25 +5,33 @@ changed_tests := `git status --porcelain | grep '^\(M\| M\|A\| A\)' | awk '{ pri
 install:
 	pip install -e .
 
-test:
+test: .env
 	@echo "Full test run starting..."
 	@time docker-compose run test tox
 
-test-unit:
+test-unit: .env
 	@echo "Unit test run starting..."
 	@time docker-compose run test tox -e unit-py36,flake8
 
-test-integration:
+test-integration: .env
 	@echo "Integration test run starting..."
 	@time docker-compose run test tox -e integration-postgres-py36,integration-redshift-py36,integration-snowflake-py36,integration-bigquery-py36
 
-test-quick:
+test-quick: .env
 	@echo "Integration test run starting..."
 	@time docker-compose run test tox -e integration-postgres-py36 -- -x
 
+# This rule creates a file named .env that is used by docker-compose for passing
+# the USER_ID and GROUP_ID arguments to the Docker image.
+.env:
+	@echo USER_ID=$(shell id -u) > .env
+	@echo GROUP_ID=$(shell id -g) >> .env
+	@time docker-compose build
+
 clean:
 	rm -f .coverage
 	rm -rf .eggs/
+	rm -f .env
 	rm -rf .tox/
 	rm -rf build/
 	rm -rf dbt.egg-info/

README.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <img src="https://github.com/fishtown-analytics/dbt/blob/master/etc/dbt-horizontal.png?raw=true" alt="dbt logo"/>
+  <img src="/etc/dbt-logo-full.svg" alt="dbt logo" width="500"/>
 </p>
 <p align="center">
   <a href="https://codeclimate.com/github/fishtown-analytics/dbt">

azure-pipelines.yml

@@ -37,13 +37,13 @@ jobs:
       Set-Service -InputObject $serviceName -StartupType Automatic
       Start-Service -InputObject $serviceName
 
-      createdb.exe -U postgres dbt
-      psql.exe  -U postgres -c "CREATE ROLE root WITH PASSWORD 'password';"
-      psql.exe  -U postgres -c "ALTER ROLE root WITH LOGIN;"
-      psql.exe  -U postgres -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;"
-      psql.exe  -U postgres -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;"
-      psql.exe  -U postgres -c "ALTER ROLE noaccess WITH LOGIN;"
-      psql.exe  -U postgres -c "GRANT CONNECT ON DATABASE dbt TO noaccess;"
+      & $env:PGBIN\createdb.exe -U postgres dbt
+      & $env:PGBIN\psql.exe  -U postgres -c "CREATE ROLE root WITH PASSWORD 'password';"
+      & $env:PGBIN\psql.exe  -U postgres -c "ALTER ROLE root WITH LOGIN;"
+      & $env:PGBIN\psql.exe  -U postgres -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;"
+      & $env:PGBIN\psql.exe  -U postgres -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;"
+      & $env:PGBIN\psql.exe  -U postgres -c "ALTER ROLE noaccess WITH LOGIN;"
+      & $env:PGBIN\psql.exe  -U postgres -c "GRANT CONNECT ON DATABASE dbt TO noaccess;"
     displayName: Install postgresql and set up database
 
   - task: UsePythonVersion@0

core/dbt/adapters/base/connections.py

@@ -10,7 +10,6 @@
 import agate
 
 import dbt.exceptions
-import dbt.flags
 from dbt.contracts.connection import (
     Connection, Identifier, ConnectionState, AdapterRequiredConfig, LazyHandle
 )
@@ -19,6 +18,7 @@
     MacroQueryStringSetter,
 )
 from dbt.logger import GLOBAL_LOGGER as logger
+from dbt import flags
 
 
 class BaseConnectionManager(metaclass=abc.ABCMeta):
@@ -39,7 +39,7 @@ class BaseConnectionManager(metaclass=abc.ABCMeta):
     def __init__(self, profile: AdapterRequiredConfig):
         self.profile = profile
         self.thread_connections: Dict[Hashable, Connection] = {}
-        self.lock: RLock = dbt.flags.MP_CONTEXT.RLock()
+        self.lock: RLock = flags.MP_CONTEXT.RLock()
         self.query_header: Optional[MacroQueryStringSetter] = None
 
     def set_query_header(self, manifest: Manifest) -> None:
@@ -60,7 +60,7 @@ def get_thread_connection(self) -> Connection:
                 )
             return self.thread_connections[key]
 
-    def set_thread_connection(self, conn):
+    def set_thread_connection(self, conn: Connection) -> None:
         key = self.get_thread_identifier()
         if key in self.thread_connections:
             raise dbt.exceptions.InternalException(
@@ -88,6 +88,11 @@ def clear_transaction(self) -> None:
             self.begin()
             self.commit()
 
+    def rollback_if_open(self) -> None:
+        conn = self.get_if_exists()
+        if conn is not None and conn.handle and conn.transaction_open:
+            self._rollback(conn)
+
     @abc.abstractmethod
     def exception_handler(self, sql: str) -> ContextManager:
         """Create a context manager that handles exceptions caused by database
@@ -172,11 +177,9 @@ def release(self) -> None:
                 return
 
         try:
-            if conn.state == 'open':
-                if conn.transaction_open is True:
-                    self._rollback(conn)
-            else:
-                self.close(conn)
+            # always close the connection. close() calls _rollback() if there
+            # is an open transaction
+            self.close(conn)
         except Exception:
             # if rollback or close failed, remove our busted connection
             self.clear_thread_connection()
@@ -226,34 +229,34 @@ def _close_handle(cls, connection: Connection) -> None:
         """Perform the actual close operation."""
         # On windows, sometimes connection handles don't have a close() attr.
         if hasattr(connection.handle, 'close'):
-            logger.debug('On {}: Close'.format(connection.name))
+            logger.debug(f'On {connection.name}: Close')
             connection.handle.close()
         else:
-            logger.debug('On {}: No close available on handle'
-                         .format(connection.name))
+            logger.debug(f'On {connection.name}: No close available on handle')
 
     @classmethod
     def _rollback(cls, connection: Connection) -> None:
         """Roll back the given connection."""
-        if dbt.flags.STRICT_MODE:
+        if flags.STRICT_MODE:
             if not isinstance(connection, Connection):
                 raise dbt.exceptions.CompilerException(
                     f'In _rollback, got {connection} - not a Connection!'
                 )
 
         if connection.transaction_open is False:
             raise dbt.exceptions.InternalException(
-                'Tried to rollback transaction on connection "{}", but '
-                'it does not have one open!'.format(connection.name))
+                f'Tried to rollback transaction on connection '
+                f'"{connection.name}", but it does not have one open!'
+            )
 
-        logger.debug('On {}: ROLLBACK'.format(connection.name))
+        logger.debug(f'On {connection.name}: ROLLBACK')
         cls._rollback_handle(connection)
 
         connection.transaction_open = False
 
     @classmethod
     def close(cls, connection: Connection) -> Connection:
-        if dbt.flags.STRICT_MODE:
+        if flags.STRICT_MODE:
             if not isinstance(connection, Connection):
                 raise dbt.exceptions.CompilerException(
                     f'In close, got {connection} - not a Connection!'
@@ -264,6 +267,7 @@ def close(cls, connection: Connection) -> Connection:
             return connection
 
         if connection.transaction_open and connection.handle:
+            logger.debug('On {}: ROLLBACK'.format(connection.name))
             cls._rollback_handle(connection)
         connection.transaction_open = False