Ct 177/add precommit hooks

.flake8

@@ -0,0 +1,12 @@
+[flake8]
+select =
+    E
+    W
+    F
+ignore =
+    W503 # makes Flake8 work like black
+    W504
+    E203 # makes Flake8 work like black
+    E741
+max-line-length = 99
+exclude = test

.github/workflows/main.yml

@@ -37,19 +37,10 @@ defaults:
 
 jobs:
   code-quality:
-    name: ${{ matrix.toxenv }}
+    name: code-quality
 
     runs-on: ubuntu-latest
 
-    strategy:
-      fail-fast: false
-      matrix:
-        toxenv: [flake8]
-
-    env:
-      TOXENV: ${{ matrix.toxenv }}
-      PYTEST_ADDOPTS: "-v --color=yes"
-
     steps:
       - name: Check out the repository
         uses: actions/checkout@v2
@@ -62,12 +53,16 @@ jobs:
       - name: Install python dependencies
         run: |
           pip install --user --upgrade pip
-          pip install tox
+          pip install pre-commit
+          pip install mypy==0.782
+          pip install -r dev_requirements.txt
           pip --version
-          tox --version
+          pre-commit --version
+          mypy --version
+          dbt --version
 
-      - name: Run tox
-        run: tox
+      - name: Run pre-commit hooks
+        run: pre-commit run --all-files --show-diff-on-failure
 
   unit:
     name: unit test / python ${{ matrix.python-version }}

.gitignore

@@ -49,9 +49,7 @@ coverage.xml
 *,cover
 .hypothesis/
 test.env
-
-# Mypy
-.mypy_cache/
+*.pytest_cache/
 
 # Translations
 *.mo
@@ -66,10 +64,10 @@ docs/_build/
 # PyBuilder
 target/
 
-#Ipython Notebook
+# Ipython Notebook
 .ipynb_checkpoints
 
-#Emacs
+# Emacs
 *~
 
 # Sublime Text
@@ -78,6 +76,7 @@ target/
 # Vim
 *.sw*
 
+# Pyenv
 .python-version
 
 # Vim
@@ -90,6 +89,7 @@ venv/
 # AWS credentials
 .aws/
 
+# MacOS
 .DS_Store
 
 # vscode

.pre-commit-config.yaml

@@ -0,0 +1,64 @@
+# For more on configuring pre-commit hooks (see https://pre-commit.com/)
+
+# TODO: remove global exclusion of tests when testing overhaul is complete
+exclude: '^tests/.*'
+
+# Force all unspecified python hooks to run python 3.8
+default_language_version:
+  python: python3.8
+
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v3.2.0
+  hooks:
+  - id: check-yaml
+    args: [--unsafe]
+  - id: check-json
+  - id: end-of-file-fixer
+  - id: trailing-whitespace
+  - id: check-case-conflict
+- repo: https://github.com/psf/black
+  rev: 21.12b0
+  hooks:
+  - id: black
+    args:
+    - "--line-length=99"
+    - "--target-version=py38"
+  - id: black
+    alias: black-check
+    stages: [manual]
+    args:
+    - "--line-length=99"
+    - "--target-version=py38"
+    - "--check"
+    - "--diff"
+- repo: https://gitlab.com/pycqa/flake8
+  rev: 4.0.1
+  hooks:
+  - id: flake8
+  - id: flake8
+    alias: flake8-check
+    stages: [manual]
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v0.782
+  hooks:
+  - id: mypy
+    # N.B.: Mypy is... a bit fragile.
+    #
+    # By using `language: system` we run this hook in the local
+    # environment instead of a pre-commit isolated one.  This is needed
+    # to ensure mypy correctly parses the project.
+
+    # It may cause trouble in that it adds environmental variables out
+    # of our control to the mix.  Unfortunately, there's nothing we can
+    # do about per pre-commit's author.
+    # See https://github.com/pre-commit/pre-commit/issues/730 for details.
+    args: [--show-error-codes, --ignore-missing-imports]
+    files: ^dbt/adapters/.*
+    language: system
+  - id: mypy
+    alias: mypy-check
+    stages: [manual]
+    args: [--show-error-codes, --pretty, --ignore-missing-imports]
+    files: ^dbt/adapters
+    language: system

Makefile

@@ -0,0 +1,58 @@
+.DEFAULT_GOAL:=help
+
+.PHONY: dev
+dev: ## Installs adapter in develop mode along with development depedencies
+	@\
+	pip install -r dev_requirements.txt && pre-commit install
+
+.PHONY: mypy
+mypy: ## Runs mypy against staged changes for static type checking.
+	@\
+	pre-commit run --hook-stage manual mypy-check | grep -v "INFO"
+
+.PHONY: flake8
+flake8: ## Runs flake8 against staged changes to enforce style guide.
+	@\
+	pre-commit run --hook-stage manual flake8-check | grep -v "INFO"
+
+.PHONY: black
+black: ## Runs black  against staged changes to enforce style guide.
+	@\
+	pre-commit run --hook-stage manual black-check -v | grep -v "INFO"
+
+
+.PHONY: lint
+lint: ## Runs flake8 and mypy code checks against staged changes.
+	@\
+	pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \
+	pre-commit run mypy-check --hook-stage manual | grep -v "INFO"
+
+
+.PHONY: unit
+unit: ## Runs unit tests with py38.
+	@\
+	tox -e py38
+
+.PHONY: test
+test: ## Runs unit tests with py38 and code checks against staged changes.
+	@\
+	tox -p -e py38; \
+	pre-commit run black-check --hook-stage manual | grep -v "INFO"; \
+	pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \
+	pre-commit run mypy-check --hook-stage manual | grep -v "INFO"
+
+.PHONY: integration
+integration: ## Runs snowflake integration tests with py38.
+	@\
+	tox -e py38-snowflake --
+
+.PHONY: clean
+	@echo "cleaning repo"
+	@git clean -f -X
+
+.PHONY: help
+help: ## Show this help message.
+	@echo 'usage: make [target]'
+	@echo
+	@echo 'targets:'
+	@grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

dbt/adapters/snowflake/__init__.py

@@ -4,8 +4,8 @@
 from dbt.adapters.snowflake.relation import SnowflakeRelation  # noqa
 from dbt.adapters.snowflake.impl import SnowflakeAdapter
 
-from dbt.adapters.base import AdapterPlugin
-from dbt.include import snowflake
+from dbt.adapters.base import AdapterPlugin  # type: ignore
+from dbt.include import snowflake  # type: ignore
 
 Plugin = AdapterPlugin(
     adapter=SnowflakeAdapter,

dbt/adapters/snowflake/connections.py

@@ -18,10 +18,10 @@
     InternalException, RuntimeException, FailedToConnectException,
     DatabaseException, warn_or_error
 )
-from dbt.adapters.base import Credentials
+from dbt.adapters.base import Credentials  # type: ignore
 from dbt.contracts.connection import AdapterResponse
-from dbt.adapters.sql import SQLConnectionManager
-from dbt.events import AdapterLogger
+from dbt.adapters.sql import SQLConnectionManager  # type: ignore
+from dbt.events import AdapterLogger  # type: ignore
 
 
 logger = AdapterLogger("Snowflake")

dbt/adapters/snowflake/impl.py

@@ -4,7 +4,7 @@
 import agate
 
 from dbt.adapters.base.impl import AdapterConfig
-from dbt.adapters.sql import SQLAdapter
+from dbt.adapters.sql import SQLAdapter  # type: ignore
 from dbt.adapters.sql.impl import (
     LIST_SCHEMAS_MACRO_NAME,
     LIST_RELATIONS_MACRO_NAME,

dev_requirements.txt

@@ -2,13 +2,15 @@
 # TODO: how to automate switching from develop to version branches?
 git+https://github.com/dbt-labs/dbt.git#egg=dbt-core&subdirectory=core
 
+black==21.12b0
 bumpversion
 flake8
 flaky
 freezegun==0.3.12
 ipdb
 mypy==0.782
 pip-tools
+pre-commit
 pytest
 pytest-dotenv
 pytest-logbook

tox.ini

@@ -1,15 +1,6 @@
 [tox]
 skipsdist = True
-envlist = py37,py38,py39,flake8
-
-[testenv:flake8]
-description = flake8 code checks
-basepython = python3.8
-skip_install = true
-commands = flake8 --select=E,W,F --ignore=W504,E741 --max-line-length 99 \
-  dbt
-deps =
-  -rdev_requirements.txt
+envlist = py37,py38,py39
 
 [testenv:{unit,py37,py38,py39,py}]
 description = unit testing