Merge pull request #96 from edx/rafey/dbt-upgradation

chore: dbt upgradation 1.7.10 latest
edx · Mar 28, 2024 · b8e162d · b8e162d
2 parents dc2b913 + e4a3e7a
commit b8e162d
Show file tree

Hide file tree

Showing 16 changed files with 793 additions and 589 deletions.
diff --git a/README.rst b/README.rst
@@ -47,7 +47,7 @@ How To Contribute
 -----------------
 
 Contributions are very welcome.
-Please read `How To Contribute <https://github.com/edx/edx-platform/blob/master/CONTRIBUTING.rst>`_ for details.
+Please read `The Contribution Guide <https://github.com/edx/edx-platform/blob/master/CONTRIBUTING.rst>`_ for details.
 Even though they were written with ``edx-platform`` in mind, the guidelines
 should be followed for all Open edX projects.
 
@@ -73,11 +73,11 @@ https://discuss.openedx.org where you can connect with others in the community.
 Our real-time conversations are on Slack. You can request a `Slack
 invitation`_, then join our `community Slack team`_.
 
-For more information about these options, see the `Getting Help`_ page.
+For more information about these options, see the `getting assistance`_ page.
 
 .. _Slack invitation: https://openedx-slack-invite.herokuapp.com/
 .. _community Slack team: https://openedx.slack.com/
-.. _Getting Help: https://openedx.org/getting-help
+.. _getting assistance: https://openedx.org/getting-help
 
 .. |pypi-badge| image:: https://img.shields.io/pypi/v/dbt-schema-builder.svg
     :target: https://pypi.python.org/pypi/dbt-schema-builder/

diff --git a/dbt_schema_builder/__init__.py b/dbt_schema_builder/__init__.py
@@ -2,4 +2,4 @@
 Automate management of PII redacted schemas for dbt projects.
 """
 
-__version__ = '0.4.13'
+__version__ = '0.5.0'
diff --git a/dbt_schema_builder/builder.py b/dbt_schema_builder/builder.py
@@ -9,9 +9,10 @@
 
 import dbt.utils
 import yaml
+from dbt.adapters.factory import register_adapter
 from dbt.config import RuntimeConfig
 from dbt.events import AdapterLogger
-from dbt.exceptions import DatabaseException
+from dbt.exceptions import DbtDatabaseError as DatabaseException
 from dbt.logger import log_manager
 from dbt.task.compile import CompileTask
 from dbt.task.generate import get_adapter
@@ -151,7 +152,7 @@ def run(self, source_database, schema, banned_column_names):  # pylint: disable=
         """
         # Check for any non-word characters that might indicate a SQL injection attack
         if re.search("[^a-zA-Z0-9_]", schema):
-            raise Exception(
+            raise Exception(  # pylint: disable=broad-exception-raised
                 "Non-word character in schema name '{}'! Possible SQL injection?".format(
                     schema
                 )
@@ -584,12 +585,13 @@ class SchemaBuilderTask:
     def __init__(self, args):
         self.args = args
         self.config = RuntimeConfig.from_args(args)
+        register_adapter(self.config)
         self.source_project_path, self.destination_project_path = self.get_project_dirs()
         self.builder = SchemaBuilder(
             self.config.model_paths[0],
             self.source_project_path,
             self.destination_project_path,
-            GetCatalogTask(self.args, self.config)
+            GetCatalogTask(self.args, self.config, None)
         )
 
     def get_project_dirs(self):
@@ -603,7 +605,7 @@ def get_project_dirs(self):
 
         for project_path in [source_project_path, destination_project_path]:
             if not os.path.exists(os.path.join(project_path, "dbt_project.yml")):
-                raise Exception(
+                raise Exception(  # pylint: disable=broad-exception-raised
                     "fatal: {} is not a dbt project. Does not exist or is missing a "
                     "dbt_project.yml file.".format(project_path)
                 )

diff --git a/dbt_schema_builder/schema_builder.py b/dbt_schema_builder/schema_builder.py
@@ -6,10 +6,12 @@
 import sys
 
 from dbt import flags
-from dbt.flags import PROFILES_DIR
+from dbt.flags import get_flag_dict
 
 from .builder import SchemaBuilderTask
 
+PROFILES_DIR = get_flag_dict().get('PROFILES_DIR')
+
 
 def parse_args(args):
     """
@@ -53,6 +55,12 @@ def parse_args(args):
         type=str,
         help="Which target to load for the given profile",
     )
+    base_subparser.add_argument(
+        "--threads",
+        default=None,
+        type=int,
+        help="Number of threads for dbt to run.",
+    )
 
     group = base_subparser.add_mutually_exclusive_group()
 
@@ -77,7 +85,7 @@ def parse_args(args):
         parents=[base_subparser],
         help="Creates or updates schema.yml files from database catalog",
     )
-    build_sub.set_defaults(cls=SchemaBuilderTask, which="build", defer=None, state=None)
+    build_sub.set_defaults(cls=SchemaBuilderTask, which="build", defer=None, state=None, defer_state=None)
 
     build_sub.add_argument(
         "--destination-project",

diff --git a/requirements/base.in b/requirements/base.in
@@ -1,6 +1,5 @@
 # Core requirements for using this application
 -c constraints.txt
 
-dbt-core==1.3.0
-dbt-snowflake==1.3.0
-
+dbt-core==1.7.10
+dbt-snowflake==1.7.2
diff --git a/requirements/base.txt b/requirements/base.txt
@@ -1,157 +1,191 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
 #
 #    make upgrade
 #
-agate==1.6.3
-    # via dbt-core
-asn1crypto==1.4.0
+agate==1.7.1
     # via
-    #   oscrypto
-    #   snowflake-connector-python
-attrs==21.4.0
-    # via jsonschema
-babel==2.9.1
+    #   dbt-core
+    #   dbt-snowflake
+annotated-types==0.6.0
+    # via pydantic
+asn1crypto==1.5.1
+    # via snowflake-connector-python
+attrs==23.2.0
+    # via
+    #   jsonschema
+    #   referencing
+babel==2.14.0
     # via agate
-certifi==2021.10.8
+certifi==2024.2.2
     # via
     #   requests
     #   snowflake-connector-python
-cffi==1.15.0
+cffi==1.16.0
     # via
     #   cryptography
     #   dbt-core
     #   snowflake-connector-python
-charset-normalizer==2.0.12
+charset-normalizer==3.3.2
     # via
     #   requests
     #   snowflake-connector-python
-click==8.0.4
-    # via dbt-core
-colorama==0.4.4
+click==8.1.7
+    # via
+    #   dbt-core
+    #   dbt-semantic-interfaces
+colorama==0.4.6
     # via dbt-core
-cryptography==3.4.8
+cryptography==42.0.5
     # via
-    #   dbt-snowflake
     #   pyopenssl
     #   snowflake-connector-python
-dbt-core==1.0.3
+dbt-core==1.7.10
     # via
     #   -r requirements/base.in
     #   dbt-snowflake
-dbt-extractor==0.4.0
+dbt-extractor==0.5.1
     # via dbt-core
-dbt-snowflake==1.0.0
-    # via -r requirements/base.in
-future==0.18.2
-    # via parsedatetime
-hologram==0.0.14
+dbt-semantic-interfaces==0.4.4
     # via dbt-core
-idna==3.3
+dbt-snowflake==1.7.2
+    # via -r requirements/base.in
+filelock==3.13.1
+    # via snowflake-connector-python
+idna==3.6
     # via
     #   dbt-core
     #   requests
     #   snowflake-connector-python
-importlib-metadata==4.11.2
+importlib-metadata==6.11.0
+    # via
+    #   dbt-semantic-interfaces
+    #   keyring
+importlib-resources==6.3.0
     # via
     #   jsonschema
+    #   jsonschema-specifications
     #   keyring
 isodate==0.6.1
     # via
     #   agate
     #   dbt-core
-jeepney==0.7.1
+jaraco-classes==3.3.1
+    # via keyring
+jinja2==3.1.3
     # via
-    #   keyring
-    #   secretstorage
-jinja2==2.11.3
-    # via dbt-core
-jsonschema==3.1.1
-    # via hologram
-keyring==23.5.0
+    #   dbt-core
+    #   dbt-semantic-interfaces
+jsonschema==4.21.1
+    # via
+    #   dbt-core
+    #   dbt-semantic-interfaces
+jsonschema-specifications==2023.12.1
+    # via jsonschema
+keyring==24.3.1
     # via snowflake-connector-python
-leather==0.3.4
+leather==0.4.0
     # via agate
 logbook==1.5.3
     # via dbt-core
-markupsafe==2.0.1
-    # via
-    #   dbt-core
-    #   jinja2
-mashumaro==2.9
+markupsafe==2.1.5
+    # via jinja2
+mashumaro[msgpack]==3.12
     # via dbt-core
 minimal-snowplow-tracker==0.0.2
     # via dbt-core
-msgpack==1.0.3
+more-itertools==10.2.0
+    # via
+    #   dbt-semantic-interfaces
+    #   jaraco-classes
+msgpack==1.0.8
     # via mashumaro
-networkx==2.7
+networkx==3.1
     # via dbt-core
-oscrypto==1.2.1
-    # via snowflake-connector-python
-packaging==20.9
+packaging==23.1
     # via
     #   -c requirements/constraints.txt
     #   dbt-core
-parsedatetime==2.4
+    #   snowflake-connector-python
+parsedatetime==2.6
     # via agate
+pathspec==0.11.2
+    # via dbt-core
+pkgutil-resolve-name==1.3.10
+    # via jsonschema
+platformdirs==3.11.0
+    # via snowflake-connector-python
+protobuf==4.25.3
+    # via dbt-core
 pycparser==2.21
     # via cffi
-pycryptodomex==3.14.1
+pydantic==2.6.4
+    # via dbt-semantic-interfaces
+pydantic-core==2.16.3
+    # via pydantic
+pyjwt==2.8.0
     # via snowflake-connector-python
-pyjwt==2.3.0
+pyopenssl==24.1.0
     # via snowflake-connector-python
-pyopenssl==21.0.0
-    # via snowflake-connector-python
-pyparsing==3.0.7
-    # via packaging
-pyrsistent==0.18.1
-    # via jsonschema
-python-dateutil==2.8.2
-    # via hologram
-python-slugify==6.1.1
+python-dateutil==2.9.0.post0
+    # via dbt-semantic-interfaces
+python-slugify==8.0.4
     # via agate
 pytimeparse==1.1.8
     # via agate
-pytz==2021.3
+pytz==2024.1
     # via
     #   babel
+    #   dbt-core
     #   snowflake-connector-python
-pyyaml==6.0
-    # via mashumaro
-requests==2.27.1
+pyyaml==6.0.1
+    # via
+    #   dbt-core
+    #   dbt-semantic-interfaces
+referencing==0.33.0
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+requests==2.31.0
     # via
     #   dbt-core
-    #   dbt-snowflake
     #   minimal-snowplow-tracker
     #   snowflake-connector-python
-secretstorage==3.3.1
-    # via keyring
+rpds-py==0.18.0
+    # via
+    #   jsonschema
+    #   referencing
 six==1.16.0
     # via
-    #   agate
     #   isodate
-    #   jsonschema
-    #   leather
     #   minimal-snowplow-tracker
-    #   pyopenssl
     #   python-dateutil
-snowflake-connector-python[secure-local-storage]==2.7.4
+snowflake-connector-python[secure-local-storage]==3.7.1
     # via dbt-snowflake
-sqlparse==0.4.2
+sortedcontainers==2.4.0
+    # via snowflake-connector-python
+sqlparse==0.4.4
     # via dbt-core
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==3.10.0.2
+tomlkit==0.12.4
+    # via snowflake-connector-python
+typing-extensions==4.10.0
     # via
+    #   annotated-types
     #   dbt-core
+    #   dbt-semantic-interfaces
     #   mashumaro
-urllib3==1.26.8
-    # via requests
-werkzeug==2.0.3
-    # via dbt-core
-zipp==3.7.0
-    # via importlib-metadata
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
+    #   pydantic
+    #   pydantic-core
+    #   snowflake-connector-python
+urllib3==1.26.18
+    # via
+    #   dbt-core
+    #   requests
+    #   snowflake-connector-python
+zipp==3.18.0
+    # via
+    #   importlib-metadata
+    #   importlib-resources