Add --ephemeral flag option to knockoff run command and unit tests

.gitignore

@@ -65,4 +65,8 @@ target/
 .idea/
 
 # pyenv
-.python-version
+.python-version
+
+# jupyter
+*.ipynb_checkpoints/
+

CHANGELOG.md

@@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file.
 `knockoff-factory` adheres to [Semantic Versioning](http://semver.org/).
 
 #### 4.x Releases
-- `4.1.x` Releases - [4.1.0](#410) | [4.1.1](#411)
+- `4.1.x` Releases - [4.1.0](#410) | [4.2.0](#420)
 - `4.0.x` Releases - [4.0.0](#400)
 
 #### 3.x Releases
@@ -41,17 +41,22 @@ All notable changes to this project will be documented in this file.
 
 ---
 
-## 4.1.1
+## 4.2.0
 
 #### Added
+- Add --ephemeral flag for `knockoff run` CLI to create temp database for loading knockoff configuration from sdk
+    - Added by [Gregory Yu](https://github.com/gregyu) in Pull Request [#4](https://github.com/Nike-Inc/knockoff-factory/pull/4)
+- Add unit tests for KnockoffDB.build and `knockoff run` CLI
+    - Added by [Gregory Yu](https://github.com/gregyu) in Pull Request [#4](https://github.com/Nike-Inc/knockoff-factory/pull/4)
 - Documentation and jupyter notebook for TempDatabaseService
-    - Added by [Gregory Yu](https://github.com/gregyu) in Pull Request [#3](https://github.com/Nike-Inc/knockoff-factory/pull/3)
+    - Added by [Gregory Yu](https://github.com/gregyu) in Pull Request [#4](https://github.com/Nike-Inc/knockoff-factory/pull/4)
 - Documentation and jupyter notebook for KnockoffDB
-    - Added by [Gregory Yu](https://github.com/gregyu) in Pull Request [#3](https://github.com/Nike-Inc/knockoff-factory/pull/3)
+    - Added by [Gregory Yu](https://github.com/gregyu) in Pull Request [#4](https://github.com/Nike-Inc/knockoff-factory/pull/4)
+
 
 #### Updated
 - Moved legacy YAML based knockoff cli from README.md to legacy.MD 
-    - Added by [Gregory Yu](https://github.com/gregyu) in Pull Request [#3](https://github.com/Nike-Inc/knockoff-factory/pull/3)
+    - Added by [Gregory Yu](https://github.com/gregyu) in Pull Request [#4](https://github.com/Nike-Inc/knockoff-factory/pull/4)
 
 #### Fixed
 

knockoff/cli_v2.py

@@ -39,11 +39,11 @@ def usage(self):
                     command=key, description=description[key]
                 ))
             except KeyError:
-                raise Exception(f"Missing usage description for {key}")
+                raise TypeError(f"Missing usage description for {key}")
 
         return "\n".join(_usage)
 
-    def __init__(self):
+    def __init__(self, argv=None):
         parser = argparse.ArgumentParser(
             description="knockoff cli",
             usage=self.usage
@@ -54,9 +54,10 @@ def __init__(self):
                             metavar="COMMAND")
         # parse_args defaults to [1:] for args, but you need to
         # exclude the rest of the args too, or validation will fail
-        args = parser.parse_args(sys.argv[1:2])
+        argv = argv or sys.argv
+        args = parser.parse_args(argv[1:2])
         command = self.get_resource(args.command)
-        command(sys.argv[2:])
+        command(argv[2:])
 
 
 def setup_logger(verbose=False):
@@ -70,10 +71,10 @@ def setup_logger(verbose=False):
                         level=level)
 
 
-def main():
+def main(argv=None):
     setup_logger()
-    KnockoffCLI()
+    KnockoffCLI(argv=argv)
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(main())

knockoff/command/run.py

@@ -6,32 +6,49 @@
 
 
 import argparse
-import inspect
 import sys
 import logging
 
-from dependency_injector import containers
-
-from knockoff.utilities.importlib_utils import resolve_package_name
+from knockoff.utilities.ioc import get_container
 from knockoff.sdk.db import KnockoffDB, DefaultDatabaseService
-
+from knockoff.sdk.blueprint import Blueprint
+from knockoff.tempdb.db import TempDatabaseService
 
 logger = logging.getLogger(__name__)
 
 
+def testable_input(prompt=None, **kwargs):
+    """pass through input so this can be unit tested"""
+    input(prompt)
+
+
 def run(knockoff_db: KnockoffDB,
-        blueprint):
+        blueprint: Blueprint,
+        temp_db: TempDatabaseService = None):
+
+    if temp_db:
+        temp_url = temp_db.start()
+        logger.info("TempDatabaseService created temp database:\n"
+                    f"{temp_url}")
+
     dfs, knockoff_db = blueprint.construct(knockoff_db)
     knockoff_db.insert()
-    logger.info("knockoff done.")
-
+    logger.info("knockoff data successfully loaded into database.")
+
+    if temp_db:
+        try:
+            testable_input(
+                "Press Enter when finished to destroy temp database.",
+                # the following kwargs can be used by a mock for assertions
+                test_temp_url=temp_url,
+                test_temp_db=temp_db,
+                test_blueprint=blueprint,
+                test_knockoff_db=knockoff_db,
+            )
+        finally:
+            temp_db.stop()
 
-def _validate_container_class(cls, package_name):
-    if not inspect.isclass(cls) or not issubclass(cls, containers.DeclarativeContainer):
-        raise TypeError(f"{package_name} resolves to "
-                         f"{cls} instead of "
-                         f"a subclass of dependency_injector"
-                         f".containers.DeclarativeContainer")
+    logger.info("knockoff done.")
 
 
 def seed(i):
@@ -55,33 +72,33 @@ def parse_args(argv=None):
                         help="Default KnockoffContainer")
     parser.add_argument("--yaml-config",
                         help="Container configuration")
+    parser.add_argument("--ephemeral",
+                        action="store_true",
+                        help="flag to run interactively with an ephemeral database")
+    parser.add_argument("--tempdb-container",
+                        default="knockoff.tempdb.container:TempDBContainer",
+                        help="Default TempDBContainer")
     parser.add_argument("-s", "--seed", type=int,
                         help="Set seed")
     return parser.parse_args(argv)
 
 
 def main(argv=None):
-    """
-    TODO:
-            - Add ephemeral flag for spinning up temporary database that's destroyed
-              after program completion (with interactive program)
-    """
     args = parse_args(argv)
 
-    KnockoffContainer = resolve_package_name(args.container)
-    _validate_container_class(KnockoffContainer, args.container)
-    container = KnockoffContainer()
-    container.init_resources()
-    if args.yaml_config:
-        container.config.from_yaml(args.yaml_config)
-    container.wire(modules=[sys.modules[__name__]])
-
     if args.seed:
         seed(args.seed)
 
+    container = get_container(args.container, args.yaml_config)
     knockoff_db = container.knockoff_db()
     blueprint = container.blueprint()
-    run(knockoff_db, blueprint)
+    temp_db = None
+
+    if args.ephemeral:
+        tempdb_container = get_container(args.tempdb_container, args.yaml_config)
+        temp_db = tempdb_container.temp_db()
+
+    run(knockoff_db, blueprint, temp_db=temp_db)
 
 
 if __name__ == "__main__":

knockoff/sdk/constraints.py

@@ -14,12 +14,12 @@ class KnockoffConstraint(six.with_metaclass(ABCMeta, object)):
     @abstractmethod
     def reset(self):
         """reset to initial state"""
-        return
+        return  # pragma: no cover
 
     @abstractmethod
     def check(self, record):
         """if record would satisfy constraint return True"""
-        return
+        return  # pragma: no cover
 
     def add(self, record):
         if not self.check(record):

knockoff/sdk/container/default.py

@@ -11,6 +11,7 @@
 from knockoff.sdk.db import KnockoffDB, DefaultDatabaseService
 from knockoff.sdk.blueprint import Blueprint
 
+
 class KnockoffContainer(containers.DeclarativeContainer):
     config = providers.Configuration()
 

knockoff/sdk/db.py

@@ -24,19 +24,19 @@
 class KnockoffDatabaseService(six.with_metaclass(ABCMeta, object)):
     @abstractmethod
     def reflect_table(self, name):
-        return
+        return  # pragma: no cover
 
     @abstractmethod
     def insert(self, name, df, dtype=None):
-        return
+        return  # pragma: no cover
 
     @abstractmethod
     def reflect_unique_constraints(self, name):
-        return
+        return  # pragma: no cover
 
     @abstractmethod
     def has_table(self, name):
-        return
+        return  # pragma: no cover
 
 
 class DefaultDatabaseService(KnockoffDatabaseService):

knockoff/tempdb/container.py

@@ -0,0 +1,38 @@
+# Copyright 2021-present, Nike, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the Apache-2.0 license found in
+# the LICENSE file in the root directory of this source tree.
+
+
+from dependency_injector import containers, providers
+
+from knockoff.utilities.importlib_utils import resolve_package_name
+from knockoff.tempdb.db import TempDatabaseService
+from knockoff.tempdb.initialize_tables import SqlAlchemyInitTablesFunc
+
+
+class TempDBContainer(containers.DeclarativeContainer):
+    config = providers.Configuration()
+
+    setup_teardown = providers.Factory(
+        resolve_package_name,
+        config.tempdb.setup_teardown.package
+    )
+
+    base = providers.Factory(
+        resolve_package_name,
+        config.tempdb.initialize_tables.base.package
+    )
+
+    initialize_tables = providers.Factory(
+        SqlAlchemyInitTablesFunc,
+        base
+    )
+
+    temp_db = providers.Factory(
+        TempDatabaseService,
+        url=config.tempdb.url,
+        setup_teardown=setup_teardown,
+        initialize_tables=initialize_tables,
+    )

knockoff/utilities/ioc.py

@@ -0,0 +1,32 @@
+# Copyright 2021-present, Nike, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the Apache-2.0 license found in
+# the LICENSE file in the root directory of this source tree.
+
+
+import inspect
+import sys
+
+from dependency_injector import containers
+from knockoff.utilities.importlib_utils import resolve_package_name
+
+
+def get_container(package_name, config_path=None):
+    """parses declarative container, loads optional config, wires and returns"""
+    Container = resolve_package_name(package_name)
+    _validate_container_class(Container, package_name)
+    container = Container()
+    container.init_resources()
+    if config_path:
+        container.config.from_yaml(config_path)
+    container.wire(modules=[sys.modules[__name__]])
+    return container
+
+
+def _validate_container_class(cls, package_name):
+    if not inspect.isclass(cls) or not issubclass(cls, containers.DeclarativeContainer):
+        raise TypeError(f"{package_name} resolves to "
+                         f"{cls} instead of "
+                         f"a subclass of dependency_injector"
+                         f".containers.DeclarativeContainer")

knockoff/utilities/regex.py

@@ -28,12 +28,12 @@ class RegexParser(six.with_metaclass(ABCMeta, object)):
 
     @abstractproperty2to3
     def compiled_regex(self):
-        raise NotImplementedError
+        raise NotImplementedError  # pragma: no cover
 
     @classmethod
     @abstractmethod
     def make(cls, **kwargs):
-        return
+        return  # pragma: no cover
 
     @classmethod
     def parse(cls, string):

tests/knockoff/command/knockoff.yaml

@@ -0,0 +1,21 @@
+# Copyright 2021-present, Nike, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the Apache-2.0 license found in
+# the LICENSE file in the root directory of this source tree.
+
+
+database_service:
+  url: postgresql://postgres@localhost:5432/postgres
+
+blueprint:
+  plan:
+    package: knockoff.sdk.blueprint:noplan
+
+tempdb:
+  url: postgresql://postgres@localhost:5432/postgres
+  setup_teardown:
+    package: knockoff.tempdb.setup_teardown:postgres_setup_teardown
+  initialize_tables:
+    base:
+      package: tests.knockoff.data_model:Base