Merge pull request #223 from aditim1359/clifeatures

Adds new CLI features and IOPointer unflag all option

.gitattributes

@@ -0,0 +1,2 @@
+# Set the default behavior, in case people don't have core.autocrlf set.
+* text=auto

.gitignore

@@ -6,3 +6,4 @@ dist*
 *.egg-info
 *.vscode
 *.dvc*
+*.idea*

docs/make.bat

@@ -1,35 +1,35 @@
-@ECHO OFF
-
-pushd %~dp0
-
-REM Command file for Sphinx documentation
-
-if "%SPHINXBUILD%" == "" (
-	set SPHINXBUILD=sphinx-build
-)
-set SOURCEDIR=source
-set BUILDDIR=build
-
-if "%1" == "" goto help
-
-%SPHINXBUILD% >NUL 2>NUL
-if errorlevel 9009 (
-	echo.
-	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
-	echo.installed, then set the SPHINXBUILD environment variable to point
-	echo.to the full path of the 'sphinx-build' executable. Alternatively you
-	echo.may add the Sphinx directory to PATH.
-	echo.
-	echo.If you don't have Sphinx installed, grab it from
-	echo.http://sphinx-doc.org/
-	exit /b 1
-)
-
-%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-goto end
-
-:help
-%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-
-:end
-popd
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

mltrace/__init__.py

@@ -4,9 +4,7 @@
     register,
     backtrace,
     get_history,
-    get_components_with_owner,
     tag_component,
-    get_components_with_tag,
     log_component_run,
     create_random_ids,
     get_component_information,
@@ -23,6 +21,9 @@
     flag_output_id,
     unflag_output_id,
     review_flagged_outputs,
+    get_all_tags,
+    get_components,
+    unflag_all,
 )
 
 __all__ = [
@@ -31,9 +32,7 @@
     "register",
     "backtrace",
     "get_history",
-    "get_components_with_owner",
     "tag_component",
-    "get_components_with_tag",
     "log_component_run",
     "create_random_ids",
     "get_component_information",
@@ -50,4 +49,7 @@
     "flag_output_id",
     "unflag_output_id",
     "review_flagged_outputs",
+    "get_all_tags",
+    "get_components",
+    "unflag_all",
 ]

mltrace/cli/cli.py

@@ -2,6 +2,7 @@
 
 
 import click
+
 from mltrace import (
     set_address,
     get_recent_run_ids,
@@ -12,6 +13,9 @@
     flag_output_id,
     unflag_output_id,
     review_flagged_outputs,
+    get_all_tags,
+    get_components,
+    unflag_all,
 )
 import textwrap
 
@@ -204,6 +208,7 @@ def show_res(res, indent, count, pos, need_stick):
 
 @click.group()
 def mltrace():
+    # Pass
     pass
 
 
@@ -223,6 +228,20 @@ def recent(limit: int, address: str = ""):
         show_info_card(id)
 
 
+@mltrace.command("inspect")
+@click.option("--address", help="Database server address")
+@click.argument("component_run_id")
+def inspect(component_run_id, address: str = ""):
+    """
+    CLI to inspect a specific component run id.
+    """
+    # Set address
+    if address and len(address) > 0:
+        set_address(address)
+
+    show_info_card(component_run_id)
+
+
 @mltrace.command("history")
 @click.argument("component_name")
 @click.option("--limit", default=5, help="Limit of recent objects.")
@@ -274,16 +293,31 @@ def flag(output_id: str, address: str = ""):
 
 
 @mltrace.command("unflag")
-@click.argument("output_id")
+@click.option("--output_id", help="Output ID to unflag")
+@click.option("--all", is_flag=True, help="Add flag to unflag all")
 @click.option("--address", help="Database server address")
-def unflag(output_id: str, address: str = ""):
+def unflag(output_id: str = "", all: bool = False, address: str = ""):
     """
-    Command to set the flag property of an output_id to false.
+    Command to set flag property of an output_id or all output_ids to false.
     """
+    # Check if set --all and --output_id
+    if all and output_id:
+        raise click.ClickException("Can set either --all=True or specify an "
+                                   "--output_id. Cannot set both.")
+
+    if not all and not output_id:
+        raise click.ClickException("Need to either set --all=True or specify "
+                                   "an --output_id to unflag.")
+
     # Set address
     if address and len(address) > 0:
         set_address(address)
-    unflag_output_id(output_id)
+
+    if all:
+        unflag_all()
+
+    elif not all and output_id:
+        unflag_output_id(output_id)
 
 
 @mltrace.command("review")
@@ -307,3 +341,44 @@ def review(limit: int = 5, address: str = ""):
     # Print component runs
     for component, count in component_counts[:limit]:
         show_info_card(component.id, count, len(outputs))
+
+
+@mltrace.command("components")
+@click.option("--owner", help="Owner of components")
+@click.option("--tag", help="Tag of components")
+@click.option("--address", help="Database server address")
+def components(owner: str = "", tag: str = "", address: str = ""):
+    """
+    Command to list the components with options to filter by tag or owner.
+    """
+    if address and len(address) > 0:
+        set_address(address)
+
+    # Make return result
+    try:
+        result = get_components(tag, owner)
+    except RuntimeError:
+        raise click.ClickException("No components could be found with the "
+                                   "flags passed.")
+
+    # Display components, one per line
+    for comp in result:
+        click.echo(f"Name: {comp.name}")
+        click.echo()
+
+
+@mltrace.command("tags")
+@click.option("--address", help="Database server address")
+def tags(address: str = ""):
+    """
+    Command to list all the tags currently used.
+    """
+    # Set address
+    if address and len(address) > 0:
+        set_address(address)
+
+    # Get all tags, automatically unique
+    all_tags = get_all_tags()
+
+    click.echo(all_tags)
+    click.echo()

mltrace/client.py

@@ -30,6 +30,7 @@ def _set_address_helper(old_uri: str, address: str):
             Otherwise, DB_URI is set to {_db_uri}."
         )
 
+
 # --------------------- Database management functions ------------------- #
 
 
@@ -57,7 +58,7 @@ def clean_db():
 
 
 def create_component(
-    name: str, description: str, owner: str, tags: typing.List[str] = []
+        name: str, description: str, owner: str, tags: typing.List[str] = []
 ):
     """Creates a component entity in the database."""
     store = Store(_db_uri)
@@ -71,9 +72,9 @@ def tag_component(component_name: str, tags: typing.List[str]):
 
 
 def log_component_run(
-    component_run: ComponentRun,
-    set_dependencies_from_inputs=True,
-    staleness_threshold: int = (60 * 60 * 24 * 30),
+        component_run: ComponentRun,
+        set_dependencies_from_inputs=True,
+        staleness_threshold: int = (60 * 60 * 24 * 30),
 ):
     """Takes client-facing ComponentRun object and logs it to the DB."""
     store = Store(_db_uri)
@@ -143,13 +144,13 @@ def create_random_ids(num_outputs=1) -> typing.List[str]:
 
 
 def register(
-    component_name: str,
-    inputs: typing.List[str] = [],
-    outputs: typing.List[str] = [],
-    input_vars: typing.List[str] = [],
-    output_vars: typing.List[str] = [],
-    endpoint: bool = False,
-    staleness_threshold: int = (60 * 60 * 24 * 30),
+        component_name: str,
+        inputs: typing.List[str] = [],
+        outputs: typing.List[str] = [],
+        input_vars: typing.List[str] = [],
+        output_vars: typing.List[str] = [],
+        endpoint: bool = False,
+        staleness_threshold: int = (60 * 60 * 24 * 30),
 ):
     def actual_decorator(func):
         @functools.wraps(func)
@@ -322,14 +323,19 @@ def unflag_output_id(output_id: str) -> bool:
     return store.set_io_pointer_flag(output_id, False)
 
 
+def unflag_all():
+    store = Store(_db_uri)
+    store.unflag_all()
+
+
 # ----------------- Basic retrieval functions ------------------- #
 
 
 def get_history(
-    component_name: str,
-    limit: int = 10,
-    date_lower: typing.Union[datetime, str] = datetime.min,
-    date_upper: typing.Union[datetime, str] = datetime.max,
+        component_name: str,
+        limit: int = 10,
+        date_lower: typing.Union[datetime, str] = datetime.min,
+        date_upper: typing.Union[datetime, str] = datetime.max,
 ) -> typing.List[ComponentRun]:
     """Returns a list of ComponentRuns that are part of the component's
     history."""
@@ -368,23 +374,6 @@ def get_history(
     return component_runs
 
 
-def get_components_with_owner(owner: str) -> typing.List[Component]:
-    """Returns a list of all the components associated with the specified
-    order."""
-    store = Store(_db_uri)
-    res = store.get_components_with_owner(owner)
-
-    # Convert to client-facing Components
-    components = []
-    for c in res:
-        tags = [tag.name for tag in c.tags]
-        d = copy.deepcopy(c.__dict__)
-        d.update({"tags": tags})
-        components.append(Component.from_dictionary(d))
-
-    return components
-
-
 def get_component_information(component_name: str) -> Component:
     """Returns a Component with the name, info, owner, and tags."""
     store = Store(_db_uri)
@@ -421,10 +410,11 @@ def get_component_run_information(component_run_id: str) -> ComponentRun:
     return ComponentRun.from_dictionary(d)
 
 
-def get_components_with_tag(tag: str) -> typing.List[Component]:
-    """Returns a list of components with the specified tag."""
+def get_components(tag="", owner="") -> typing.List[Component]:
+    """Returns all components with the specified owner and/or tag.
+    Else, returns all components."""
     store = Store(_db_uri)
-    res = store.get_components_with_tag(tag)
+    res = store.get_components(tag=tag, owner=owner)
 
     # Convert to client-facing Components
     components = []
@@ -450,6 +440,13 @@ def get_io_pointer(io_pointer_id: str, create=True):
     return IOPointer.from_dictionary(iop.__dict__)
 
 
+def get_all_tags() -> typing.List[str]:
+    store = Store(_db_uri)
+    res = store.get_all_tags()
+    tags = [t.name for t in res]
+    return tags
+
+
 # --------------- Complex retrieval functions ------------------ #
 
 

mltrace/db/migrations/versions/a2cdf9aa818c_create_stale.py

@@ -1,7 +1,7 @@
 """create stale
 
 Revision ID: a2cdf9aa818c
-Revises: 0a8485e5ba50
+Revises: None
 Create Date: 2021-05-18 14:05:25.540236
 
 """