Merge pull request #111 from simleo/update_cli

Update CLI

rocrate/cli.py

@@ -23,17 +23,39 @@
 from .model.computerlanguage import LANG_MAP
 from .model.testservice import SERVICE_MAP
 from .model.softwareapplication import APP_MAP
+from .utils import is_url
 
 
 LANG_CHOICES = list(LANG_MAP)
 SERVICE_CHOICES = list(SERVICE_MAP)
 ENGINE_CHOICES = list(APP_MAP)
 
 
+def add_hash(id_):
+    if id_ is None or id_.startswith("#") or is_url(id_):
+        return id_
+    return "#" + id_
+
+
 class State:
     pass
 
 
+class CSVParamType(click.ParamType):
+    name = "csv"
+
+    def convert(self, value, param, ctx):
+        if isinstance(value, (list, tuple, set, frozenset)):
+            return value
+        try:
+            return value.split(",") if value else []
+        except AttributeError:
+            self.fail(f"{value!r} is not splittable", param, ctx)
+
+
+CSV = CSVParamType()
+
+
 @click.group()
 @click.option('-c', '--crate-dir', type=click.Path())
 @click.pass_context
@@ -44,9 +66,10 @@ def cli(ctx, crate_dir):
 
 @cli.command()
 @click.option('--gen-preview', is_flag=True)
+@click.option('-e', '--exclude', type=CSV)
 @click.pass_obj
-def init(state, gen_preview):
-    crate = ROCrate(state.crate_dir, init=True, gen_preview=gen_preview)
+def init(state, gen_preview, exclude):
+    crate = ROCrate(state.crate_dir, init=True, gen_preview=gen_preview, exclude=exclude)
     crate.metadata.write(state.crate_dir)
     if crate.preview:
         crate.preview.write(state.crate_dir)
@@ -80,7 +103,7 @@ def workflow(state, path, language):
 @click.option('-m', '--main-entity')
 @click.pass_obj
 def suite(state, identifier, name, main_entity):
-    suite_ = state.crate.add_test_suite(identifier=identifier, name=name, main_entity=main_entity)
+    suite_ = state.crate.add_test_suite(identifier=add_hash(identifier), name=name, main_entity=main_entity)
     state.crate.metadata.write(state.crate_dir)
     print(suite_.id)
 
@@ -94,7 +117,10 @@ def suite(state, identifier, name, main_entity):
 @click.option('-n', '--name')
 @click.pass_obj
 def instance(state, suite, url, resource, service, identifier, name):
-    instance_ = state.crate.add_test_instance(suite, url, resource=resource, service=service, identifier=identifier, name=name)
+    instance_ = state.crate.add_test_instance(
+        add_hash(suite), url, resource=resource, service=service,
+        identifier=add_hash(identifier), name=name
+    )
     state.crate.metadata.write(state.crate_dir)
     print(instance_.id)
 

rocrate/rocrate.py

@@ -19,7 +19,6 @@
 
 import errno
 import json
-import os
 import uuid
 import zipfile
 import atexit
@@ -47,7 +46,7 @@
 from .model.softwareapplication import SoftwareApplication, get_app, PLANEMO_DEFAULT_VERSION
 from .model.testsuite import TestSuite
 
-from .utils import is_url, subclasses, get_norm_value
+from .utils import is_url, subclasses, get_norm_value, walk
 
 
 def read_metadata(metadata_path):
@@ -81,7 +80,8 @@ def pick_type(json_entity, type_map, fallback=None):
 
 class ROCrate():
 
-    def __init__(self, source=None, gen_preview=False, init=False):
+    def __init__(self, source=None, gen_preview=False, init=False, exclude=None):
+        self.exclude = exclude
         self.__entity_map = {}
         self.default_entities = []
         self.data_entities = []
@@ -108,7 +108,7 @@ def __init_from_tree(self, top_dir, gen_preview=False):
         if not top_dir.is_dir():
             raise NotADirectoryError(errno.ENOTDIR, f"'{top_dir}': not a directory")
         self.add(RootDataset(self), Metadata(self))
-        for root, dirs, files in os.walk(top_dir):
+        for root, dirs, files in walk(top_dir, exclude=self.exclude):
             root = Path(root)
             for name in dirs:
                 source = root / name
@@ -453,7 +453,7 @@ def delete(self, *entities):
         # fetch all files defined in the crate
 
     def _copy_unlisted(self, top, base_path):
-        for root, dirs, files in os.walk(top):
+        for root, dirs, files in walk(top, exclude=self.exclude):
             root = Path(root)
             for name in dirs:
                 source = root / name

rocrate/utils.py

@@ -18,6 +18,7 @@
 # limitations under the License.
 
 import collections
+import os
 from datetime import datetime, timezone
 from urllib.parse import urlsplit
 
@@ -78,3 +79,12 @@ def get_norm_value(json_entity, prop):
         return [_ if isinstance(_, str) else _["@id"] for _ in value]
     except (TypeError, KeyError):
         raise ValueError(f"Malformed value for {prop!r}: {json_entity.get(prop)!r}")
+
+
+def walk(top, topdown=True, onerror=None, followlinks=False, exclude=None):
+    exclude = frozenset(exclude or [])
+    for root, dirs, files in os.walk(top):
+        if exclude:
+            dirs[:] = [_ for _ in dirs if _ not in exclude]
+            files[:] = [_ for _ in files if _ not in exclude]
+        yield root, dirs, files

test/test_cli.py

@@ -20,6 +20,7 @@
 from click.testing import CliRunner
 
 from rocrate.cli import cli
+from rocrate.model.file import File
 from rocrate.model.metadata import TESTING_EXTRA_TERMS
 from rocrate.rocrate import ROCrate
 
@@ -53,6 +54,22 @@ def test_cli_init(test_data_dir, helpers, monkeypatch, cwd, gen_preview):
     assert json_entities["sort-and-change-case.ga"]["@type"] == "File"
 
 
+def test_cli_init_exclude(test_data_dir, helpers):
+    crate_dir = test_data_dir / "ro-crate-galaxy-sortchangecase"
+    (crate_dir / helpers.METADATA_FILE_NAME).unlink()
+    exclude = "test,README.md"
+    runner = CliRunner()
+    args = ["-c", str(crate_dir), "init", "-e", exclude]
+    assert runner.invoke(cli, args).exit_code == 0
+    crate = ROCrate(crate_dir)
+    for p in "LICENSE", "sort-and-change-case.ga":
+        assert isinstance(crate.dereference(p), File)
+    for p in exclude.split(",") + ["test/"]:
+        assert not crate.dereference(p)
+    for e in crate.data_entities:
+        assert not(e.id.startswith("test"))
+
+
 @pytest.mark.parametrize("cwd", [False, True])
 def test_cli_add_workflow(test_data_dir, helpers, monkeypatch, cwd):
     # init
@@ -94,7 +111,7 @@ def test_cli_add_test_metadata(test_data_dir, helpers, monkeypatch, cwd):
     assert json_entities[def_id]["@type"] == "File"
     # add workflow
     wf_path = crate_dir / "sort-and-change-case.ga"
-    runner.invoke(cli, ["-c", str(crate_dir), "add", "workflow", "-l", "galaxy", str(wf_path)]).exit_code == 0
+    assert runner.invoke(cli, ["-c", str(crate_dir), "add", "workflow", "-l", "galaxy", str(wf_path)]).exit_code == 0
     # add test suite
     result = runner.invoke(cli, ["-c", str(crate_dir), "add", "test-suite"])
     assert result.exit_code == 0
@@ -133,6 +150,32 @@ def test_cli_add_test_metadata(test_data_dir, helpers, monkeypatch, cwd):
     assert set(TESTING_EXTRA_TERMS.items()).issubset(extra_terms.items())
 
 
+@pytest.mark.parametrize("hash_", [False, True])
+def test_cli_add_test_metadata_explicit_ids(test_data_dir, helpers, monkeypatch, hash_):
+    crate_dir = test_data_dir / "ro-crate-galaxy-sortchangecase"
+    runner = CliRunner()
+    assert runner.invoke(cli, ["-c", str(crate_dir), "init"]).exit_code == 0
+    wf_path = crate_dir / "sort-and-change-case.ga"
+    assert runner.invoke(cli, ["-c", str(crate_dir), "add", "workflow", "-l", "galaxy", str(wf_path)]).exit_code == 0
+    suite_id = "#foo"
+    cli_suite_id = suite_id if hash_ else suite_id[1:]
+    result = runner.invoke(cli, ["-c", str(crate_dir), "add", "test-suite", "-i", cli_suite_id])
+    assert result.exit_code == 0
+    assert result.output.strip() == suite_id
+    json_entities = helpers.read_json_entities(crate_dir)
+    assert suite_id in json_entities
+    instance_id = "#bar"
+    cli_instance_id = instance_id if hash_ else instance_id[1:]
+    result = runner.invoke(
+        cli, ["-c", str(crate_dir), "add", "test-instance", cli_suite_id,
+              "http://example.com", "-r", "jobs", "-i", cli_instance_id]
+    )
+    assert result.exit_code == 0
+    assert result.output.strip() == instance_id
+    json_entities = helpers.read_json_entities(crate_dir)
+    assert instance_id in json_entities
+
+
 @pytest.mark.parametrize("cwd", [False, True])
 def test_cli_write_zip(test_data_dir, monkeypatch, cwd):
     crate_dir = test_data_dir / "ro-crate-galaxy-sortchangecase"

test/test_read.py

@@ -224,6 +224,29 @@ def test_init(test_data_dir, tmpdir, helpers, override):
             assert f1.read() == f2.read()
 
 
+def test_exclude(test_data_dir, tmpdir, helpers):
+    def check(out=False):
+        for p in "LICENSE", "sort-and-change-case.ga":
+            assert isinstance(crate.dereference(p), File)
+        for p in exclude + ["test/"]:
+            assert not crate.dereference(p)
+            if out:
+                assert not(crate.source / p).exists()
+        for e in crate.data_entities:
+            assert not(e.id.startswith("test"))
+        if out:
+            assert not(crate.source / "test").exists()
+    crate_dir = test_data_dir / "ro-crate-galaxy-sortchangecase"
+    (crate_dir / helpers.METADATA_FILE_NAME).unlink()
+    exclude = ["test", "README.md"]
+    crate = ROCrate(crate_dir, init=True, exclude=exclude)
+    check()
+    out_path = tmpdir / 'ro_crate_out'
+    crate.write(out_path)
+    crate = ROCrate(out_path)
+    check(out=True)
+
+
 @pytest.mark.parametrize("gen_preview,preview_exists", [(False, False), (False, True), (True, False), (True, True)])
 def test_init_preview(test_data_dir, tmpdir, helpers, gen_preview, preview_exists):
     crate_dir = test_data_dir / "ro-crate-galaxy-sortchangecase"