From dec192f2683b8588110619a6c78c0cc690ec01d1 Mon Sep 17 00:00:00 2001 From: jaska Date: Thu, 21 May 2020 23:01:42 +0100 Subject: [PATCH] Delete generated files by moban (#381) * :fire: remove unwanted error message * :sparkles: add delete support. resolves #167 * :books: update change log * :newspaper: add missing file --- .moban.cd/changelog.yml | 6 +++ CHANGELOG.rst | 8 ++++ docs/README.rst | 6 ++- docs/index.rst | 1 + .../.moban.yaml | 11 +++++ .../README.rst | 42 +++++++++++++++++++ .../data.yml | 1 + .../original.jj2 | 1 + moban/constants.py | 1 + moban/core/definitions.py | 13 ------ moban/core/moban_factory.py | 27 +++++++----- moban/core/mobanfile/store.py | 14 ++++++- moban/core/mobanfile/targets.py | 17 ++++---- moban/core/mobanfile/templates.py | 23 +++++----- moban/core/plugins.py | 1 + moban/externals/file_system.py | 3 -- moban/externals/reporter.py | 17 +++++--- moban/plugins/delete.py | 36 ++++++++++++++++ tests/test_definitions.py | 10 +---- tests/test_docs.py | 10 +++++ 20 files changed, 187 insertions(+), 61 deletions(-) create mode 100644 docs/level-25-delete-intermediate-targets/.moban.yaml create mode 100644 docs/level-25-delete-intermediate-targets/README.rst create mode 100644 docs/level-25-delete-intermediate-targets/data.yml create mode 100644 docs/level-25-delete-intermediate-targets/original.jj2 create mode 100644 moban/plugins/delete.py diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index 8dfd9a32..016b00b2 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -1,6 +1,12 @@ name: moban organisation: moremoban releases: + - changes: + - action: Added + details: + - "`#167`: reverse what moban have done: delete" + date: 21.5.2020 + version: 0.7.5 - changes: - action: Fixed details: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 429fdccf..9f894aac 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,14 @@ Change log ================================================================================ +0.7.5 - 21.5.2020 +-------------------------------------------------------------------------------- + +**Added** + +#. `#167 `_: reverse what moban + have done: delete + 0.7.4 - 13.5.2020 -------------------------------------------------------------------------------- diff --git a/docs/README.rst b/docs/README.rst index 483590b5..f8854309 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -24,9 +24,10 @@ This section covers the use cases for moban. Please check them out individually. #. `Select a group target to run`_ #. `Template files in a zip or tar`_ #. `Template copying from a zip to a zip`_ -#. `Intermeidate targets`_ +#. `Intermediate targets`_ #. `Mobanfile inheritance`_ #. `Files over http(s)`_ +#. `Remove intermediate targets`_ .. _Jinja2 command line: level-1-jinja2-cli .. _Template inheritance: level-2-template-inheritance @@ -49,6 +50,7 @@ This section covers the use cases for moban. Please check them out individually. .. _Select a group target to run: level-19-moban-a-sub-group-in-targets .. _Template files in a zip or tar: level-20-templates-configs-in-zip-or-tar .. _Template copying from a zip to a zip: level-21-copy-templates-into-an-alien-file-system -.. _Intermeidate targets: level-22-intermediate-targets +.. _Intermediate targets: level-22-intermediate-targets .. _Mobanfile inheritance: level-23-inherit-organisational-moban-file .. _Files over http(s): level-24-files-over-http +.. _Remove intermediate targets: level-25-delete-intermediate diff --git a/docs/index.rst b/docs/index.rst index 1d420d57..e622e103 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -39,6 +39,7 @@ examples folder. level-22-intermediate-targets/README.rst level-23-inherit-organisational-moban-file/README.rst level-24-files-over-http/README.rst + level-25-delete-intermediate/README.rst For more complex use case, please look at `its usage in pyexcel project `_ diff --git a/docs/level-25-delete-intermediate-targets/.moban.yaml b/docs/level-25-delete-intermediate-targets/.moban.yaml new file mode 100644 index 00000000..a0033e76 --- /dev/null +++ b/docs/level-25-delete-intermediate-targets/.moban.yaml @@ -0,0 +1,11 @@ +targets: + - intermediate.jj2: original.jj2 + - intermediate2.jj2: original.jj2 + - intermediate3.jj2: original.jj2 + - final: intermediate.jj2 + - output: what_ever_here_will_be_ignored + template: intermediate.jj2 + template_type: delete + - output: '' + template: intermediate2.jj2 + - delete!: intermediate3.jj2 diff --git a/docs/level-25-delete-intermediate-targets/README.rst b/docs/level-25-delete-intermediate-targets/README.rst new file mode 100644 index 00000000..19583e9b --- /dev/null +++ b/docs/level-25-delete-intermediate-targets/README.rst @@ -0,0 +1,42 @@ +Level 25: delete intermediate targets +================================================================================ + +Continue with level 22, we would like to delete intermediate files. + +.. note:: + + What is intermediate targets? Simply they are the files moban generates + but in the end those files are not really used. + + +For safety reasons, we only delete intermediate targets. We are not allowing +moban to delete any files in template folders and staic folder. + +Here is the short syntax:: + + targets: + - delete!: intermediate_file.jj2 + +Here are the full syntax:: + + targets: + - output: what_ever_here_will_be_ignored + template: intermediate.jj2 + template_type: delete + - output: '' + template: intermediate2.jj2 + + +Example mobanfile:: + + targets: + - intermediate.jj2: original.jj2 + - intermediate2.jj2: original.jj2 + - intermediate3.jj2: original.jj2 + - output: x + template: intermediate.jj2 + template_type: delete + - output: '' + template: intermediate2.jj2 + - delete!: intermediate3.jj2 + diff --git a/docs/level-25-delete-intermediate-targets/data.yml b/docs/level-25-delete-intermediate-targets/data.yml new file mode 100644 index 00000000..bb56b055 --- /dev/null +++ b/docs/level-25-delete-intermediate-targets/data.yml @@ -0,0 +1 @@ +hello: world diff --git a/docs/level-25-delete-intermediate-targets/original.jj2 b/docs/level-25-delete-intermediate-targets/original.jj2 new file mode 100644 index 00000000..8b08a03e --- /dev/null +++ b/docs/level-25-delete-intermediate-targets/original.jj2 @@ -0,0 +1 @@ +a {{hello}} diff --git a/moban/constants.py b/moban/constants.py index 530ab7f1..d03a167f 100644 --- a/moban/constants.py +++ b/moban/constants.py @@ -3,6 +3,7 @@ # Template type TEMPLATE_JINJA2 = "jinja2" TEMPLATE_COPY = "copy" +TEMPLATE_DELETE = "delete" # Configurations PROGRAM_NAME = "moban" diff --git a/moban/core/definitions.py b/moban/core/definitions.py index d81fd2e0..e72f3d88 100644 --- a/moban/core/definitions.py +++ b/moban/core/definitions.py @@ -46,16 +46,3 @@ def __repr__(self): self.output, self.template_type, ) - - -class Store: - def __init__(self): - self.init() - - def add(self, target): - self.targets.append(target) - self.look_up_by_output[target.output] = target - - def init(self): - self.targets = [] - self.look_up_by_output = {} diff --git a/moban/core/moban_factory.py b/moban/core/moban_factory.py index e80013f5..f56b65cf 100644 --- a/moban/core/moban_factory.py +++ b/moban/core/moban_factory.py @@ -219,17 +219,24 @@ def _render_with_finding_data_first(self, data_file_index): data = self.context.get_data(data_file) for (template_file, output) in template_output_pairs: template = self.engine.get_template(template_file) - template_abs_path = self.template_fs.geturl( - template_file, purpose="fs" - ) - flag = self.apply_template( - template_abs_path, template, data, output - ) - if flag: - reporter.report_templating( - self.engine_action, template_file, output + if isinstance(template, bool): + if template: + reporter.report_templating( + self.engine_action, template_file, None + ) + self.templated_count += 1 + else: + template_abs_path = self.template_fs.geturl( + template_file, purpose="fs" ) - self.templated_count += 1 + flag = self.apply_template( + template_abs_path, template, data, output + ) + if flag: + reporter.report_templating( + self.engine_action, template_file, output + ) + self.templated_count += 1 self.file_count += 1 diff --git a/moban/core/mobanfile/store.py b/moban/core/mobanfile/store.py index eda34174..8b1adee9 100644 --- a/moban/core/mobanfile/store.py +++ b/moban/core/mobanfile/store.py @@ -1,3 +1,15 @@ -from moban.core.definitions import Store +class Store: + def __init__(self): + self.init() + + def add(self, target): + self.targets.append(target) + self.look_up_by_output[target.output] = target + + def init(self): + self.targets = [] + self.look_up_by_output = {} + self.intermediate_targets = [] + STORE = Store() diff --git a/moban/core/mobanfile/targets.py b/moban/core/mobanfile/targets.py index c496cf9e..c475fb13 100644 --- a/moban/core/mobanfile/targets.py +++ b/moban/core/mobanfile/targets.py @@ -55,7 +55,12 @@ def parse_targets(options, targets): STORE.add(template_target) else: for output, template_file in target.items(): - if isinstance(template_file, str) is False: + if isinstance(template_file, str): + for template_target in _handle_implicit_target( + options, template_file, output + ): + STORE.add(template_target) + else: # grouping by template type feature group_template_type = output a_list_short_hand_targets = template_file @@ -63,11 +68,6 @@ def parse_targets(options, targets): options, a_list_short_hand_targets, group_template_type ): STORE.add(template_target) - else: - for template_target in _handle_implicit_target( - options, template_file, output - ): - STORE.add(template_target) def _handle_explicit_target(options, target): @@ -78,7 +78,10 @@ def _handle_explicit_target(options, target): ) data_file = target.get(constants.LABEL_CONFIG, common_data_file) output = target[constants.LABEL_OUTPUT] - template_type = target.get(constants.LABEL_TEMPLATE_TYPE) + if output: + template_type = target.get(constants.LABEL_TEMPLATE_TYPE) + else: + template_type = constants.TEMPLATE_DELETE if template_type and len(template_type) > 0: if constants.TEMPLATE_TYPES_FILE_EXTENSIONS in template_type: reporter.report_file_extension_not_needed() diff --git a/moban/core/mobanfile/templates.py b/moban/core/mobanfile/templates.py index f3cd55ac..a60c99c9 100644 --- a/moban/core/mobanfile/templates.py +++ b/moban/core/mobanfile/templates.py @@ -20,22 +20,22 @@ def handle_template(template_file, output, template_dirs): fs, source_dir, output ) else: - if STORE.look_up_by_output.get(template_file) is None: - reporter.report_error_message( - f"{template_file} cannot be found" - ) + reporter.report_error_message(f"{template_file} cannot be found") else: - _, fs = multi_fs.which(template_file) - if fs is None: - if STORE.look_up_by_output.get(template_file) is None: + if STORE.look_up_by_output.get(template_file) is None: + _, fs = multi_fs.which(template_file) + if fs is None: reporter.report_error_message( f"{template_file} cannot be found" ) + elif fs.isdir(template_file): + yield from _list_dir_files(fs, template_file, output) else: yield _create_a_single_target(template_file, output) - elif fs.isdir(template_file): - yield from _list_dir_files(fs, template_file, output) else: + # when template_file is not found, it means + it_is_generated_by_moban = template_file + STORE.intermediate_targets.append(it_is_generated_by_moban) yield _create_a_single_target(template_file, output) @@ -66,7 +66,10 @@ def _listing_directory_files_recusively(fs, source, dest): def _create_a_single_target(template_file, output): - template_type = _get_template_type(template_file) + if output == constants.TEMPLATE_DELETE + "!": + template_type = constants.TEMPLATE_DELETE + else: + template_type = _get_template_type(template_file) # output.jj2: source.jj2 means 'copy' if template_type and output.endswith("." + template_type): LOG.info( diff --git a/moban/core/plugins.py b/moban/core/plugins.py index 486c64cc..a19be56a 100644 --- a/moban/core/plugins.py +++ b/moban/core/plugins.py @@ -7,6 +7,7 @@ "moban.plugins.yaml_loader", "moban.plugins.json_loader", "moban.plugins.copy", + "moban.plugins.delete", ] diff --git a/moban/externals/file_system.py b/moban/externals/file_system.py index 6639f98c..65d214eb 100644 --- a/moban/externals/file_system.py +++ b/moban/externals/file_system.py @@ -36,11 +36,8 @@ def wrapper(*args, **kwds): try: return function_in_this_module(*args, **kwds) except fs.errors.CreateFailed: - from moban.externals import reporter - message = "Failed to open %s" % args[0] LOG.debug(message) - reporter.report_error_message(message) raise exceptions.FileNotFound(args[0]) except fs.opener.errors.UnsupportedProtocol as e: LOG.exception(e) diff --git a/moban/externals/reporter.py b/moban/externals/reporter.py index fe06345d..aef91d89 100644 --- a/moban/externals/reporter.py +++ b/moban/externals/reporter.py @@ -17,13 +17,18 @@ def report_templating( action_in_present_continuous_tense, source_file, destination_file ): - do_print( - MESSAGE_TEMPLATING.format( - action_in_present_continuous_tense, - crayons.yellow(source_file), - crayons.green(destination_file), + if destination_file: + do_print( + MESSAGE_TEMPLATING.format( + action_in_present_continuous_tense, + crayons.yellow(source_file), + crayons.green(destination_file), + ) + ) + else: + do_print( + f"{action_in_present_continuous_tense} {crayons.yellow(source_file)}" ) - ) def report_no_action(): diff --git a/moban/plugins/delete.py b/moban/plugins/delete.py new file mode 100644 index 00000000..c750cf97 --- /dev/null +++ b/moban/plugins/delete.py @@ -0,0 +1,36 @@ +import fs +from lml.plugin import PluginInfo + +from moban import constants +from moban.core.mobanfile.store import STORE + + +@PluginInfo(constants.TEMPLATE_ENGINE_EXTENSION, tags=["delete"]) +class ContentForwardEngine(object): + """ + Does no templating but delete generated intermediate targets + + """ + + ACTION_IN_PRESENT_CONTINUOUS_TENSE = "Deleting" + ACTION_IN_PAST_TENSE = "Deleted" + + def __init__(self, template_fs, extensions=None): + self.template_fs = template_fs + + def get_template(self, template_file): + if template_file in STORE.intermediate_targets: + with fs.open_fs(".") as the_fs: + if the_fs.exists(template_file): + the_fs.remove(template_file) + return True + else: + return False + else: + raise Exception(f"Cannot remove {template_file}") + + def get_template_from_string(self, string): + raise NotImplementedError("Not sure what to do") + + def apply_template(self, template, *_): + raise NotImplementedError("Not sure what to do") diff --git a/tests/test_definitions.py b/tests/test_definitions.py index 8992ca38..a9a6a9d5 100644 --- a/tests/test_definitions.py +++ b/tests/test_definitions.py @@ -1,7 +1,7 @@ from nose.tools import eq_ from moban.deprecated import GitRequire -from moban.core.definitions import Store, TemplateTarget +from moban.core.definitions import TemplateTarget def test_git_require_repr(): @@ -43,11 +43,3 @@ def test_branch_params(): actual = require.clone_params() expected = {"single_branch": True, "branch": "ghpages", "depth": 2} eq_(expected, actual) - - -def test_store(): - store = Store() - output = "output" - target = TemplateTarget("template_file", "data_file", output) - store.add(target) - eq_(target, store.look_up_by_output.get(output)) diff --git a/tests/test_docs.py b/tests/test_docs.py index bbfd821b..ee4b9482 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -321,6 +321,16 @@ def test_level_22_intermediate_targets(self): folder = "level-22-intermediate-targets" self.run_moban(["moban"], folder, [("final", expected)]) + assert os.path.exists("intermediate.jj2") + + def test_level_25_delete_intermediate_targets(self): + expected = "a world\n" + + folder = "level-25-delete-intermediate-targets" + self.run_moban(["moban"], folder, [("final", expected)]) + assert not os.path.exists("intermediate.jj2") + assert not os.path.exists("intermediate2.jj2") + assert not os.path.exists("intermediate3.jj2") def test_level_23_inherit_parent_moban_file(self): folder = "level-23-inherit-organisational-moban-file"