diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index 91db33ca..d6e6d4ab 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -3,6 +3,14 @@ organisation: moremoban releases: - changes: - action: Added + details: + - "`#322`: Implicit targets with template extensions default to copy" + - "`#257`: '-e' to load extensions for template engines, i.e. jinja2" + - "`#333`: command line template fails with version 0.6.1" + date: 15.09.2019 + version: 0.6.2 + - changes: + - action: Fixed details: - "`#328`: update backward compatibility" date: 10.09.2019 @@ -14,7 +22,7 @@ releases: - "`#185`: -v will enable moban application logging for development. And -V is for version." - "`#325`: -vv show debug trace" - "`#126`: Allow mobanfile to include data from arbitrary config files" - - "`#256`: jinja2-cli parity: '-D hello=world' to define custom variable on cli" + - "`#256`: jinja2-cli parity: '-d hello=world' to define custom variable on cli" - action: Updated details: - "`#275`: fix moban 0.4.5 test failures on openSUSE Tumbleweed" diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index b92eef9d..0f245d75 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -4,9 +4,9 @@ organisation: moremoban author: C. W. contact: wangc_2011@hotmail.com license: MIT -version: 0.6.1 -current_version: 0.6.1 -release: 0.6.1 +version: 0.6.2 +current_version: 0.6.2 +release: 0.6.2 branch: master master: index command_line_interface: "moban" diff --git a/.moban.d/setup.py b/.moban.d/moban_setup.py.jj2 similarity index 100% rename from .moban.d/setup.py rename to .moban.d/moban_setup.py.jj2 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 52be1511..e271273f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,12 +1,25 @@ Change log ================================================================================ -0.6.1 - 10.09.2019 +0.6.2 - 15.09.2019 -------------------------------------------------------------------------------- Added ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +#. `#322 `_: Implicit targets + with template extensions default to copy +#. `#257 `_: '-e' to load + extensions for template engines, i.e. jinja2 +#. `#333 `_: command line + template fails with version 0.6.1 + +0.6.1 - 10.09.2019 +-------------------------------------------------------------------------------- + +Fixed +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + #. `#328 `_: update backward compatibility @@ -24,7 +37,7 @@ Added #. `#126 `_: Allow mobanfile to include data from arbitrary config files #. `#256 `_: jinja2-cli parity: - '-D hello=world' to define custom variable on cli + '-d hello=world' to define custom variable on cli Updated ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/README.rst b/README.rst index cd4fd545..308723c9 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ ================================================================================ -moban - 模板 Yet another jinja2 cli command for static text generation +moban - 模板 Any template, any data in any location ================================================================================ .. image:: https://api.travis-ci.org/moremoban/moban.svg?branch=master @@ -17,26 +17,25 @@ moban - 模板 Yet another jinja2 cli command for static text generation .. image:: https://img.shields.io/gitter/room/gitterHQ/gitter.svg :target: https://gitter.im/chfw_moban/Lobby -:Author: C.W. +:Author: C.W. and its contributors (See contributors.rst) :Issues: http://github.com/moremoban/moban/issues :License: MIT :Version: |version| :Generated: |today| -**moban** brings the high performance template engine (JINJA2) for web into -static text generation. It is used in `pyexcel` and `coala` project to keep -documentation consistent across the documentations of individual libraries. +**moban** started with bringing the high performance template engine (JINJA2) for web +into static text generation. It has been used in `pyexcel` and `coala` project to keep +documentation consistent across the documentations of individual libraries in the same +organisation. -Our vision is: any template, any data in any location. Our current architecture -enables moban to plugin any python template engine: mako, handlebars, velocity, -haml, slim and tornado, to plugin any data format: json and yaml, and in -any location: zip, git, pypi package, s3, etc. Please -look at our issues. We have many more template engines and data format on the -road map. +**moban** can use other python template engine: mako, handlebars, velocity, +haml, slim and tornado, can read other data format: json and yaml, and can access both +template file and configuration file in +any location: zip, git, pypi package, s3, etc. -Documentation -================================================================================= +Please look at our issues. We have many more template engines and data format on the +road map. All use cases are documented `here `_ @@ -104,12 +103,10 @@ Please note that data.yml will take precedence over environment variables. Work with files in a git repo ================================================================================ -Please install `gitfs2 `_:: +`gitfs2 `_ is installed by default since v0.6.1 - $ pip install gitfs2 - -And then you can do the following: +You can do the following with moban: .. code-block:: bash @@ -127,12 +124,9 @@ And then you can do the following: Work with files in a python package ================================================================================ -Please install `pypifs `_:: - - $ pip install pypifs +`pypifs `_ is installed by default since v0.6.1 - -And then you can do the following: +You can do the following with moban: .. code-block:: bash @@ -156,6 +150,9 @@ Please install `fs-s3fs `_:: $ pip install fs-s3fs + +Then you can access your files in s3 bucket: + .. code-block:: bash $ moban -c s3://${client_id}:${client_secrect}@moremoban/s3data.yml \ diff --git a/docs/conf.py b/docs/conf.py index 21913442..c7d0042f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -25,9 +25,9 @@ copyright = '2017-2019 Onni Software Ltd.' author = 'C. W.' # The short X.Y version -version = '0.6.1' +version = '0.6.2' # The full version, including alpha/beta/rc tags -release = '0.6.1' +release = '0.6.2' # -- General configuration --------------------------------------------------- diff --git a/docs/level-15-copy-templates-as-target/.moban.yml b/docs/level-15-copy-templates-as-target/.moban.yml index 72351dd3..2771111d 100644 --- a/docs/level-15-copy-templates-as-target/.moban.yml +++ b/docs/level-15-copy-templates-as-target/.moban.yml @@ -8,6 +8,7 @@ targets: - output: target_without_template_type template: file_extension_will_trigger.copy - target_in_short_form: as_long_as_this_one_has.copy + - output_is_copied.same_file_extension: when_source_have.same_file_extension - output: "misc-1-copying/can-create-folder/if-not-exists.txt" template: file-in-template-sources-folder.txt template_type: copy diff --git a/docs/level-15-copy-templates-as-target/README.rst b/docs/level-15-copy-templates-as-target/README.rst index 2e73a082..a153a0b5 100644 --- a/docs/level-15-copy-templates-as-target/README.rst +++ b/docs/level-15-copy-templates-as-target/README.rst @@ -23,7 +23,7 @@ Shorthand syntax:: targets: - explicit: template_file.copy - + - output_is_copied.same_file_extension: when_source_have.same_file_extension No implicit nor short hand syntax for the following directory copying unless you take a look at `force-template-type`. When you read @@ -64,6 +64,7 @@ Here is example moban file for copying:: - output: target_without_template_type template: file_extension_will_trigger.copy - target_in_short_form: as_long_as_this_one_has.copy + - output_is_copied.same_file_extension: when_source_have.same_file_extension - output: "misc-1-copying/can-create-folder/if-not-exists.txt" template: file-in-template-sources-folder.txt template_type: copy diff --git a/docs/level-15-copy-templates-as-target/template-sources/when_source_have.same_file_extension b/docs/level-15-copy-templates-as-target/template-sources/when_source_have.same_file_extension new file mode 100644 index 00000000..6ee981e1 --- /dev/null +++ b/docs/level-15-copy-templates-as-target/template-sources/when_source_have.same_file_extension @@ -0,0 +1 @@ +it is implicit copy as well \ No newline at end of file diff --git a/docs/level-3-data-override/README.rst b/docs/level-3-data-override/README.rst index 66109d58..d7874c6f 100644 --- a/docs/level-3-data-override/README.rst +++ b/docs/level-3-data-override/README.rst @@ -34,12 +34,12 @@ command to launch it: New development -================================================================================ +-------------------------------------------------------------------------------- Since verison 0.6.0, `overrides` syntax support two more use cases: 1 override more than one configuration file ---------------------------------------------- +********************************************* For example:: @@ -48,7 +48,7 @@ For example:: - config-file-b.yaml 2 override more than one configuration file ---------------------------------------------- +******************************************** For example:: diff --git a/moban/_version.py b/moban/_version.py index 6f26cf69..bbb51f5d 100644 --- a/moban/_version.py +++ b/moban/_version.py @@ -1,2 +1,2 @@ -__version__ = "0.6.1" +__version__ = "0.6.2" __author__ = "C. W." diff --git a/moban/constants.py b/moban/constants.py index 8b86bb64..d726f084 100644 --- a/moban/constants.py +++ b/moban/constants.py @@ -37,7 +37,9 @@ LABEL_VERSION = "version" LABEL_GROUP = "group" LABEL_DEFINE = "define" +LABEL_EXTENSION = "extension" CLI_DICT = "user_dict" +EXTENSION_DICT = "user_extensions" DEFAULT_CONFIGURATION_DIRNAME = ".%s.cd" % PROGRAM_NAME DEFAULT_TEMPLATE_DIRNAME = ".%s.td" % PROGRAM_NAME diff --git a/moban/core/moban_factory.py b/moban/core/moban_factory.py index adc4c1b1..e430f097 100644 --- a/moban/core/moban_factory.py +++ b/moban/core/moban_factory.py @@ -1,6 +1,7 @@ import os import sys import logging +from collections import defaultdict from moban import utils, reporter, constants, exceptions, file_system from fs.errors import ResourceNotFound @@ -18,11 +19,24 @@ class MobanFactory(PluginManager): def __init__(self): super(MobanFactory, self).__init__(constants.TEMPLATE_ENGINE_EXTENSION) - self.extensions = {} + self.extensions = defaultdict(set) self.options_registry = {} def register_extensions(self, extensions): - self.extensions.update(extensions) + for user_template_type in extensions.keys(): + template_type = self.get_primary_key(user_template_type) + + log.debug( + "Registering extensions: {0}={1}".format( + user_template_type, extensions[user_template_type] + ) + ) + if template_type in self.extensions: + self.extensions[template_type] = self.extensions[ + user_template_type + ].union(extensions[user_template_type]) + else: + self.extensions[template_type] = extensions[user_template_type] def register_options(self, template_types): # need the value of 'template_types' @@ -34,6 +48,7 @@ def get_engine(self, template_type, template_dirs, context_dirs): template_dirs = utils.verify_the_existence_of_directories( template_dirs ) + if template_type in self.options_registry: custom_engine_spec = self.options_registry[template_type] engine_cls = self.load_me_now( @@ -43,7 +58,10 @@ def get_engine(self, template_type, template_dirs, context_dirs): else: engine_cls = self.load_me_now(template_type) engine_extensions = self.extensions.get(template_type) - options = dict(extensions=engine_extensions) + if engine_extensions: + options = dict(extensions=list(engine_extensions)) + else: + options = dict() template_fs = file_system.get_multi_fs(template_dirs) engine = engine_cls(template_fs, options) return MobanEngine(template_fs, context_dirs, engine) diff --git a/moban/definitions.py b/moban/definitions.py index 982e3e5d..e72f3d88 100644 --- a/moban/definitions.py +++ b/moban/definitions.py @@ -1,5 +1,9 @@ +import logging + from moban import constants +LOG = logging.getLogger(__name__) + class TemplateTarget(object): def __init__( @@ -16,6 +20,7 @@ def __init__( self.output = self.original_output self.set_template_type(template_type) + LOG.info("create a target {}".format(self)) def set_template_type(self, new_template_type): self.template_type = new_template_type diff --git a/moban/jinja2/engine.py b/moban/jinja2/engine.py index eadf6df3..87aeb76d 100644 --- a/moban/jinja2/engine.py +++ b/moban/jinja2/engine.py @@ -2,6 +2,7 @@ import logging from importlib import import_module +import fs.errors from moban import constants, file_system from jinja2 import Template, Environment from lml.loader import scan_plugins_regex @@ -11,7 +12,7 @@ from jinja2_fsloader import FSLoader JINJA2_LIBRARIES = "^moban_jinja2_.+$" -JINJA2_EXENSIONS = [ +JINJA2_EXTENSIONS = [ "moban.jinja2.filters.repr", "moban.jinja2.filters.github", "moban.jinja2.filters.text", @@ -87,7 +88,6 @@ def __init__(self, template_fs, options=None): # because it is modified here env_params["extensions"] += extensions import_module_of_extension(extensions) - env_params.update(options) self.jj2_environment = Environment(**env_params) for filter_name, filter_function in FILTERS.get_all(): @@ -117,10 +117,13 @@ def get_template(self, template_file): """ try: template = self.jj2_environment.get_template(template_file) + return template except TemplateNotFound: - content = file_system.read_unicode(template_file) - return Template(content) - return template + try: + content = file_system.read_unicode(template_file) + return self.jj2_environment.from_string(content) + except fs.errors.ResourceNotFound: + return self.jj2_environment.from_string(template_file) def get_template_from_string(self, string): return Template(string) @@ -143,7 +146,7 @@ def apply_template(self, template, data, output): def load_jinja2_extensions(): - scan_plugins_regex(JINJA2_LIBRARIES, "moban", None, JINJA2_EXENSIONS) + scan_plugins_regex(JINJA2_LIBRARIES, "moban", None, JINJA2_EXTENSIONS) def is_extension_list_valid(extensions): diff --git a/moban/main.py b/moban/main.py index d7872b1b..d8d66563 100644 --- a/moban/main.py +++ b/moban/main.py @@ -12,6 +12,7 @@ import logging import argparse import logging.config +from collections import defaultdict from moban import ( core, @@ -37,15 +38,17 @@ def main(): """ parser = create_parser() options = vars(parser.parse_args()) + handle_verbose(options[constants.LABEL_VERBOSE]) + load_engine_factory_and_engines() # Error: jinja2 if removed HASH_STORE.IGNORE_CACHE_FILE = options[constants.LABEL_FORCE] options[constants.CLI_DICT] = handle_custom_variables( options.pop(constants.LABEL_DEFINE) ) + options[constants.EXTENSION_DICT] = handle_custom_extensions( + options.pop(constants.LABEL_EXTENSION) + ) OPTIONS.update(options) - handle_verbose(options[constants.LABEL_VERBOSE]) - moban_file = options[constants.LABEL_MOBANFILE] - load_engine_factory_and_engines() # Error: jinja2 if removed if moban_file is None: moban_file = find_default_moban_file() if moban_file: @@ -155,6 +158,12 @@ def create_parser(): nargs="+", help="to take a list of VAR=VALUEs", ) + parser.add_argument( + "-e", + "--%s" % constants.LABEL_EXTENSION, + nargs="+", + help="to add an extension to TEMPLATE_TYPE=EXTENSION_NAME", + ) return parser @@ -265,6 +274,15 @@ def handle_custom_variables(list_of_definitions): return custom_data +def handle_custom_extensions(list_of_definitions): + user_extensions = defaultdict(set) + if list_of_definitions: + for definition in list_of_definitions: + key, value = definition.split("=") + user_extensions[key].add(value) + core.ENGINES.register_extensions(user_extensions) + + def handle_verbose(verbose_level): if verbose_level > len(LOG_LEVEL): verbose_level = 3 diff --git a/moban/mobanfile/templates.py b/moban/mobanfile/templates.py index 4042ac0e..893ee09d 100644 --- a/moban/mobanfile/templates.py +++ b/moban/mobanfile/templates.py @@ -1,6 +1,6 @@ import logging -from moban import reporter, file_system +from moban import reporter, constants, file_system LOG = logging.getLogger(__name__) @@ -33,6 +33,14 @@ def handle_template(template_file, output, template_dirs): yield a_triple else: template_type = _get_template_type(template_file) + # output.jj2: source.jj2 means 'copy' + if template_type and output.endswith("." + template_type): + LOG.info( + "template type switched to from {0} to {1}".format( + template_type, constants.TEMPLATE_COPY + ) + ) + template_type = constants.TEMPLATE_COPY yield (template_file, output, template_type) diff --git a/mobanfile b/mobanfile index 38d82987..e284e011 100644 --- a/mobanfile +++ b/mobanfile @@ -1,15 +1,10 @@ -requires: - - type: git - url: https://github.com/moremoban/pypi-mobans - submodule: true - branch: dev configuration: template_dir: - - "pypi-mobans:templates" + - "git://github.com/moremoban/pypi-mobans.git?submodule=true&brach=dev!/templates" - ".moban.d" configuration: moban.yml targets: - - setup.py: setup.py + - setup.py: moban_setup.py.jj2 - moban/__init__.py: __init__.py.jj2 - moban/_version.py: _version.py.jj2 - docs/conf.py: custom_conf.py.jj2 diff --git a/setup.py b/setup.py index dcbae103..1030de0f 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ NAME = "moban" AUTHOR = "C. W." -VERSION = "0.6.1" +VERSION = "0.6.2" EMAIL = "wangc_2011@hotmail.com" LICENSE = "MIT" ENTRY_POINTS = { @@ -50,7 +50,7 @@ "Yet another jinja2 cli command for static text generation" ) URL = "https://github.com/moremoban/moban" -DOWNLOAD_URL = "%s/archive/0.6.1.tar.gz" % URL +DOWNLOAD_URL = "%s/archive/0.6.2.tar.gz" % URL FILES = ["README.rst", "CONTRIBUTORS.rst", "CHANGELOG.rst"] KEYWORDS = [ "python", @@ -97,8 +97,8 @@ } # You do not need to read beyond this line PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) -GS_COMMAND = ("gs moban v0.6.1 " + - "Find 0.6.1 in changelog for more details") +GS_COMMAND = ("gs moban v0.6.2 " + + "Find 0.6.2 in changelog for more details") NO_GS_MESSAGE = ("Automatic github release is disabled. " + "Please install gease to enable it.") UPLOAD_FAILED_MSG = ( diff --git a/tests/integration_tests/test_command_line_options.py b/tests/integration_tests/test_command_line_options.py index 302b99ff..bd333cf0 100644 --- a/tests/integration_tests/test_command_line_options.py +++ b/tests/integration_tests/test_command_line_options.py @@ -3,6 +3,7 @@ from shutil import copyfile from mock import patch +from nose import SkipTest from nose.tools import eq_, raises, assert_raises from moban.definitions import TemplateTarget @@ -109,15 +110,6 @@ def tearDown(self): os.unlink(self.config_file) -@raises(Exception) -def test_missing_configuration(): - test_args = ["moban", "-t", "a.jj2"] - with patch.object(sys, "argv", test_args): - from moban.main import main - - main() - - class TestNoOptions: def setUp(self): self.config_file = ".moban.yml" @@ -476,7 +468,7 @@ def test_git_repo_example(_): main() with open("test_git_repo_example.py") as f: content = f.read() - eq_(content, '__version__ = "0.1.1rc3"\n__author__ = "C.W."') + eq_(content, '__version__ = "0.1.1rc3"\n__author__ = "C.W."\n') os.unlink("test_git_repo_example.py") @@ -497,5 +489,38 @@ def test_pypi_pkg_example(_): main() with open("test_pypi_pkg_example.py") as f: content = f.read() - eq_(content, '__version__ = "0.1.1rc3"\n__author__ = "C.W."') + eq_(content, '__version__ = "0.1.1rc3"\n__author__ = "C.W."\n') os.unlink("test_pypi_pkg_example.py") + + +def test_add_extension(): + if sys.version_info[0] == 2: + raise SkipTest("jinja2-python-version does not support python 2") + test_commands = [ + [ + "moban", + "-t", + "{{ python_version }}", + "-e", + "jinja2=jinja2_python_version.PythonVersionExtension", + ], + [ + "moban", + "-t", + "{{ python_version }}", + "-e", + "jj2=jinja2_python_version.PythonVersionExtension", + ], + ] + for test_args in test_commands: + with patch.object(sys, "argv", test_args): + from moban.main import main + + main() + with open("moban.output") as f: + content = f.read() + eq_( + content, + "{}.{}".format(sys.version_info[0], sys.version_info[1]), + ) + os.unlink("moban.output") diff --git a/tests/requirements.txt b/tests/requirements.txt index 055d0047..176572ca 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -14,3 +14,4 @@ arrow;python_version!="3.4" jinja2_time pypifs gitfs2 +jinja2-python-version diff --git a/tests/test_docs.py b/tests/test_docs.py index 2c711357..e3cc36e9 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -233,6 +233,10 @@ def test_level_15_copy_templates_as_target(self): + "so as to trigger ContentForwardEngine, 'copy' engine.\n" ), ), + ( + "output_is_copied.same_file_extension", + "it is implicit copy as well", + ), ] self.run_moban(["moban"], folder, assertions)