diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..50a8726 --- /dev/null +++ b/.gitignore @@ -0,0 +1,51 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +bin/ +build/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Rope +.ropeproject + +# Django stuff: +*.log +*.pot + +# Sphinx documentation +docs/_build/ + diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..35990e8 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,35 @@ +# This file is a template, and might need editing before it works on your project. +# Official language image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/plone/tags/ + +# Change pip's cache directory to be inside the project directory since we can +# only cache local items. +variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache" + +# Pip's cache doesn't store the python packages +# https://pip.pypa.io/en/stable/reference/pip_install/#caching +# +# If you want to also cache the installed packages, you have to install +# them in a virtualenv and cache it as well. +cache: + paths: + - .cache/pip + - venv/ + - downloads/ + - eggs/ + - builds/ + +# Set execution order: first run jobs on 'test' stage on parallel +# then run jobs on 'report' stage +stages: + - test + +before_script: + - pip install -U tox + +test: + image: python3.6 + stage: test + script: + - tox -e py{36} \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..393c47b --- /dev/null +++ b/.travis.yml @@ -0,0 +1,6 @@ +language: python +python: + - 3.6 +install: + - bin/buildout -t 3 +script: bin/test diff --git a/CHANGES.txt b/CHANGES.txt new file mode 100644 index 0000000..dfcec5f --- /dev/null +++ b/CHANGES.txt @@ -0,0 +1,60 @@ +0.7 (unreleased) +================ + + - Fix test for python3 version + [aormazabal] + - Add support for c.r.backup + [mamico] + - Document virtual mount points. [jean] + - Disallow cross-database references finally (see https://dev.plone.org/ticket/11150) + [jean] + +0.6 (2010-08-13) +================ + + - Support use of the recipe when the ZEO server and client are in different + buildouts. + [ramon] + +0.5 (2010-02-03) +================ + + - Support plone.recipe.zeoserver in addition to plone.recipe.zope2zeoserver. + [davisagli] + +0.4 (2009-12-30) +================ + + - Add support for creating blob storages associated with the created + filestorage parts. + [davisagli] + + - Fixed message indicating which parts the filestorage part must be installed + before. This fixes https://bugs.launchpad.net/bugs/433877 (Thanks, Jean + Jordaan) + [davisagli] + +0.3 (2009-03-19) +================ + + - Avoid causing the installation of all parts, even those not listed in the + [buildout] parts variable. Thanks to Dylan Jay for the diagnosis and fix. + [davisagli] + + - Fixed restructured text of long description (README.txt). + [maurits] + +0.2 (2008-04-30) +================ + + - Added zodb_container_class option. + [davisagli] + +0.1 (2008-04-22) +================ + + - Happy Earth Day! + [davisagli] + + - Created recipe with ZopeSkel. Initial implementation. + [davisagli] diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt new file mode 100644 index 0000000..ac0e9e5 --- /dev/null +++ b/CONTRIBUTORS.txt @@ -0,0 +1,7 @@ + - David Glick [davisagli], Author + +Thanks to: + + - Dylan Jay + - Ramon Navarro Bosch + - Albert Ormazabal diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..d2a219b --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,8 @@ +include *.txt +include *.rst +include *.md + +recursive-include docs * +recursive-include plone * + +global-exclude *pyc diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..d6c2ce9 --- /dev/null +++ b/README.rst @@ -0,0 +1,306 @@ +Introduction +============ + +This recipe adds additional ``filestorage`` and ``zodb_db`` stanzas to the ``zope.conf`` +and ``zeo.conf`` files generated by the ``plone.recipe.zope2instance`` and +``plone.recipe.[zope2]zeoserver`` recipes. It also creates the directories in which +the extra filestorage data files will be created. This makes it easy to add +additional filestorages and mountpoints to a Zope 5 instance. + + +Supported options +================= + +parts + A list of filestorage sub-parts to be generated, one per line. (This is + different from buildout parts.) +zeo + The name of a ``plone.recipe.zope2zeoserver`` or ``plone.recipe.zeoserver`` part to + which we want to add the extra filestorage. Defaults to the first such part + in the buildout, if any. +zopes + A list of names of ``plone.recipe.zope2instance`` parts to which we want to add + the extra filestorage. Defaults to all ``plone.recipe.zope2instance`` parts + connected to the associated ``zeoserver`` part, if any, or all + ``plone.recipe.zope2instance`` parts, if no ZEO is found. +backup + The name of a ``collective.recipe.backup`` (>= 2.7) part to which we want + to add extra filestorage/blobstorage backup entries. + +The following options affect the generated ``zope.conf`` and ``zeo.conf``. Each may be +specified for all filestorage subparts in the ``plone.recipe.filestorage`` +buildout part, or for one particular filestorage subpart by placing the option +in a new buildout part called ``filestorage_subpart``, where *subpart* is the name +of the subpart listed in the ``parts`` option of this recipe. The name of the +subpart may be interpolated by using ``%(fs_part_name)s`` in the option. + +allow-implicit-cross-references + Allow references across mounted databases. This is generally a bad idea. + Defaults to ``False``. +location + The location of the ``Data.fs`` file, relative to the buildout root directory. + Defaults to ``var/filestorage/%(fs_part_name)s/Data.fs`` +zodb-name + The name of the ZODB. Defaults to ``%(fs_part_name)s``. +zodb-cache-size + Set the ZODB cache size, i.e. the number of objects which the ZODB cache + will try to hold. Inherits from the associated Zope part. Defaults to + 5000. +zodb-mountpoint + Set the path to the mountpoint. Defaults to ``/%(fs_part_name)s``. + If the object is being mounted at a different path than its location + in the source database, two paths may be provided using the + ``virtualpath:realpath`` syntax. For example, when mounting ``/source/bar`` + from ``source.fs`` at ``/foo/bar`` in ``target.fs``: + ``/foo/bar:/%(fs_part_name)s/bar``. **Note**: the name of the mount point + and the mounted object must be the same, i.e. ``/foo:/bar`` will fail. +zodb-container-class + Set the class of the object being mounted. Defaults to not being set. +zeo-address + Set the port of the associated ZEO server. Inherits from the associated + Zope and ZEO parts. Defaults to 8100. +zeo-client-cache-size + Set the size of the ZEO client cache. Inherits from the associated Zope + part. Defaults to '30MB'. +zeo-storage + Set the id of the ZEO storage. Defaults to ``%(fs_part_name)s``. +zeo-client-name + Set the name of the ZEO client. Defaults to ``%(fs_part_name)s_zeostorage``. +zeo-client-client + Set the persistent cache name that is used to construct the cache + filenames. Persistent cache files are disabled by default. +blob-storage + Set the directory to be used to store blobs for a standalone Zope instance + or a ZEO server. Optional. Required if you're going to store blobs, though. + Recommended value: ``var/blobstorage-%(fs_part_name)s`` +zeo-blob-storage + Set the directory to be used to store blobs for a ZEO client. Defaults to + using the same value as ``blob-storage``. +zeo-shared-blob-dir + Boolean that should be 'on' if the blob dir is being shared by the ZEO + server and client. Defaults to 'on'. + + +Example usage +============= + +Here's a minimal buildout that adds an extra filestorage:: + + [buildout] + extends = base.cfg + parts = + filestorage + instance + + [instance] + recipe = plone.recipe.zope2instance + user = me + + [filestorage] + recipe = plone.recipe.filestorage + parts = + mystorage + +By default, the location of the new filestorage will be: +``var/filestorage/mystorage/mystorage.fs`` + +See above for options to override the defaults. + +A setting can be modified just for one particular filestorage, by creating +a new part with the ``filestorage_`` prefix, like so:: + + [filestorage] + recipe = plone.recipe.filestorage + parts = + myfirststorage + mysecondstorage + + [filestorage_mysecondstorage] + zodb-cache-size = 1000 + +By default, the recipe adds the extra filestorages to each +``plone.recipe.zope2instance`` part in the buildout, +but you can tell it to only add it to certain parts:: + + [buildout] + extends = base.cfg + parts = + filestorage + instance1 + instance2 + + [instance1] + recipe = plone.recipe.zope2instance + + [instance2] + recipe = plone.recipe.zope2instance + + [filestorage] + recipe = plone.recipe.filestorage + zopes = instance1 + parts = + my-fs + +Here is a minimal buildout including a ZEO server and two ZODB clients:: + + [buildout] + extends = base.cfg + parts = + filestorage + zeoserver + primary + secondary + + [zeoserver] + recipe = plone.recipe.zope2zeoserver + + [primary] + recipe = plone.recipe.zope2instance + zeo-client = on + + [secondary] + recipe = plone.recipe.zope2instance + zeo-client = on + + [filestorage] + recipe = plone.recipe.filestorage + parts = + my-fs + +As above, we can override a number of the default parameters:: + + [buildout] + extends = base.cfg + parts = + filestorage + zeoserver + primary + secondary + + [zeoserver] + recipe = plone.recipe.zope2zeoserver + + [primary] + recipe = plone.recipe.zope2instance + zeo-client = on + + [secondary] + recipe = plone.recipe.zope2instance + zeo-client = on + + [filestorage] + recipe = plone.recipe.filestorage + location = var/filestorage/%(fs_part_name)s/Data.fs + blob-storage = var/blobstorage-%(fs_part_name)s + zodb-cache-size = 1000 + zodb-name = %(fs_part_name)s_db + zodb-mountpoint = /%(fs_part_name)s_mountpoint + zeo-address = 8101 + zeo-client-cache-size = 50MB + zeo-storage = %(fs_part_name)s_storage + zeo-client-name = %(fs_part_name)s_zeostorage_name + parts = + my-fs + +By default, the recipe adds the extra filestorages to the first +``plone.recipe.zope2zeoserver`` part in the buildout, and will throw an error if +there is more than one part using this recipe. However, you can override this +behavior by specifying a particular ZEO part. In this case, the filestorages +will only be added to the Zopes using that ZEO, by default:: + + [buildout] + extends = base.cfg + parts = + filestorage + zeoserver1 + zeoserver2 + primary + secondary + other-zope + + [zeoserver1] + recipe = plone.recipe.zope2zeoserver + zeo-address = 8100 + + [zeoserver2] + recipe = plone.recipe.zope2zeoserver + zeo-address = 8101 + + [primary] + recipe = plone.recipe.zope2instance + zeo-client = 1 + zeo-address = 8101 + + [secondary] + recipe = plone.recipe.zope2instance + zeo-client = 1 + zeo-address = 8101 + + [other-zope] + recipe = plone.recipe.zope2instance + zeo-client = 1 + zeo-address = 8100 + + [filestorage] + recipe = plone.recipe.filestorage + zeo = zeoserver2 + parts = + my-fs + + +Backup integration +================== + +Here's a buildout that illustrates backup integration:: + + [buildout] + extends = base.cfg + parts = + filestorage + instance + backup + + [instance] + recipe = plone.recipe.zope2instance + user = me:pass + + [backup] + recipe = collective.recipe.backup>=2.7 + + [filestorage] + recipe = plone.recipe.filestorage + parts = + foo + bar + backup = backup + + +Running the tests +================= + +The github checkout of ``plone.recipe.filestorage`` includes a buildout +which installs a script for running the tests. For this to work, you need to +have the test dependencies installed:: + + python bootstrap.py + bin/buildout + bin/test + +Alternatively, you can change the final step to ``./bin/python setup.py test`` +which will retrieve the test dependencies and run the tests. + +Known issue: The tests run buildout in a separate process, so it's currently +impossible to put a ``pdb`` breakpoint in the recipe and debug during the test. +If you need to do this, set up another buildout which uses +``plone.recipe.filestorage`` +as a development egg. + + +Reporting bugs or asking questions +================================== + +Use the github tracker: +https://github.com/collective/plone.recipe.filestorage/issues + +Some old bugs are at Launchpad: +https://bugs.launchpad.net/collective.buildout/ diff --git a/buildout.cfg b/buildout.cfg new file mode 100644 index 0000000..a597a82 --- /dev/null +++ b/buildout.cfg @@ -0,0 +1,16 @@ +[buildout] +index = https://pypi.org/simple/ +develop = . +parts = + test +show-picked-versions = true +versions = versions +# network speedup +socket-timeout = 3 + +[test] +recipe = zc.recipe.testrunner +eggs = plone.recipe.filestorage[test] +defaults = ['--auto-color', '--auto-progress', '--ndiff'] + +[versions] diff --git a/plone/__init__.py b/plone/__init__.py new file mode 100644 index 0000000..f48ad10 --- /dev/null +++ b/plone/__init__.py @@ -0,0 +1,6 @@ +# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages +try: + __import__('pkg_resources').declare_namespace(__name__) +except ImportError: + from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) diff --git a/plone/recipe/__init__.py b/plone/recipe/__init__.py new file mode 100644 index 0000000..f48ad10 --- /dev/null +++ b/plone/recipe/__init__.py @@ -0,0 +1,6 @@ +# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages +try: + __import__('pkg_resources').declare_namespace(__name__) +except ImportError: + from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) diff --git a/plone/recipe/filestorage/README.txt b/plone/recipe/filestorage/README.txt new file mode 100644 index 0000000..fd6c08e --- /dev/null +++ b/plone/recipe/filestorage/README.txt @@ -0,0 +1 @@ +Code repository: https://github.com/aormazabal/plone.recipe.filestorage diff --git a/plone/recipe/filestorage/TODO.txt b/plone/recipe/filestorage/TODO.txt new file mode 100644 index 0000000..241a31c --- /dev/null +++ b/plone/recipe/filestorage/TODO.txt @@ -0,0 +1,4 @@ +[ ] make sure the mode setting for new blobstorage dirs gets set correctly + +nice to have: +[ ] relstorage support diff --git a/plone/recipe/filestorage/__init__.py b/plone/recipe/filestorage/__init__.py new file mode 100644 index 0000000..49b1347 --- /dev/null +++ b/plone/recipe/filestorage/__init__.py @@ -0,0 +1,342 @@ +# -*- coding: utf-8 -*- +"""Recipe filestorage""" + +import os + +from zc.buildout import UserError + + +class Recipe(object): + """zc.buildout recipe""" + + def __init__(self, buildout, name, options): + self.buildout, self.name, self.options = buildout, name, options + active_parts = [p.strip() for p in self.buildout['buildout']['parts'].split()] + + # figure out which ZEO we're going to inject filestorage configuration into, if any + zeo_address = None + self.zeo_part = options.get('zeo', None) + if self.zeo_part is not None: + if self.zeo_part in self.buildout: + zeo_address = self.buildout[self.zeo_part].get('zeo-address', 8100) + else: + raise UserError('[plone.recipe.filestorage] "%s" ' + 'part specifies nonexistant zeo part "%s".' % (name, self.zeo_part)) + else: + for part_name in active_parts: + part = self.buildout[part_name] + if 'recipe' not in part: + continue + elif part['recipe'] in ('plone.recipe.zope2zeoserver', 'plone.recipe.zeoserver'): + if self.zeo_part is not None: + raise UserError('[plone.recipe.filestorage] "%s" part found multiple zeoserver parts; ' + 'please specify which one to use with the "zeo" option.' % name) + self.zeo_part = part_name + zeo_address = part.get('zeo-address', 8100) + + # figure out which Zopes we're going to inject filestorage configuration into + self.zope_parts = options.get('zopes', '').split() + if len(self.zope_parts) == 0: + for part_name in active_parts: + part = self.buildout[part_name] + if 'recipe' not in part: + continue + elif part['recipe'] == 'plone.recipe.zope2instance': + if zeo_address is None or zeo_address == part.get('zeo-address', 8100): + self.zope_parts.append(part_name) + + # figure out c.r.backup recipe + self.backup_part = options.get('backup', None) + if self.backup_part is not None: + if not self.backup_part in self.buildout: + raise UserError('[plone.recipe.filestorage] "%s" part specifies ' + 'nonexistant backup part "%s".' % (name, self.backup_part)) + + # make sure this part is before any associated zeo/zope parts in the + # buildout parts list + self._validate_part_order() + + # inject the extra sections into the correct zope-conf-additional or + # zeo-conf-additional variables. + self.subparts = options.get('parts', '').split() + for subpart in self.subparts: + for zope_part in self.zope_parts: + self._inject_zope_conf(zope_part, subpart) + if self.zeo_part is not None: + self._inject_zeo_conf(self.zeo_part, subpart) + if self.backup_part is not None: + self._inject_backup_additional(self.backup_part, subpart) + + def install(self): + + for subpart in self.subparts: + # create the directory for this filestorage + location = self._subpart_option(subpart, 'location', + default=os.path.join('var', 'filestorage', '%(fs_part_name)s', + '%(fs_part_name)s.fs')) + location = os.path.join(self.buildout['buildout']['directory'], location) + fs_dir = os.path.dirname(location) + if not os.path.exists(fs_dir): + os.makedirs(fs_dir) + + # create blobstorage dirs + blob_storage = os.path.join('var', 'blobstorage-%(fs_part_name)s') + if self._subpart_option(subpart, 'blob-storage', default=''): + blob_storage = self._subpart_option(subpart, 'blob-storage', default=blob_storage) + if not blob_storage.startswith(os.path.sep): + blob_storage = os.path.join(self.buildout['buildout']['directory'], blob_storage) + if not os.path.exists(blob_storage): + os.makedirs(blob_storage) + + if self.zeo_part: + zeo_blob_storage = self._subpart_option(subpart, 'zeo-blob-storage', default=blob_storage) + if not zeo_blob_storage.startswith(os.path.sep): + zeo_blob_storage = os.path.join(self.buildout['buildout']['directory'], zeo_blob_storage) + if not os.path.exists(zeo_blob_storage): + os.makedirs(zeo_blob_storage) + + # return an empty list because we don't have anything we want buildout to automatically remove + return tuple() + + def update(self): + """Updater""" + pass + + def _validate_part_order(self): + """ Make sure this part is before any associated zeo/zope parts in the + buildout parts list. + """ + injector_parts = [self.name] + target_parts = self.zope_parts[:] + if self.zeo_part is not None: + target_parts.append(self.zeo_part) + if self.backup_part is not None: + target_parts.append(self.backup_part) + for part_name in self.buildout['buildout']['parts'].split(): + if part_name in injector_parts: + injector_parts.remove(part_name) + continue + if part_name in target_parts: + if len(injector_parts) > 0: + raise UserError( + '[plone.recipe.filestorage] The "%s" part must be listed ' + 'before the following parts in ${buildout:parts}: %s' % ( + self.name, ', '.join(target_parts))) + target_parts.remove(part_name) + if len(target_parts) > 0: + raise UserError( + '[plone.recipe.filestorage] The "%s" part expected but failed ' + 'to find the following parts in ${buildout:parts}: %s' % ( + self.name, ', '.join(target_parts))) + + def _inject_zope_conf(self, zope_part, subpart): + zope_options = self.buildout[zope_part] + + location = self._subpart_option(subpart, 'location', + default=os.path.join('var', 'filestorage', '%(fs_part_name)s', + '%(fs_part_name)s.fs')) + location = os.path.join(self.buildout['buildout']['directory'], location) + + storage_template = file_storage_template + blob_storage = os.path.join('var', 'blobstorage-%(fs_part_name)s') + blob_enabled = False + if self._subpart_option(subpart, 'blob-storage', default=''): + blob_enabled = True + blob_storage = self._subpart_option(subpart, 'blob-storage', default=blob_storage) + if not blob_storage.startswith(os.path.sep): + blob_storage = os.path.join(self.buildout['buildout']['directory'], blob_storage) + storage_template = self._blob_storage_template(zope_part) + storage_snippet = storage_template % dict( + fs_name='', + fs_path=location, + blob_storage=blob_storage, + ) + + if zope_options.get('zeo-client', 'false').lower() in ('yes', 'true', 'on', '1'): + zeo_address = self._subpart_option(subpart, 'zeo-address', default='8100', + inherit=(zope_part, self.zeo_part)) + zeo_client_cache_size = self._subpart_option(subpart, 'zeo-client-cache-size', default='30MB', + inherit=zope_part) + zeo_client_client = self._subpart_option(subpart, 'zeo-client-client', default='', inherit=zope_part) + if zeo_client_client: + zeo_client_client = 'client %s' % zeo_client_client + zeo_storage = self._subpart_option(subpart, 'zeo-storage', default='%(fs_part_name)s') + zeo_client_name = self._subpart_option(subpart, 'zeo-client-name', default='%(fs_part_name)s_zeostorage') + zeo_client_var = self._subpart_option(subpart, 'zeo-client-var', + default=os.path.join(zope_options['location'], 'var')) + + zeo_storage_template = zeo_file_storage_template + zeo_blob_storage = self._subpart_option(subpart, 'zeo-blob-storage', default=blob_storage) + if not zeo_blob_storage.startswith(os.path.sep): + zeo_blob_storage = os.path.join(self.buildout['buildout']['directory'], zeo_blob_storage) + zeo_shared_blob_dir = self._subpart_option(subpart, 'zeo-shared-blob-dir', default='on') + if blob_enabled: + zeo_storage_template = zeo_blob_storage_template + + storage_snippet = zeo_storage_template % dict( + zeo_address=zeo_address, + zeo_client_cache_size=zeo_client_cache_size, + zeo_client_client=zeo_client_client, + zeo_storage=zeo_storage, + zeo_client_name=zeo_client_name, + zeo_client_var=zeo_client_var, + zeo_blob_storage=zeo_blob_storage, + zeo_shared_blob_dir=zeo_shared_blob_dir, + ) + + zodb_cache_size = self._subpart_option(subpart, 'zodb-cache-size', default='5000', inherit=zope_part) + allow_implicit_cross_references = self._subpart_option(subpart, 'allow-implicit-cross-references', + default='false') + zodb_name = self._subpart_option(subpart, 'zodb-name', default='%(fs_part_name)s') + zodb_mountpoint = self._subpart_option(subpart, 'zodb-mountpoint', default='/%(fs_part_name)s') + zodb_container_class = self._subpart_option(subpart, 'zodb-container-class', default='') + if zodb_container_class: + zodb_container_class = "\n container-class %s" % zodb_container_class + zodb_stanza = zodb_template % dict( + zodb_name=zodb_name, + allow_implicit_cross_references=allow_implicit_cross_references, + zodb_mountpoint=zodb_mountpoint, + zodb_container_class=zodb_container_class, + zodb_cache_size=zodb_cache_size, + storage_snippet=storage_snippet.strip() + ) + + zope_conf_additional = zope_options.get('zope-conf-additional', '') + zope_options['zope-conf-additional'] = zope_conf_additional + zodb_stanza + + def _inject_zeo_conf(self, zeo_part, subpart): + zeo_options = self.buildout[zeo_part] + + location = self._subpart_option(subpart, 'location', + default=os.path.join('var', 'filestorage', '%(fs_part_name)s', + '%(fs_part_name)s.fs')) + location = os.path.join(self.buildout['buildout']['directory'], location) + zeo_storage = self._subpart_option(subpart, 'zeo-storage', default='%(fs_part_name)s') + + storage_template = file_storage_template + blob_storage = os.path.join('var', 'blobstorage-%(fs_part_name)s') + if self._subpart_option(subpart, 'blob-storage', default=''): + blob_storage = self._subpart_option(subpart, 'blob-storage', default=blob_storage) + if not blob_storage.startswith(os.path.sep): + blob_storage = os.path.join(self.buildout['buildout']['directory'], blob_storage) + storage_template = self._blob_storage_template(zeo_part) + + storage_snippet = storage_template % dict( + fs_name=zeo_storage, + fs_path=location, + blob_storage=blob_storage, + ) + + zeo_conf_additional = zeo_options.get('zeo-conf-additional', '') + zeo_options['zeo-conf-additional'] = zeo_conf_additional + storage_snippet + + def _inject_backup_additional(self, backup_part, subpart): + backup_options = self.buildout[backup_part] + + location = self._subpart_option(subpart, 'location', + default=os.path.join('var', 'filestorage', '%(fs_part_name)s', + '%(fs_part_name)s.fs')) + location = os.path.join(self.buildout['buildout']['directory'], location) + zeo_storage = self._subpart_option(subpart, 'zeo-storage', default='%(fs_part_name)s') + backup_template = "%(fs_name)s %(fs_path)s" + blob_storage = os.path.join('var', 'blobstorage-%(fs_part_name)s') + if self._subpart_option(subpart, 'blob-storage', default=''): + blob_storage = self._subpart_option(subpart, 'blob-storage', default=blob_storage) + if not blob_storage.startswith(os.path.sep): + blob_storage = os.path.join(self.buildout['buildout']['directory'], blob_storage) + backup_template = "%(fs_name)s %(fs_path)s %(blob_storage)s" + + additional = backup_template % dict( + fs_name=zeo_storage, + fs_path=location, + blob_storage=blob_storage, + ) + + backup_additionals = backup_options.get('additional_filestorages', '') + backup_options['additional_filestorages'] = '\n'.join([backup_additionals, additional]) + + def _subpart_option(self, subpart, option, default=None, inherit=()): + """ Retrieve an option for a filestorage subpart, perhaps falling back to other specified parts. + Also substitutes the name of the subpart. + """ + + parts_to_check = ['filestorage_' + subpart, self.name] + if type(inherit) == type(''): + inherit = (inherit,) + parts_to_check.extend(inherit) + + val = default + for part in parts_to_check: + if part not in self.buildout: + continue + if option in self.buildout[part]: + val = self.buildout[part][option] + break + + return val % dict( + fs_part_name=subpart + ) + + def _blob_storage_template(self, part): + if 'zope2-location' in self.buildout[part]: + # non-eggified Zope; assume ZODB 3.8.x + return blob_storage_zodb_3_8_template + else: + return blob_storage_zodb_3_9_template + + +# Storage snippets for zope.conf template +file_storage_template = """ + + path %(fs_path)s + +""" + +blob_storage_zodb_3_8_template = """ + + blob-dir %(blob_storage)s + + path %(fs_path)s + + +""" + +blob_storage_zodb_3_9_template = """ + + path %(fs_path)s + blob-dir %(blob_storage)s + +""" + +zeo_file_storage_template = """ + + server %(zeo_address)s + storage %(zeo_storage)s + name %(zeo_client_name)s + var %(zeo_client_var)s + cache-size %(zeo_client_cache_size)s + %(zeo_client_client)s + +""".strip() + +zeo_blob_storage_template = """ + + blob-dir %(zeo_blob_storage)s + shared-blob-dir %(zeo_shared_blob_dir)s + server %(zeo_address)s + storage %(zeo_storage)s + name %(zeo_client_name)s + var %(zeo_client_var)s + cache-size %(zeo_client_cache_size)s + %(zeo_client_client)s + +""".strip() + +zodb_template = """ + + cache-size %(zodb_cache_size)s + allow-implicit-cross-references %(allow_implicit_cross_references)s + %(storage_snippet)s + mount-point %(zodb_mountpoint)s%(zodb_container_class)s + +""" diff --git a/plone/recipe/filestorage/tests/__init__.py b/plone/recipe/filestorage/tests/__init__.py new file mode 100644 index 0000000..5bb534f --- /dev/null +++ b/plone/recipe/filestorage/tests/__init__.py @@ -0,0 +1 @@ +# package diff --git a/plone/recipe/filestorage/tests/doctests.rst b/plone/recipe/filestorage/tests/doctests.rst new file mode 100644 index 0000000..7d4510e --- /dev/null +++ b/plone/recipe/filestorage/tests/doctests.rst @@ -0,0 +1,883 @@ +Example usage +============= + +Let's create and run a minimal buildout that adds an extra filestorage:: + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... parts = + ... filestorage + ... instance + ... + ... [instance] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... # dead chicken .. if we don't specify no eggs, 'instance' is assumed + ... # https://dev.plone.org/ticket/14023#comment:1 + ... eggs = + ... + ... [filestorage] + ... recipe = plone.recipe.filestorage + ... parts = + ... my-fs + ... ''' % globals()) + >>> print(system(join('bin', 'buildout') + ' -q')) + +Our ``zope.conf`` should get the extra filestorage stanza automatically injected into it:: + + >>> instance = os.path.join(sample_buildout, 'parts', 'instance') + >>> print(open(os.path.join(instance, 'etc', 'zope.conf')).read()) + %define INSTANCEHOME...instance + ... + + + cache-size 5000 + allow-implicit-cross-references false + + path .../var/filestorage/my-fs/my-fs.fs + + mount-point /my-fs + + + +The recipe will also create a directory for the new filestorage:: + + >>> 'my-fs' in os.listdir(os.path.join(sample_buildout, 'var', 'filestorage')) + True + +Let's make sure that the conf files will be regenerated whenever we make a change to a filestorage part, +even if the direct configuration for the zope/zeo parts hasn't changed:: + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... parts = + ... filestorage + ... instance + ... + ... [instance] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... # dead chicken .. if we don't specify no eggs, 'instance' is assumed + ... # https://dev.plone.org/ticket/14023#comment:1 + ... eggs = + ... + ... [filestorage] + ... recipe = plone.recipe.filestorage + ... parts = + ... my-fs + ... my-fs-2 + ... ''' % globals()) + >>> print(system(join('bin', 'buildout') + ' -q')) + >>> 'my-fs-2' in open('parts/instance/etc/zope.conf').read() + True + +Let's make sure that the filestorage directory is not clobbered even if the filestorage part is removed +from the buildout:: + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... parts = + ... instance + ... + ... [instance] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... eggs = + ... ''' % globals()) + >>> print(system(join('bin', 'buildout') + ' -q')) + >>> 'my-fs' in os.listdir(os.path.join(sample_buildout, 'var', 'filestorage')) + True + +We can override the defaults for a number of settings:: + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... parts = + ... filestorage + ... instance + ... + ... [instance] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... eggs = + ... + ... [filestorage] + ... recipe = plone.recipe.filestorage + ... location = var/filestorage/%(fs_part_name)s/Data.fs + ... blob-storage = var/blobstorage-%(fs_part_name)s + ... zodb-name = %(fs_part_name)s_db + ... zodb-cache-size = 1000 + ... zodb-mountpoint = /%(fs_part_name)s_mountpoint + ... parts = + ... my-fs + ... ''') + >>> print(system(join('bin', 'buildout') + ' -q')) + >>> instance = os.path.join(sample_buildout, 'parts', 'instance') + >>> print(open(os.path.join(instance, 'etc', 'zope.conf')).read()) + %define INSTANCEHOME...instance + ... + + + cache-size 1000 + allow-implicit-cross-references false + + path .../var/filestorage/my-fs/Data.fs + blob-dir .../var/blobstorage-my-fs + + mount-point /my-fs_mountpoint + + + +A setting can also be modified just for one particular filestorage, by creating a new part with +the ``filestorage_`` prefix, like so:: + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... parts = + ... filestorage + ... instance + ... + ... [instance] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... eggs = + ... + ... [filestorage] + ... recipe = plone.recipe.filestorage + ... parts = + ... my-fs + ... + ... [filestorage_my-fs] + ... zodb-cache-size = 1000 + ... ''' % globals()) + >>> print(system(join('bin', 'buildout') + ' -q')) + >>> instance = os.path.join(sample_buildout, 'parts', 'instance') + >>> print(open(os.path.join(instance, 'etc', 'zope.conf')).read()) + %define INSTANCEHOME...instance + ... + + + cache-size 1000 + allow-implicit-cross-references false + + path .../var/filestorage/my-fs/my-fs.fs + + mount-point /my-fs + + + + +By default, the recipe adds the extra filestorages to each plone.recipe.zope2instance part in the buildout, +but you can tell it to only add it to certain parts:: + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... parts = + ... filestorage + ... instance1 + ... instance2 + ... + ... [instance1] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... eggs = + ... + ... [instance2] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... eggs = + ... + ... [filestorage] + ... recipe = plone.recipe.filestorage + ... zopes = instance1 + ... parts = + ... my-fs + ... ''' % globals()) + >>> print(system(join('bin', 'buildout') + ' -q')) + >>> 'my-fs' in open('parts/instance1/etc/zope.conf').read() + True + >>> 'my-fs' in open('parts/instance2/etc/zope.conf').read() + False + +Example Usage with ZEO +====================== + +Here is a minimal buildout including a ZEO server and two ZODB clients:: + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... parts = + ... filestorage + ... zeoserver + ... primary + ... secondary + ... + ... [zeoserver] + ... recipe = plone.recipe.zeoserver + ... + ... [primary] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... zeo-client = 1 + ... eggs = + ... + ... [secondary] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... eggs = + ... zeo-client = 1 + ... + ... [filestorage] + ... recipe = plone.recipe.filestorage + ... parts = + ... my-fs + ... ''' % globals()) + >>> print(system(join('bin', 'buildout') + ' -q')) + Created directory .../parts/zeoserver + Created directory .../parts/zeoserver/etc + Created directory .../parts/zeoserver/var + Created directory .../parts/zeoserver/log + Created directory .../parts/zeoserver/bin + Wrote file .../parts/zeoserver/etc/zeo.conf + Wrote file .../parts/zeoserver/bin/zeoctl + Changed mode for .../parts/zeoserver/bin/zeoctl to ... + Wrote file .../parts/zeoserver/bin/runzeo + Changed mode for .../parts/zeoserver/bin/runzeo to ... + +This should result in the appropriate additions to ``zeo.conf`` and both ``zope.conf``'s:: + + >>> zeoserver = os.path.join(sample_buildout, 'parts', 'zeoserver') + >>> print(open(os.path.join(zeoserver, 'etc', 'zeo.conf')).read()) + %define INSTANCE /sample-buildout/parts/zeoserver + ... + + + path /sample-buildout/var/filestorage/my-fs/my-fs.fs + + + + >>> primary = os.path.join(sample_buildout, 'parts', 'primary') + >>> print(open(os.path.join(primary, 'etc', 'zope.conf')).read()) + %define INSTANCEHOME /sample-buildout/parts/primary + ... + + + cache-size 5000 + allow-implicit-cross-references false + + server 8100 + storage my-fs + name my-fs_zeostorage + var /sample-buildout/parts/primary/var + cache-size 30MB + + + mount-point /my-fs + + + + >>> secondary = os.path.join(sample_buildout, 'parts', 'secondary') + >>> print(open(os.path.join(secondary, 'etc', 'zope.conf')).read()) + %define INSTANCEHOME /sample-buildout/parts/secondary + ... + + + cache-size 5000 + allow-implicit-cross-references false + + server 8100 + storage my-fs + name my-fs_zeostorage + var /sample-buildout/parts/secondary/var + cache-size 30MB + + + mount-point /my-fs + + + +As above, we can override a number of the default parameters:: + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... parts = + ... filestorage + ... zeoserver + ... primary + ... secondary + ... + ... [zeoserver] + ... recipe = plone.recipe.zeoserver + ... + ... [primary] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... zeo-client = 1 + ... eggs = + ... + ... [secondary] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... zeo-client = 1 + ... eggs = + ... + ... [filestorage] + ... recipe = plone.recipe.filestorage + ... location = var/filestorage/%(fs_part_name)s/Data.fs + ... blob-storage = var/blobstorage-%(fs_part_name)s + ... zodb-cache-size = 1000 + ... zodb-name = %(fs_part_name)s_db + ... zodb-mountpoint = /%(fs_part_name)s_mountpoint + ... zeo-address = 8101 + ... zeo-client-cache-size = 50MB + ... zeo-storage = %(fs_part_name)s_storage + ... zeo-client-name = %(fs_part_name)s_zeostorage_name + ... parts = + ... my-fs + ... ''') + >>> print(system(join('bin', 'buildout') + ' -q')) + Created directory .../parts/zeoserver + Created directory .../parts/zeoserver/etc + Created directory .../parts/zeoserver/var + Created directory .../parts/zeoserver/log + Created directory .../parts/zeoserver/bin + Wrote file .../parts/zeoserver/etc/zeo.conf + Wrote file .../parts/zeoserver/bin/zeoctl + Changed mode for .../parts/zeoserver/bin/zeoctl to ... + Wrote file .../parts/zeoserver/bin/runzeo + Changed mode for .../parts/zeoserver/bin/runzeo to ... + + >>> zeoserver = os.path.join(sample_buildout, 'parts', 'zeoserver') + >>> print(open(os.path.join(zeoserver, 'etc', 'zeo.conf')).read()) + %define INSTANCE /sample-buildout/parts/zeoserver + ... + + + path /sample-buildout/var/filestorage/my-fs/Data.fs + blob-dir /sample-buildout/var/blobstorage-my-fs + + + >>> primary = os.path.join(sample_buildout, 'parts', 'primary') + >>> print(open(os.path.join(primary, 'etc', 'zope.conf')).read()) + %define INSTANCEHOME /sample-buildout/parts/primary + ... + + + cache-size 1000 + allow-implicit-cross-references false + + blob-dir /sample-buildout/var/blobstorage-my-fs + shared-blob-dir on + server 8101 + storage my-fs_storage + name my-fs_zeostorage_name + var /sample-buildout/parts/primary/var + cache-size 50MB + + + mount-point /my-fs_mountpoint + + + >>> secondary = os.path.join(sample_buildout, 'parts', 'secondary') + >>> print(open(os.path.join(secondary, 'etc', 'zope.conf')).read()) + %define INSTANCEHOME /sample-buildout/parts/secondary + ... + + + cache-size 1000 + allow-implicit-cross-references false + + blob-dir /sample-buildout/var/blobstorage-my-fs + shared-blob-dir on + server 8101 + storage my-fs_storage + name my-fs_zeostorage_name + var /sample-buildout/parts/secondary/var + cache-size 50MB + + + mount-point /my-fs_mountpoint + + + +By default, the recipe adds the extra filestorages to the first +``plone.recipe.zeoserver`` part in the buildout, and will throw an error if +there is more than one part using this recipe. However, you can override this +behavior by specifying a particular ZEO part. In this case, the filestorages +will only be added to the Zopes using that ZEO, by default:: + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... parts = + ... filestorage + ... zeoserver1 + ... zeoserver2 + ... primary + ... secondary + ... other-zope + ... + ... [zeoserver1] + ... recipe = plone.recipe.zeoserver + ... zeo-address = 8100 + ... + ... [zeoserver2] + ... recipe = plone.recipe.zeoserver + ... zeo-address = 8101 + ... + ... [primary] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... zeo-client = 1 + ... zeo-address = 8101 + ... eggs = + ... + ... [secondary] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... zeo-client = 1 + ... zeo-address = 8101 + ... eggs = + ... + ... [other-zope] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... zeo-client = 1 + ... zeo-address = 8100 + ... eggs = + ... + ... [filestorage] + ... recipe = plone.recipe.filestorage + ... zeo = zeoserver2 + ... parts = + ... my-fs + ... ''' % globals()) + >>> print(system(join('bin', 'buildout') + ' -q')) + Created directory .../parts/zeoserver2 + Created directory .../parts/zeoserver2/etc + Created directory .../parts/zeoserver2/var + Created directory .../parts/zeoserver2/log + Created directory .../parts/zeoserver2/bin + Wrote file .../parts/zeoserver2/etc/zeo.conf + Wrote file .../parts/zeoserver2/bin/zeoctl + Changed mode for .../parts/zeoserver2/bin/zeoctl to ... + Wrote file .../parts/zeoserver2/bin/runzeo + Changed mode for .../parts/zeoserver2/bin/runzeo to ... + Created directory .../parts/zeoserver1 + Created directory .../parts/zeoserver1/etc + Created directory .../parts/zeoserver1/var + Created directory .../parts/zeoserver1/log + Created directory .../parts/zeoserver1/bin + Wrote file .../parts/zeoserver1/etc/zeo.conf + Wrote file .../parts/zeoserver1/bin/zeoctl + Changed mode for .../parts/zeoserver1/bin/zeoctl to ... + Wrote file .../parts/zeoserver1/bin/runzeo + Changed mode for .../parts/zeoserver1/bin/runzeo to ... + + >>> 'my-fs' in open('parts/zeoserver2/etc/zeo.conf').read() + True + >>> 'my-fs' in open('parts/zeoserver1/etc/zeo.conf').read() + False + >>> 'my-fs' in open('parts/primary/etc/zope.conf').read() + True + >>> 'my-fs' in open('parts/other-zope/etc/zope.conf').read() + False + +Backup integration +================== + +Backup integration:: + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... parts = + ... filestorage + ... instance + ... backup + ... + ... [instance] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... eggs = + ... + ... [backup] + ... recipe = collective.recipe.backup>=2.7 + ... + ... [filestorage] + ... recipe = plone.recipe.filestorage + ... parts = + ... foo + ... bar + ... backup = backup + ... ''') + >>> print(system(join('bin', 'buildout') + ' -q')) + >>> print(re.search( + ... r"storages\s*=\s*\[([^\]]+)\]", + ... open('bin/backup').read(), + ... flags=re.M).group(1)) + {'backup_location': '/sample-buildout/var/backups_foo', + 'blobdir': '', + 'datafs': '/sample-buildout/var/filestorage/foo/foo.fs', + 'snapshot_location': '/sample-buildout/var/snapshotbackups_foo', + 'storage': 'foo', + 'zip_location': '/sample-buildout/var/zipbackups_foo'}, + {'backup_location': '/sample-buildout/var/backups_bar', + 'blobdir': '', + 'datafs': '/sample-buildout/var/filestorage/bar/bar.fs', + 'snapshot_location': '/sample-buildout/var/snapshotbackups_bar', + 'storage': 'bar', + 'zip_location': '/sample-buildout/var/zipbackups_bar'}, + {'backup_location': '/sample-buildout/var/backups', + 'blob_backup_location': '/sample-buildout/var/blobstoragebackups', + 'blob_snapshot_location': '/sample-buildout/var/blobstoragesnapshots', + 'blob_zip_location': '/sample-buildout/var/blobstoragezips', + 'blobdir': '/sample-buildout/var/blobstorage', + 'datafs': '/sample-buildout/var/filestorage/Data.fs', + 'snapshot_location': '/sample-buildout/var/snapshotbackups', + 'storage': '1', + 'zip_location': '/sample-buildout/var/zipbackups'} + +Backup with blob storage and custom filestorage location:: + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... parts = + ... filestorage + ... instance + ... backup + ... + ... [instance] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... eggs = + ... + ... [backup] + ... recipe = collective.recipe.backup>=2.7 + ... + ... [filestorage] + ... recipe = plone.recipe.filestorage + ... location = var/filestorage/%(fs_part_name)s/Data.fs + ... blob-storage = var/blobstorage-%(fs_part_name)s + ... zodb-name = %(fs_part_name)s_db + ... parts = + ... foo + ... bar + ... backup = backup + ... ''') + >>> print(system(join('bin', 'buildout') + ' -q')) + >>> print(re.search( + ... r"storages\s*=\s*\[([^\]]+)\]", + ... open('bin/backup').read(), + ... flags=re.M).group(1)) + {'backup_location': '/sample-buildout/var/backups_foo', + 'blob_backup_location': '/sample-buildout/var/blobstoragebackups_foo', + 'blob_snapshot_location': '/sample-buildout/var/blobstoragesnapshots_foo', + 'blob_zip_location': '/sample-buildout/var/blobstoragezips_foo', + 'blobdir': '/sample-buildout/var/blobstorage-foo', + 'datafs': '/sample-buildout/var/filestorage/foo/Data.fs', + 'snapshot_location': '/sample-buildout/var/snapshotbackups_foo', + 'storage': 'foo', + 'zip_location': '/sample-buildout/var/zipbackups_foo'}, + {'backup_location': '/sample-buildout/var/backups_bar', + 'blob_backup_location': '/sample-buildout/var/blobstoragebackups_bar', + 'blob_snapshot_location': '/sample-buildout/var/blobstoragesnapshots_bar', + 'blob_zip_location': '/sample-buildout/var/blobstoragezips_bar', + 'blobdir': '/sample-buildout/var/blobstorage-bar', + 'datafs': '/sample-buildout/var/filestorage/bar/Data.fs', + 'snapshot_location': '/sample-buildout/var/snapshotbackups_bar', + 'storage': 'bar', + 'zip_location': '/sample-buildout/var/zipbackups_bar'}, + {'backup_location': '/sample-buildout/var/backups', + 'blob_backup_location': '/sample-buildout/var/blobstoragebackups', + 'blob_snapshot_location': '/sample-buildout/var/blobstoragesnapshots', + 'blob_zip_location': '/sample-buildout/var/blobstoragezips', + 'blobdir': '/sample-buildout/var/blobstorage', + 'datafs': '/sample-buildout/var/filestorage/Data.fs', + 'snapshot_location': '/sample-buildout/var/snapshotbackups', + 'storage': '1', + 'zip_location': '/sample-buildout/var/zipbackups'} + +No backup integration:: + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... parts = + ... filestorage + ... instance + ... backup + ... + ... [instance] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... eggs = + ... + ... [backup] + ... recipe = collective.recipe.backup>=2.7 + ... additional_filestorages = + ... lorem + ... ipsum + ... + ... [filestorage] + ... recipe = plone.recipe.filestorage + ... parts = + ... foo + ... bar + ... ''') + >>> print(system(join('bin', 'buildout') + ' -q')) + >>> 'lorem' in open('bin/backup').read() + True + >>> 'ipsum' in open('bin/backup').read() + True + >>> 'foo' in open('bin/backup').read() + False + >>> 'bar' in open('bin/backup').read() + False + + +Error conditions +================ + +Important note: You must place all parts using the +plone.recipe.filestorage recipe before the part for the instances and +zeoservers that you are adding the filestorage to. Otherwise you'll get an +error:: + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... parts = + ... instance + ... filestorage + ... + ... [instance] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... eggs = + ... + ... [filestorage] + ... recipe = plone.recipe.filestorage + ... parts = + ... my-fs + ... ''' % globals()) + >>> print(system(join('bin', 'buildout') + ' -q')) + While: + ... + Error: [plone.recipe.filestorage] The "filestorage" part must be listed before the following parts in ${buildout:parts}: instance + + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... parts = + ... backup + ... filestorage + ... instance + ... + ... [instance] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... eggs = + ... + ... [backup] + ... recipe = collective.recipe.backup>=2.7 + ... + ... [filestorage] + ... recipe = plone.recipe.filestorage + ... parts = + ... my-fs + ... backup = backup + ... ''' % globals()) + >>> print(system(join('bin', 'buildout') + ' -q')) + While: + ... + Error: [plone.recipe.filestorage] The "filestorage" part must be listed before the following parts in ${buildout:parts}: instance, backup + + +Buildouts with multiple zeoserver parts will result in an +error if the desired ZEO to associate with is not explicitly specified:: + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... parts = + ... filestorage + ... zeoserver1 + ... zeoserver2 + ... primary + ... secondary + ... + ... [zeoserver1] + ... recipe = plone.recipe.zeoserver + ... + ... [zeoserver2] + ... recipe = plone.recipe.zeoserver + ... + ... [primary] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... zeo-client = 1 + ... eggs = + ... + ... [secondary] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... zeo-client = 1 + ... eggs = + ... + ... [filestorage] + ... recipe = plone.recipe.filestorage + ... parts = + ... my-fs + ... ''' % globals()) + >>> print(system(join('bin', 'buildout') + ' -q')) + While: + ... + Error: [plone.recipe.filestorage] "filestorage" part found multiple zeoserver parts; please specify which one to use with the "zeo" option. + +Specifying a nonexistent ZEO should result in an error:: + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... parts = + ... filestorage + ... zeoserver + ... primary + ... + ... [zeoserver] + ... recipe = plone.recipe.zeoserver + ... + ... [primary] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... zeo-client = 1 + ... eggs = + ... + ... [filestorage] + ... recipe = plone.recipe.filestorage + ... zeo = foobar + ... parts = + ... my-fs + ... ''' % globals()) + >>> print(system(join('bin', 'buildout') + ' -q')) + While: + ... + Error: [plone.recipe.filestorage] "filestorage" part specifies nonexistant zeo part "foobar". + +Specifying a nonexistent backup part should result in an error:: + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... parts = + ... backup + ... filestorage + ... instance + ... + ... [instance] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... eggs = + ... + ... [backup] + ... recipe = collective.recipe.backup>=2.7 + ... + ... [filestorage] + ... recipe = plone.recipe.filestorage + ... parts = + ... my-fs + ... backup = foobar + ... ''') + >>> print(system(join('bin', 'buildout') + ' -q')) + While: + ... + Error: [plone.recipe.filestorage] "filestorage" part specifies nonexistant backup part "foobar". + +So should specifying a nonexistent Zope part:: + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... parts = + ... filestorage + ... zeoserver + ... primary + ... + ... [zeoserver] + ... recipe = plone.recipe.zeoserver + ... + ... [primary] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... zeo-client = 1 + ... eggs = + ... + ... [filestorage] + ... recipe = plone.recipe.filestorage + ... zopes = foobar + ... parts = + ... my-fs + ... ''' % globals()) + >>> print(system(join('bin', 'buildout') + ' -q')) + While: + ... + Error: [plone.recipe.filestorage] The "filestorage" part expected but failed to find the following parts in ${buildout:parts}: foobar + +If the Zope/ZEO parts are being automatically identified, let's make sure +that we don't accidentally "wake up" parts that would not otherwise be +included in the buildout:: + + >>> write('buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... parts = + ... filestorage + ... instance + ... + ... [instance] + ... recipe = plone.recipe.zope2instance + ... user = me:pass + ... eggs = + ... + ... [filestorage] + ... recipe = plone.recipe.filestorage + ... parts = + ... my-fs + ... + ... [foobar] + ... recipe = plone.recipe.distros + ... urls = + ... ''' % globals()) + >>> print(system(join('bin', 'buildout') + ' -q')) + >>> 'foobar' in os.listdir(os.path.join(sample_buildout, 'parts')) + False diff --git a/plone/recipe/filestorage/tests/test_docs.py b/plone/recipe/filestorage/tests/test_docs.py new file mode 100644 index 0000000..d6ea680 --- /dev/null +++ b/plone/recipe/filestorage/tests/test_docs.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +""" +Doctest runner for 'plone.recipe.filestorage'. +""" +__docformat__ = 'restructuredtext' + +import os +import re +import unittest +import doctest +import zc.buildout.tests +import zc.buildout.testing + +from zope.testing import renormalizing + +optionflags = (doctest.ELLIPSIS | + doctest.NORMALIZE_WHITESPACE | + doctest.REPORT_ONLY_FIRST_FAILURE) + +current_dir = os.path.abspath(os.path.dirname(__file__)) +recipe_location = current_dir +zope2_location = os.path.join(current_dir, 'zope2') + + +def setUp(test): + zc.buildout.testing.buildoutSetUp(test) + + # Install any other recipes that should be available in the tests + zc.buildout.testing.install('plone.recipe.zope2instance', test) + zc.buildout.testing.install('zc.recipe.egg', test) + + # Install the recipe in develop mode + zc.buildout.testing.install_develop('plone.recipe.filestorage', test) + + # Add a base.cfg we can extend + zc.buildout.testing.write('base.cfg', ''' +[buildout] +extends = https://dist.plone.org/release/5.2.9/versions.cfg +index = https://pypi.org/simple +[versions] +# pin to a version that doesn't pull in an eggified Zope +waitress = 2.0.0 +''') + + +def test_suite(): + suite = unittest.TestSuite(( + doctest.DocFileSuite( + 'doctests.rst', + setUp=setUp, + tearDown=zc.buildout.testing.buildoutTearDown, + optionflags=optionflags, + checker=renormalizing.RENormalizing([ + # ignore warnings in output + (re.compile('^.*?WARNING:.*?$', re.M), ''), + (re.compile('^.*?warning:.*?$', re.M), ''), + (re.compile('^.*?zip_safe.*?$', re.M), ''), + (re.compile('^.*?module references __path__.*?$', re.M), ''), + zc.buildout.testing.normalize_path, + ]), + globs=globals() + ), + )) + return suite + + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e200c54 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +setuptools==42.0.2 +zc.buildout==2.13.7 +wheel==0.37.1 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..d5bdbff --- /dev/null +++ b/setup.cfg @@ -0,0 +1,3 @@ +# http://cberner.com/2012/02/26/git-revision-numbers-for-setuptools-packages/ +[egg_info] +tag_date = true diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..5816e5c --- /dev/null +++ b/setup.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +""" +This module contains the tool of plone.recipe.filestorage +""" +import os +from setuptools import setup, find_packages + + +def read(*rnames): + return open(os.path.join(os.path.dirname(__file__), *rnames)).read() + + +version = '0.7dev' + +long_description = ( + 'Detailed Documentation\n' + '**********************\n' + + '\n' + + read('README.rst') + + '\n' + + 'Change history\n' + '**************\n' + + '\n' + + read('CHANGES.txt') + + '\n' + + 'Contributors\n' + '************\n' + + '\n' + + read('CONTRIBUTORS.txt') +) +entry_point = 'plone.recipe.filestorage:Recipe' +entry_points = {"zc.buildout": ["default = %s" % entry_point]} + +tests_require = ['zope.testing', 'manuel'] + +setup(name='plone.recipe.filestorage', + version=version, + description="This recipe aids the creation and management of multiple Zope 2 filestorages.", + long_description=long_description, + # Get more strings from http://www.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + 'Framework :: Buildout', + 'Framework :: Zope5', + 'Intended Audience :: Developers', + 'Topic :: Software Development :: Build Tools', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Programming Language :: Python :: 3.6', + 'License :: OSI Approved :: Zope Public License', + ], + keywords='buildout zope zeo zodb mountpoint filestorage', + author='Albert Ormazabal', + author_email='aormazabal@tda.ad', + url='https://github.com/aormazabal/plone.recipe.filestorage', + license='ZPL', + packages=find_packages(exclude=['ez_setup']), + namespace_packages=['plone', 'plone.recipe'], + include_package_data=True, + zip_safe=False, + install_requires=['setuptools', + 'zc.buildout', + 'plone.recipe.zope2instance', + # -*- Extra requirements: -*- + ], + tests_require=tests_require, + extras_require=dict(test=tests_require), + test_suite='plone.recipe.filestorage.tests.test_docs.test_suite', + entry_points=entry_points, + ) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..36bc752 --- /dev/null +++ b/tox.ini @@ -0,0 +1,13 @@ +[tox] +envlist = + py{36} + +[testenv] +usedevelop = false +skip_install = true +deps = + -r requirements.txt +commands_pre = + {envbindir}/buildout -nc {toxinidir}/buildout.cfg buildout:directory={envdir} buildout:develop={toxinidir} install test +commands = + {envbindir}/test {posargs:-cv} \ No newline at end of file