Sphinx extensions for workflow documentation

.gitignore

@@ -16,4 +16,5 @@
 
 # files created by coverage
 *\,cover
-coverage/
+coverage/
+.cache

.travis-data/test_script.sh

@@ -5,7 +5,7 @@ set -ev
 
 case "$TEST_TYPE" in
     docs)
-        # Compile the docs (HTML format); -W to convert warnings in errors, 
+        # Compile the docs (HTML format); -W to convert warnings in errors,
         # -n to warn about all missing references
         SPHINXOPTS="-nW" make -C docs html
         ;;
@@ -26,4 +26,7 @@ case "$TEST_TYPE" in
     pre-commit)
         pre-commit run --all-files || ( git status --short ; git diff ; exit 1 )
         ;;
+    sphinxext)
+        py.test -vv aiida/sphinxext/tests
+        ;;
 esac

.travis.yml

@@ -44,7 +44,7 @@ install:
     - pip install -U pip wheel setuptools
     - pip install -r requirements.txt
     # Install AiiDA with some optional dependencies
-    - pip install .[REST,docs,atomic_tools,testing,dev_precommit]
+    - pip install .[REST,docs,atomic_tools,testing,dev_precommit,dev_sphinxext]
 
 
 env:
@@ -53,6 +53,7 @@ env:
 ## also when building the docs
 ## because otherwise the code would complain. Also, I need latex.
 - TEST_TYPE="pre-commit"
+- TEST_AIIDA_BACKEND=django TEST_TYPE="sphinxext"
 - TEST_AIIDA_BACKEND=django TEST_TYPE="docs"
 - TEST_AIIDA_BACKEND=django TEST_TYPE="tests"
 - TEST_AIIDA_BACKEND=sqlalchemy TEST_TYPE="tests"

aiida/sphinxext/__init__.py

@@ -0,0 +1,16 @@
+"""
+Defines reStructuredText directives to simplify documenting AiiDA and its plugins.
+"""
+
+__version__ = '0.1.0'
+
+from . import workchain
+
+
+def setup(app):
+    """
+    Setup function to add the extension classes / nodes to Sphinx.
+    """
+    workchain.setup_aiida_workchain(app)
+
+    return {'version': __version__, 'parallel_read_safe': True}

aiida/sphinxext/tests/.gitignore

@@ -0,0 +1 @@
+build

aiida/sphinxext/tests/Makefile

@@ -0,0 +1,36 @@
+# Makefile for Sphinx documentation
+#
+
+SPHINXBUILD  = sphinx-build
+BUILDDIR     = build
+HTMLBUILDDIR = build/html
+SOURCEDIR    = workchain_source
+
+# User-friendly check for sphinx-build
+ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
+$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
+endif
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -n -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(SOURCEDIR)
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(SOURCEDIR)
+
+.PHONY: all help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext
+
+all: html
+
+clean:
+	rm -r $(BUILDDIR)
+	rm -r $(HTMLBUILDDIR)
+
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(HTMLBUILDDIR)
+	@echo
+	@echo "Build finished. The HTML pages are in $(HTMLBUILDDIR)."
+
+
+view:
+	xdg-open $(HTMLBUILDDIR)/index.html

aiida/sphinxext/tests/conftest.py

@@ -0,0 +1,63 @@
+import os
+import sys
+import shutil
+import tempfile
+import subprocess
+from os.path import join, dirname
+import xml.etree.ElementTree as ET
+
+import pytest
+
+@pytest.fixture
+def reference_result():
+    def inner(name):
+        return join(dirname(__file__), 'reference_results', name)
+    return inner
+
+@pytest.fixture
+def build_dir():
+    # Python 2 doesn't have tempfile.TemporaryDirectory
+    dirname = tempfile.mkdtemp()
+    try:
+        yield dirname
+    except Exception as e:
+        raise e
+    finally:
+        shutil.rmtree(dirname)
+
+
+@pytest.fixture
+def build_sphinx(build_dir):
+    def inner(source_dir, builder='xml'):
+        doctree_dir = join(build_dir, 'doctrees')
+        out_dir = join(build_dir, builder)
+
+        subprocess.check_call([
+            sys.executable, '-m', 'sphinx', '-b', builder, '-d', doctree_dir,
+            source_dir, out_dir
+        ])
+
+        return out_dir
+
+    return inner
+
+
+@pytest.fixture
+def xml_equal():
+    def inner(test_file, reference_file):
+        if not os.path.isfile(reference_file):
+            shutil.copyfile(test_file, reference_file)
+            raise ValueError('Reference file does not exist!')
+        assert _flatten_xml(test_file) == _flatten_xml(reference_file)
+    return inner
+
+
+def _flatten_xml(filename):
+    return [
+        (
+            el.tag,
+            {k: v for k, v in el.attrib.items() if k not in ['source']},
+            el.text
+        )
+        for el in ET.parse(filename).iter()
+    ]

aiida/sphinxext/tests/reference_results/workchain.xml

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE document PUBLIC "+//IDN docutils.sourceforge.net//DTD Docutils Generic//EN//XML" "http://docutils.sourceforge.net/docs/ref/docutils.dtd">
+<!-- Generated by Docutils 0.13.1 -->
+<document source="/home/greschd/programming/aiida/aiida_core/aiida/sphinxext/tests/workchain_source/index.rst">
+    <section ids="sphinx-aiida-demo" names="sphinx-aiida\ demo">
+        <title>sphinx-aiida demo</title>
+        <paragraph>This is a demo documentation to show off the features of the <literal>sphinx-aiida</literal> extension.</paragraph>
+        <compound classes="toctree-wrapper">
+        </compound>
+        <desc desctype="class" domain="py" noindex="False" objtype="class">
+            <desc_signature first="False" fullname="Workchain"><desc_annotation>workchain</desc_annotation><desc_addname>demo_workchain.</desc_addname><desc_name>DemoWorkChain</desc_name></desc_signature>
+            <desc_content>
+                <paragraph>
+                        A demo workchain to show how the workchain auto-documentation works.
+                        </paragraph>
+                <paragraph><strong>Inputs:</strong><bullet_list bullet="*"><list_item><paragraph><literal_strong>x</literal_strong>, <emphasis>Float</emphasis>, required -- First input argument.</paragraph></list_item><list_item><literal_strong>y</literal_strong>, <emphasis>Namespace</emphasis><bullet_list bullet="*"><list_item><paragraph><literal_strong>z</literal_strong>, <emphasis>Int</emphasis>, required -- Input in a separate namespace.</paragraph></list_item></bullet_list></list_item></bullet_list></paragraph>
+                <paragraph><strong>Inputs not stored in the database:</strong><bullet_list bullet="*"><list_item><paragraph><literal_strong>description</literal_strong>, <emphasis>basestring</emphasis>, optional</paragraph></list_item><list_item><paragraph><literal_strong>label</literal_strong>, <emphasis>basestring</emphasis>, optional</paragraph></list_item><list_item><paragraph><literal_strong>store_provenance</literal_strong>, <emphasis>bool</emphasis>, optional</paragraph></list_item></bullet_list></paragraph>
+                <paragraph><strong>Outputs:</strong><bullet_list bullet="*"><list_item><paragraph><literal_strong>z</literal_strong>, <emphasis>Bool</emphasis>, required -- Output of the demoworkchain.</paragraph></list_item></bullet_list></paragraph>
+            </desc_content>
+        </desc>
+        <paragraph>You can use the <literal>:hide-unstored-inputs:</literal> option to not show the inputs which are not stored in the DB:</paragraph>
+        <desc desctype="class" domain="py" noindex="False" objtype="class">
+            <desc_signature first="False" fullname="Workchain"><desc_annotation>workchain</desc_annotation><desc_addname>demo_workchain.</desc_addname><desc_name>DemoWorkChain</desc_name></desc_signature>
+            <desc_content>
+                <paragraph>
+                        A demo workchain to show how the workchain auto-documentation works.
+                        </paragraph>
+                <paragraph><strong>Inputs:</strong><bullet_list bullet="*"><list_item><paragraph><literal_strong>x</literal_strong>, <emphasis>Float</emphasis>, required -- First input argument.</paragraph></list_item><list_item><literal_strong>y</literal_strong>, <emphasis>Namespace</emphasis><bullet_list bullet="*"><list_item><paragraph><literal_strong>z</literal_strong>, <emphasis>Int</emphasis>, required -- Input in a separate namespace.</paragraph></list_item></bullet_list></list_item></bullet_list></paragraph>
+                <paragraph><strong>Outputs:</strong><bullet_list bullet="*"><list_item><paragraph><literal_strong>z</literal_strong>, <emphasis>Bool</emphasis>, required -- Output of the demoworkchain.</paragraph></list_item></bullet_list></paragraph>
+            </desc_content>
+        </desc>
+        <paragraph>The command is also hooked into <literal>sphinx.ext.autodoc</literal>, so you can also use that.</paragraph>
+        <target ids="module-demo_workchain" ismod="True"></target>
+        <index entries="['single',\ u'demo_workchain\ (module)',\ u'module-demo_workchain',\ '',\ None]"></index>
+        <paragraph>This module defines an example workchain for the aiida-workchain documentation directive.</paragraph>
+        <desc desctype="class" domain="py" noindex="False" objtype="class">
+            <desc_signature first="False" fullname="Workchain"><desc_annotation>workchain</desc_annotation><desc_addname>demo_workchain.</desc_addname><desc_name>DemoWorkChain</desc_name></desc_signature>
+            <desc_content>
+                <paragraph>
+                        A demo workchain to show how the workchain auto-documentation works.
+                        </paragraph>
+                <paragraph><strong>Inputs:</strong><bullet_list bullet="*"><list_item><paragraph><literal_strong>x</literal_strong>, <emphasis>Float</emphasis>, required -- First input argument.</paragraph></list_item><list_item><literal_strong>y</literal_strong>, <emphasis>Namespace</emphasis><bullet_list bullet="*"><list_item><paragraph><literal_strong>z</literal_strong>, <emphasis>Int</emphasis>, required -- Input in a separate namespace.</paragraph></list_item></bullet_list></list_item></bullet_list></paragraph>
+                <paragraph><strong>Inputs not stored in the database:</strong><bullet_list bullet="*"><list_item><paragraph><literal_strong>description</literal_strong>, <emphasis>basestring</emphasis>, optional</paragraph></list_item><list_item><paragraph><literal_strong>label</literal_strong>, <emphasis>basestring</emphasis>, optional</paragraph></list_item><list_item><paragraph><literal_strong>store_provenance</literal_strong>, <emphasis>bool</emphasis>, optional</paragraph></list_item></bullet_list></paragraph>
+                <paragraph><strong>Outputs:</strong><bullet_list bullet="*"><list_item><paragraph><literal_strong>z</literal_strong>, <emphasis>Bool</emphasis>, required -- Output of the demoworkchain.</paragraph></list_item></bullet_list></paragraph>
+            </desc_content>
+        </desc>
+        <index entries="['single',\ u'NormalClass\ (class\ in\ demo_workchain)',\ u'demo_workchain.NormalClass',\ '',\ None]"></index>
+        <desc desctype="class" domain="py" noindex="False" objtype="class">
+            <desc_signature class="" first="False" fullname="NormalClass" ids="demo_workchain.NormalClass" module="demo_workchain" names="demo_workchain.NormalClass"><desc_annotation>class </desc_annotation><desc_addname>demo_workchain.</desc_addname><desc_name>NormalClass</desc_name></desc_signature>
+            <desc_content>
+                <paragraph>This is here to check that we didn't break the regular 'autoclass.</paragraph>
+            </desc_content>
+        </desc>
+    </section>
+</document>

aiida/sphinxext/tests/test_workchain.py

@@ -0,0 +1,13 @@
+"""Tests for the AiiDA workchain Sphinx directive."""
+
+from os.path import join, dirname
+
+import pytest
+
+WORKCHAIN = join(dirname(__file__), 'workchain_source')
+
+
+def test_workchain_build(build_sphinx, xml_equal, reference_result):
+    out_dir = build_sphinx(WORKCHAIN)
+    index_file = join(out_dir, 'index.xml')
+    xml_equal(index_file, reference_result('workchain.xml'))    

aiida/sphinxext/tests/workchain_source/conf.py

@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+#
+# sphinx-aiida-demo documentation build configuration file, created by
+# sphinx-quickstart on Mon Oct  2 13:04:07 2017.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['sphinx.ext.mathjax', 'sphinx.ext.autodoc', 'aiida.sphinxext']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'sphinx-aiida-demo'
+copyright = u'2018, The AiiDA team'
+author = u'The AiiDA team'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = u'0.0.0'
+# The full version, including alpha/beta/rc tags.
+release = u'0.0.0'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'sphinx-aiida-demodoc'
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (
+        master_doc, 'sphinx-aiida-demo.tex',
+        u'sphinx-aiida-demo Documentation', u'Dominik Gresch', 'manual'
+    ),
+]
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [(
+    master_doc, 'sphinx-aiida-demo', u'sphinx-aiida-demo Documentation',
+    [author], 1
+)]
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (
+        master_doc, 'sphinx-aiida-demo', u'sphinx-aiida-demo Documentation',
+        author, 'sphinx-aiida-demo', 'One line description of project.',
+        'Miscellaneous'
+    ),
+]