From 3adfee2c1f8967b8364713193a77fe5d16f40f4a Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Fri, 8 Jan 2016 18:37:44 -0800 Subject: [PATCH] Adding check that all public modules are documented. Fixes #714. --- scripts/run_pylint.py | 6 +- scripts/verify_included_modules.py | 135 +++++++++++++++++++++++++++++ tox.ini | 2 + 3 files changed, 141 insertions(+), 2 deletions(-) create mode 100644 scripts/verify_included_modules.py diff --git a/scripts/run_pylint.py b/scripts/run_pylint.py index 8e2c7d302d9e2..1d2e17b358a46 100644 --- a/scripts/run_pylint.py +++ b/scripts/run_pylint.py @@ -21,6 +21,8 @@ violations (hence it has a reduced number of style checks). """ +from __future__ import print_function + import ConfigParser import copy import os @@ -211,7 +213,7 @@ def lint_fileset(filenames, rcfile, description): if status_code != 0: error_message = ('Pylint failed on %s with ' 'status %d.' % (description, status_code)) - print >> sys.stderr, error_message + print(error_message, file=sys.stderr) sys.exit(status_code) else: print 'Skipping %s, no files to lint.' % (description,) @@ -229,7 +231,7 @@ def main(): raise message = 'Restricted lint failed, expanding to full fileset.' - print >> sys.stderr, message + print(message, file=sys.stderr) all_files, _ = get_files_for_linting(allow_limited=False) library_files, non_library_files, _ = get_python_files( all_files=all_files) diff --git a/scripts/verify_included_modules.py b/scripts/verify_included_modules.py new file mode 100644 index 0000000000000..897206a5ff226 --- /dev/null +++ b/scripts/verify_included_modules.py @@ -0,0 +1,135 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Check if all public modules are included in our docs.""" + + +from __future__ import print_function + +import os +import sys +import warnings + +from sphinx.ext.intersphinx import fetch_inventory + + +BASE_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..')) +DOCS_DIR = os.path.join(BASE_DIR, 'docs') +OBJECT_INVENTORY_RELPATH = os.path.join('_build', 'html', 'objects.inv') +IGNORED_PREFIXES = ('test_', '_') +IGNORED_MODULES = frozenset([ + 'gcloud.bigquery.query', + 'gcloud.bigtable.client', + 'gcloud.bigtable.cluster', + 'gcloud.bigtable.column_family', + 'gcloud.bigtable.happybase.connection', + 'gcloud.bigtable.row', + 'gcloud.bigtable.table', + 'gcloud.datastore.demo.demo', + 'gcloud.demo', + 'gcloud.storage.demo.demo', +]) + + +class SphinxApp(object): + """Mock app to interact with Sphinx helpers.""" + warn = warnings.warn + srcdir = DOCS_DIR + + +def is_valid_module(filename): + """Determines if a filename is a valid Python module. + + Assumes if is just the end of a path (i.e. does not contain + ``os.path.sep``. + + :type filename: string + :param filename: The name of a file. + + :rtype: bool + :returns: Flag indicating if the filename is valid. + """ + if not filename.endswith('.py'): + return False + for prefix in IGNORED_PREFIXES: + if filename.startswith(prefix): + return False + return True + + +def get_public_modules(path, base_package=None): + """Get list of all public modules relative to a path. + + :type path: string + :param path: The path containing the python modules. + + :type base_package: string + :param base_package: (Optional) A package to prepend in + front of the path. + + :rtype: list + :returns: List of all modules found. + """ + result = [] + for subdir, _, files in os.walk(path): + # Skip folders that start with _. + if any([part.startswith('_') + for part in subdir.split(os.path.sep)]): + continue + _, rel_dir = subdir.split(path) + rel_dir = rel_dir.lstrip(os.path.sep) + for filename in files: + if is_valid_module(filename): + mod_name, _ = os.path.splitext(filename) + rel_path = os.path.join(rel_dir, mod_name) + if base_package is not None: + rel_path = os.path.join(base_package, rel_path) + # Turn into a Python module rather than a file path. + result.append(rel_path.replace(os.path.sep, '.')) + + return result + + +def main(): + """Main script to verify modules included.""" + mock_uri = '' + inventory = fetch_inventory(SphinxApp, mock_uri, + OBJECT_INVENTORY_RELPATH) + sphinx_mods = set(inventory['py:module'].keys()) + + library_dir = os.path.join(BASE_DIR, 'gcloud') + public_mods = get_public_modules(library_dir, + base_package='gcloud') + public_mods = set(public_mods) + + if not sphinx_mods <= public_mods: + message = ('Unexpected error. There were modules referenced by ' + 'Sphinx that are not among the public modules.') + print(message, file=sys.stderr) + sys.exit(1) + + undocumented_mods = public_mods - sphinx_mods + # Remove ignored modules. + undocumented_mods -= IGNORED_MODULES + if undocumented_mods: + message_parts = ['Found undocumented public modules:'] + message_parts.extend(['- ' + mod_name + for mod_name in sorted(undocumented_mods)]) + print('\n'.join(message_parts), file=sys.stderr) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/tox.ini b/tox.ini index bd1b0773f4efe..f8716653d945a 100644 --- a/tox.ini +++ b/tox.ini @@ -45,6 +45,7 @@ basepython = commands = python -c "import shutil; shutil.rmtree('docs/_build', ignore_errors=True)" sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html + python {toxinidir}/scripts/verify_included_modules.py deps = Sphinx passenv = {[testenv:system-tests]passenv} SPHINX_RELEASE READTHEDOCS LOCAL_RTD @@ -57,6 +58,7 @@ basepython = {[testenv:docs]basepython} commands = python -c "import shutil; shutil.rmtree('docs/_build_rtd', ignore_errors=True)" sphinx-build -W -b html -d docs/_build_rtd/doctrees docs docs/_build_rtd/html + python {toxinidir}/scripts/verify_included_modules.py deps = {[testenv:docs]deps} passenv = {[testenv:docs]passenv}