From 5fd62fcee7d5add380502c90c4bd9a31af65221b Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 22 Feb 2018 13:42:42 +0100 Subject: [PATCH 01/14] DOC: script to build single docstring page --- doc/make.py | 81 ++++++++++++++++++++++++++++++++++++++++++---- doc/source/conf.py | 30 ----------------- 2 files changed, 75 insertions(+), 36 deletions(-) diff --git a/doc/make.py b/doc/make.py index e3cb29aa3e086..297ddac518889 100755 --- a/doc/make.py +++ b/doc/make.py @@ -18,6 +18,7 @@ import argparse from contextlib import contextmanager import jinja2 +import webbrowser DOC_PATH = os.path.dirname(os.path.abspath(__file__)) @@ -26,7 +27,7 @@ BUILD_DIRS = ['doctrees', 'html', 'latex', 'plots', '_static', '_templates'] -def _generate_index(include_api, single_doc=None): +def _generate_index(include_api=True, single_doc=None): """Create index.rst file with the specified sections. Parameters @@ -48,6 +49,37 @@ def _generate_index(include_api, single_doc=None): single_doc=single_doc)) +def _generate_exclude_pattern(include_api=True, single_doc=None): + + if not include_api: + rst_files = ['api.rst', 'generated/*.rst'] + elif single_doc is not None: + rst_files = [f for f in os.listdir(SOURCE_PATH) + if ((f.endswith('.rst') or f.endswith('.ipynb')) + and (f != 'index.rst') and (f != single_doc))] + rst_files += ['generated/*.rst'] + else: + rst_files = [] + + exclude_patterns = ",".join( + ['{!r}'.format(i) for i in ['**.ipynb_checkpoints'] + rst_files]) + + return exclude_patterns + + +def _write_temp_file(classtype, module, function): + + s = """{1}.{2} +================================= + +.. currentmodule:: {1} + +.. auto{0}:: {2}""".format(classtype, module, function) + + with open(os.path.join(SOURCE_PATH, "temp.rst"), 'w') as f: + f.write(s) + + @contextmanager def _maybe_exclude_notebooks(): """Skip building the notebooks if pandoc is not installed. @@ -96,8 +128,9 @@ class DocBuilder: All public methods of this class can be called as parameters of the script. """ - def __init__(self, num_jobs=1): + def __init__(self, num_jobs=1, exclude_patterns=None): self.num_jobs = num_jobs + self.exclude_patterns = exclude_patterns @staticmethod def _create_build_structure(): @@ -142,8 +175,8 @@ def _sphinx_build(self, kind): self._run_os('sphinx-build', '-j{}'.format(self.num_jobs), '-b{}'.format(kind), - '-d{}'.format(os.path.join(BUILD_PATH, - 'doctrees')), + '-d{}'.format(os.path.join(BUILD_PATH, 'doctrees')), + # TODO integrate exclude_patterns SOURCE_PATH, os.path.join(BUILD_PATH, kind)) @@ -199,6 +232,23 @@ def zip_html(self): '-q', *fnames) + def build_docstring(self): + """Build single docstring page""" + self._create_build_structure() + + args = ('sphinx-build', + '-bhtml', + '-d{}'.format(os.path.join(BUILD_PATH, 'doctrees')), + '-Dexclude_patterns={}'.format(self.exclude_patterns), + SOURCE_PATH, + os.path.join(BUILD_PATH, 'html'), + os.path.join(SOURCE_PATH, 'temp.rst') + ) + # for some reason it does not work with run_os, but it does if I + # directly call the joined command + # self._run_os(*args) + os.system(" ".join(args)) + def main(): cmds = [method for method in dir(DocBuilder) if not method.startswith('_')] @@ -228,6 +278,12 @@ def main(): type=str, default=os.path.join(DOC_PATH, '..'), help='path') + argparser.add_argument('--docstring', + metavar='FILENAME', + type=str, + default=None, + help=('method or function name to compile, ' + 'e.g. "DataFrame.join"')) args = argparser.parse_args() if args.command not in cmds: @@ -235,8 +291,21 @@ def main(): args.command, ', '.join(cmds))) os.environ['PYTHONPATH'] = args.python_path - _generate_index(not args.no_api, args.single) - getattr(DocBuilder(args.num_jobs), args.command)() + + if args.docstring is not None: + _write_temp_file('method', 'pandas', args.docstring) + exclude_patterns = _generate_exclude_pattern(single_doc='temp.rst') + _generate_index(single_doc='temp.rst') + DocBuilder(args.num_jobs, exclude_patterns).build_docstring() + url = "file://" + os.getcwd() + "/build/html/temp.html" + webbrowser.open(url, new=2) + os.remove('source/temp.rst') + + else: + _generate_index(not args.no_api, args.single) + exclude_patterns = _generate_exclude_pattern( + not args.no_api, args.single) + getattr(DocBuilder(args.num_jobs, exclude_patterns), args.command)() if __name__ == '__main__': diff --git a/doc/source/conf.py b/doc/source/conf.py index 7c4edd0486636..bf90867743fd5 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -83,36 +83,6 @@ if any(re.match("\s*api\s*", l) for l in index_rst_lines): autosummary_generate = True -files_to_delete = [] -for f in os.listdir(os.path.dirname(__file__)): - if (not f.endswith(('.ipynb', '.rst')) or - f.startswith('.') or os.path.basename(f) == 'index.rst'): - continue - - _file_basename = os.path.splitext(f)[0] - _regex_to_match = "\s*{}\s*$".format(_file_basename) - if not any(re.match(_regex_to_match, line) for line in index_rst_lines): - files_to_delete.append(f) - -if files_to_delete: - print("I'm about to DELETE the following:\n%s\n" % list(sorted(files_to_delete))) - sys.stdout.write("WARNING: I'd like to delete those to speed up processing (yes/no)? ") - if PY3: - answer = input() - else: - answer = raw_input() - - if answer.lower().strip() in ('y','yes'): - for f in files_to_delete: - f = os.path.join(os.path.join(os.path.dirname(__file__),f)) - f= os.path.abspath(f) - try: - print("Deleting %s" % f) - os.unlink(f) - except: - print("Error deleting %s" % f) - pass - # Add any paths that contain templates here, relative to this directory. templates_path = ['../_templates'] From 2766dde3803306d89096d4a1002cd41b86f3294b Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 22 Feb 2018 14:54:32 +0100 Subject: [PATCH 02/14] try autosummary way --- doc/make.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/doc/make.py b/doc/make.py index 297ddac518889..8c34d6ee7614a 100755 --- a/doc/make.py +++ b/doc/make.py @@ -67,7 +67,7 @@ def _generate_exclude_pattern(include_api=True, single_doc=None): return exclude_patterns -def _write_temp_file(classtype, module, function): +def _write_temp_file2(classtype, module, function): s = """{1}.{2} ================================= @@ -80,6 +80,22 @@ def _write_temp_file(classtype, module, function): f.write(s) +def _write_temp_file(classtype, module, function): + + s = """API docs +======== + +.. autosummary:: + :toctree: generated_temp/ + + {0}.{1} + +""".format(module, function) + + with open(os.path.join(SOURCE_PATH, "temp.rst"), 'w') as f: + f.write(s) + + @contextmanager def _maybe_exclude_notebooks(): """Skip building the notebooks if pandoc is not installed. @@ -242,7 +258,8 @@ def build_docstring(self): '-Dexclude_patterns={}'.format(self.exclude_patterns), SOURCE_PATH, os.path.join(BUILD_PATH, 'html'), - os.path.join(SOURCE_PATH, 'temp.rst') + os.path.join(SOURCE_PATH, 'temp.rst'), + os.path.join(SOURCE_PATH, 'generated_temp/*.rst'), ) # for some reason it does not work with run_os, but it does if I # directly call the joined command @@ -299,7 +316,7 @@ def main(): DocBuilder(args.num_jobs, exclude_patterns).build_docstring() url = "file://" + os.getcwd() + "/build/html/temp.html" webbrowser.open(url, new=2) - os.remove('source/temp.rst') + #os.remove('source/temp.rst') else: _generate_index(not args.no_api, args.single) From 13d31b93dfe2748eaaf27a076f3d8fd2df4e7a1a Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 22 Feb 2018 21:14:07 +0100 Subject: [PATCH 03/14] copy existing generated file if possible, otherwise generate it ourselves --- doc/make.py | 80 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 22 deletions(-) diff --git a/doc/make.py b/doc/make.py index 8c34d6ee7614a..9362d7958c877 100755 --- a/doc/make.py +++ b/doc/make.py @@ -18,6 +18,7 @@ import argparse from contextlib import contextmanager import jinja2 +import shutil import webbrowser @@ -50,7 +51,9 @@ def _generate_index(include_api=True, single_doc=None): def _generate_exclude_pattern(include_api=True, single_doc=None): + """ + """ if not include_api: rst_files = ['api.rst', 'generated/*.rst'] elif single_doc is not None: @@ -67,34 +70,60 @@ def _generate_exclude_pattern(include_api=True, single_doc=None): return exclude_patterns -def _write_temp_file2(classtype, module, function): - - s = """{1}.{2} -================================= - -.. currentmodule:: {1} - -.. auto{0}:: {2}""".format(classtype, module, function) - - with open(os.path.join(SOURCE_PATH, "temp.rst"), 'w') as f: - f.write(s) - +def _generate_temp_docstring_file(method): + """ + """ + fname = os.path.join(SOURCE_PATH, 'generated', '{}.rst'.format(method)) + + # # remove the target file to make sure it is updated again (to build + # # latest version) + # try: + # os.remove(fname) + # except OSError: + # pass + # + # # generate docstring pages + # print("Running sphinx-autogen to generate docstring stub pages") + # os.system("sphinx-autogen -t _templates -o source/generated source/*.rst") + + # create the temporary directory in which we will link the target file + try: + os.makedirs(os.path.join(SOURCE_PATH, 'generated_temp')) + except OSError: + pass -def _write_temp_file(classtype, module, function): + if os.path.exists(fname): + # link the target file + try: + # os.symlink(fname, os.path.join(SOURCE_PATH, 'generated_temp', + # '{}.rst'.format(method)), + # target_is_directory=False) + # copying to make sure sphinx always thinks it is new + # and needs to be re-generated (to pick source code changes) + shutil.copy(fname, os.path.join(SOURCE_PATH, 'generated_temp')) + linked = True + except: # noqa + linked = False + else: + linked = False - s = """API docs -======== + s = """Built docstrings +================ .. autosummary:: - :toctree: generated_temp/ + {toctree} + {name} - {0}.{1} - -""".format(module, function) + """.format(name=method, + toctree=':toctree: generated_temp/\n' if not linked else '') with open(os.path.join(SOURCE_PATH, "temp.rst"), 'w') as f: f.write(s) + if not linked: + print("Running sphinx-autogen on manually created file") + os.system("sphinx-autogen -o source/generated_temp source/temp.rst") + @contextmanager def _maybe_exclude_notebooks(): @@ -310,13 +339,20 @@ def main(): os.environ['PYTHONPATH'] = args.python_path if args.docstring is not None: - _write_temp_file('method', 'pandas', args.docstring) + shutil.rmtree(os.path.join(BUILD_PATH, 'html', 'generated_temp'), + ignore_errors=True) + _generate_temp_docstring_file(args.docstring) exclude_patterns = _generate_exclude_pattern(single_doc='temp.rst') _generate_index(single_doc='temp.rst') DocBuilder(args.num_jobs, exclude_patterns).build_docstring() - url = "file://" + os.getcwd() + "/build/html/temp.html" + # open generated page in new browser tab + url = os.path.join("file://", DOC_PATH, "build", "html", + "generated_temp", "{}.html".format(args.docstring)) webbrowser.open(url, new=2) - #os.remove('source/temp.rst') + # clean-up generated files + os.remove('source/temp.rst') + shutil.rmtree(os.path.join(SOURCE_PATH, 'generated_temp'), + ignore_errors=True) else: _generate_index(not args.no_api, args.single) From 1477e637e0ddb84b25645af75acb8b33cb2d7e65 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 22 Feb 2018 21:31:27 +0100 Subject: [PATCH 04/14] enable multiple docstrings at once --- doc/make.py | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/doc/make.py b/doc/make.py index 9362d7958c877..a6c277764788e 100755 --- a/doc/make.py +++ b/doc/make.py @@ -70,10 +70,11 @@ def _generate_exclude_pattern(include_api=True, single_doc=None): return exclude_patterns -def _generate_temp_docstring_file(method): +def _generate_temp_docstring_file(methods): """ """ - fname = os.path.join(SOURCE_PATH, 'generated', '{}.rst'.format(method)) + fnames = [os.path.join(SOURCE_PATH, 'generated', '{}.rst'.format(method)) + for method in methods] # # remove the target file to make sure it is updated again (to build # # latest version) @@ -92,20 +93,21 @@ def _generate_temp_docstring_file(method): except OSError: pass - if os.path.exists(fname): - # link the target file - try: - # os.symlink(fname, os.path.join(SOURCE_PATH, 'generated_temp', - # '{}.rst'.format(method)), - # target_is_directory=False) - # copying to make sure sphinx always thinks it is new - # and needs to be re-generated (to pick source code changes) - shutil.copy(fname, os.path.join(SOURCE_PATH, 'generated_temp')) - linked = True - except: # noqa + for fname in fnames: + if os.path.exists(fname): + # link the target file + try: + # os.symlink(fname, os.path.join(SOURCE_PATH, 'generated_temp', + # '{}.rst'.format(method)), + # target_is_directory=False) + # copying to make sure sphinx always thinks it is new + # and needs to be re-generated (to pick source code changes) + shutil.copy(fname, os.path.join(SOURCE_PATH, 'generated_temp')) + linked = True + except: # noqa + linked = False + else: linked = False - else: - linked = False s = """Built docstrings ================ @@ -114,7 +116,7 @@ def _generate_temp_docstring_file(method): {toctree} {name} - """.format(name=method, + """.format(name='\n '.join(methods), toctree=':toctree: generated_temp/\n' if not linked else '') with open(os.path.join(SOURCE_PATH, "temp.rst"), 'w') as f: @@ -327,6 +329,7 @@ def main(): argparser.add_argument('--docstring', metavar='FILENAME', type=str, + nargs='*', default=None, help=('method or function name to compile, ' 'e.g. "DataFrame.join"')) @@ -346,8 +349,13 @@ def main(): _generate_index(single_doc='temp.rst') DocBuilder(args.num_jobs, exclude_patterns).build_docstring() # open generated page in new browser tab - url = os.path.join("file://", DOC_PATH, "build", "html", - "generated_temp", "{}.html".format(args.docstring)) + if len(args.docstring) == 1: + url = os.path.join( + "file://", DOC_PATH, "build", "html", + "generated_temp", "{}.html".format(args.docstring[0])) + else: + url = os.path.join( + "file://", DOC_PATH, "build", "html", "temp.html") webbrowser.open(url, new=2) # clean-up generated files os.remove('source/temp.rst') From 4e34b09dffc2f142378c1be6d1d20532d6780612 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 22 Feb 2018 21:40:15 +0100 Subject: [PATCH 05/14] always add toctree to temp.rst to avoid warnings --- doc/make.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/make.py b/doc/make.py index a6c277764788e..e28b9dc175607 100755 --- a/doc/make.py +++ b/doc/make.py @@ -85,7 +85,7 @@ def _generate_temp_docstring_file(methods): # # # generate docstring pages # print("Running sphinx-autogen to generate docstring stub pages") - # os.system("sphinx-autogen -t _templates -o source/generated source/*.rst") + # os.system("sphinx-autogen -o source/generated source/*.rst") # create the temporary directory in which we will link the target file try: @@ -113,11 +113,11 @@ def _generate_temp_docstring_file(methods): ================ .. autosummary:: - {toctree} + :toctree: generated_temp/ + {name} - """.format(name='\n '.join(methods), - toctree=':toctree: generated_temp/\n' if not linked else '') + """.format(name='\n '.join(methods)) with open(os.path.join(SOURCE_PATH, "temp.rst"), 'w') as f: f.write(s) From 812eb01835a04eed64a56565fb3258a3fa40de8e Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Fri, 23 Feb 2018 12:13:20 +0000 Subject: [PATCH 06/14] DOC: Implementing single doc building for api docs --- doc/make.py | 237 +++++++++++++--------------------- doc/source/index.rst.template | 5 + 2 files changed, 92 insertions(+), 150 deletions(-) diff --git a/doc/make.py b/doc/make.py index e28b9dc175607..3d4c0b4f4fbc7 100755 --- a/doc/make.py +++ b/doc/make.py @@ -14,12 +14,12 @@ import sys import os import shutil -import subprocess +# import subprocess import argparse from contextlib import contextmanager -import jinja2 -import shutil import webbrowser +import jinja2 +import pandas DOC_PATH = os.path.dirname(os.path.abspath(__file__)) @@ -28,105 +28,6 @@ BUILD_DIRS = ['doctrees', 'html', 'latex', 'plots', '_static', '_templates'] -def _generate_index(include_api=True, single_doc=None): - """Create index.rst file with the specified sections. - - Parameters - ---------- - include_api : bool - Whether API documentation will be built. - single_doc : str or None - If provided, this single documentation page will be generated. - """ - if single_doc is not None: - single_doc = os.path.splitext(os.path.basename(single_doc))[0] - include_api = False - - with open(os.path.join(SOURCE_PATH, 'index.rst.template')) as f: - t = jinja2.Template(f.read()) - - with open(os.path.join(SOURCE_PATH, 'index.rst'), 'w') as f: - f.write(t.render(include_api=include_api, - single_doc=single_doc)) - - -def _generate_exclude_pattern(include_api=True, single_doc=None): - """ - - """ - if not include_api: - rst_files = ['api.rst', 'generated/*.rst'] - elif single_doc is not None: - rst_files = [f for f in os.listdir(SOURCE_PATH) - if ((f.endswith('.rst') or f.endswith('.ipynb')) - and (f != 'index.rst') and (f != single_doc))] - rst_files += ['generated/*.rst'] - else: - rst_files = [] - - exclude_patterns = ",".join( - ['{!r}'.format(i) for i in ['**.ipynb_checkpoints'] + rst_files]) - - return exclude_patterns - - -def _generate_temp_docstring_file(methods): - """ - """ - fnames = [os.path.join(SOURCE_PATH, 'generated', '{}.rst'.format(method)) - for method in methods] - - # # remove the target file to make sure it is updated again (to build - # # latest version) - # try: - # os.remove(fname) - # except OSError: - # pass - # - # # generate docstring pages - # print("Running sphinx-autogen to generate docstring stub pages") - # os.system("sphinx-autogen -o source/generated source/*.rst") - - # create the temporary directory in which we will link the target file - try: - os.makedirs(os.path.join(SOURCE_PATH, 'generated_temp')) - except OSError: - pass - - for fname in fnames: - if os.path.exists(fname): - # link the target file - try: - # os.symlink(fname, os.path.join(SOURCE_PATH, 'generated_temp', - # '{}.rst'.format(method)), - # target_is_directory=False) - # copying to make sure sphinx always thinks it is new - # and needs to be re-generated (to pick source code changes) - shutil.copy(fname, os.path.join(SOURCE_PATH, 'generated_temp')) - linked = True - except: # noqa - linked = False - else: - linked = False - - s = """Built docstrings -================ - -.. autosummary:: - :toctree: generated_temp/ - - {name} - - """.format(name='\n '.join(methods)) - - with open(os.path.join(SOURCE_PATH, "temp.rst"), 'w') as f: - f.write(s) - - if not linked: - print("Running sphinx-autogen on manually created file") - os.system("sphinx-autogen -o source/generated_temp source/temp.rst") - - @contextmanager def _maybe_exclude_notebooks(): """Skip building the notebooks if pandoc is not installed. @@ -137,6 +38,7 @@ def _maybe_exclude_notebooks(): 1. nbconvert isn't installed, or 2. nbconvert is installed, but pandoc isn't """ + # TODO move to exclude_pattern base = os.path.dirname(__file__) notebooks = [os.path.join(base, 'source', nb) for nb in ['style.ipynb']] @@ -175,9 +77,73 @@ class DocBuilder: All public methods of this class can be called as parameters of the script. """ - def __init__(self, num_jobs=1, exclude_patterns=None): + def __init__(self, num_jobs=1, include_api=True, single_doc=None): self.num_jobs = num_jobs - self.exclude_patterns = exclude_patterns + self.include_api = include_api + self.single_doc = single_doc + self.single_doc_type = self._single_doc_type + self.exclude_patterns = self._exclude_patterns + + self._generate_index() + if self.single_doc_type == 'api': + self._run_os('sphinx-autogen', '-o', + 'source/generated_single', 'source/index.rst') + + @property + def _exclude_patterns(self): + """Docs source files that will be excluded from building.""" + # TODO move maybe_exclude_notebooks here + if self.single_doc is not None: + rst_files = [f for f in os.listdir(SOURCE_PATH) + if ((f.endswith('.rst') or f.endswith('.ipynb')) + and (f != 'index.rst') + and (f != self.single_doc))] + rst_files += ['generated/*.rst'] + elif not self.include_api: + rst_files = ['api.rst', 'generated/*.rst'] + else: + rst_files = ['generated_single/*.rst'] + + exclude_patterns = ','.join( + '{!r}'.format(i) for i in ['**.ipynb_checkpoints'] + rst_files) + + return exclude_patterns + + @property + def _single_doc_type(self): + if self.single_doc: + if os.path.exists(os.path.join(SOURCE_PATH, self.single_doc)): + return 'rst' + try: + obj = pandas + for name in self.single_doc.split('.'): + obj = getattr(obj, name) + except AttributeError: + raise ValueError('Single document not understood, it should ' + 'be a file in doc/source/*.rst (e.g. ' + '"contributing.rst" or a pandas function or ' + 'method (e.g. "pandas.DataFrame.head")') + else: + return 'api' + + def _generate_index(self): + """Create index.rst file with the specified sections.""" + if self.single_doc_type == 'rst': + single_doc = os.path.splitext(os.path.basename(self.single_doc))[0] + self.include_api = False + elif self.single_doc_type == 'api' and \ + self.single_doc.startswith('pandas.'): + single_doc = self.single_doc[len('pandas.'):] + else: + single_doc = self.single_doc + + with open(os.path.join(SOURCE_PATH, 'index.rst.template')) as f: + t = jinja2.Template(f.read()) + + with open(os.path.join(SOURCE_PATH, 'index.rst'), 'w') as f: + f.write(t.render(include_api=self.include_api, + single_doc=single_doc, + single_doc_type=self.single_doc_type)) @staticmethod def _create_build_structure(): @@ -201,7 +167,10 @@ def _run_os(*args): -------- >>> DocBuilder()._run_os('python', '--version') """ - subprocess.check_call(args, stderr=subprocess.STDOUT) + # TODO check_call should be more safe, but it fails with + # exclude patterns, needs investigation + # subprocess.check_call(args, stderr=subprocess.STDOUT) + os.system(' '.join(args)) def _sphinx_build(self, kind): """Call sphinx to build documentation. @@ -223,10 +192,16 @@ def _sphinx_build(self, kind): '-j{}'.format(self.num_jobs), '-b{}'.format(kind), '-d{}'.format(os.path.join(BUILD_PATH, 'doctrees')), - # TODO integrate exclude_patterns + '-Dexclude_patterns={}'.format(self.exclude_patterns), SOURCE_PATH, os.path.join(BUILD_PATH, kind)) + def _open_browser(self): + url = os.path.join( + 'file://', DOC_PATH, 'build', 'html', + 'generated_single', '{}.html'.format(self.single_doc)) + webbrowser.open(url, new=2) + def html(self): """Build HTML documentation.""" self._create_build_structure() @@ -236,6 +211,9 @@ def html(self): if os.path.exists(zip_fname): os.remove(zip_fname) + if self.single_doc is not None: + self._open_browser() + def latex(self, force=False): """Build PDF documentation.""" self._create_build_structure() @@ -279,24 +257,6 @@ def zip_html(self): '-q', *fnames) - def build_docstring(self): - """Build single docstring page""" - self._create_build_structure() - - args = ('sphinx-build', - '-bhtml', - '-d{}'.format(os.path.join(BUILD_PATH, 'doctrees')), - '-Dexclude_patterns={}'.format(self.exclude_patterns), - SOURCE_PATH, - os.path.join(BUILD_PATH, 'html'), - os.path.join(SOURCE_PATH, 'temp.rst'), - os.path.join(SOURCE_PATH, 'generated_temp/*.rst'), - ) - # for some reason it does not work with run_os, but it does if I - # directly call the joined command - # self._run_os(*args) - os.system(" ".join(args)) - def main(): cmds = [method for method in dir(DocBuilder) if not method.startswith('_')] @@ -341,32 +301,9 @@ def main(): os.environ['PYTHONPATH'] = args.python_path - if args.docstring is not None: - shutil.rmtree(os.path.join(BUILD_PATH, 'html', 'generated_temp'), - ignore_errors=True) - _generate_temp_docstring_file(args.docstring) - exclude_patterns = _generate_exclude_pattern(single_doc='temp.rst') - _generate_index(single_doc='temp.rst') - DocBuilder(args.num_jobs, exclude_patterns).build_docstring() - # open generated page in new browser tab - if len(args.docstring) == 1: - url = os.path.join( - "file://", DOC_PATH, "build", "html", - "generated_temp", "{}.html".format(args.docstring[0])) - else: - url = os.path.join( - "file://", DOC_PATH, "build", "html", "temp.html") - webbrowser.open(url, new=2) - # clean-up generated files - os.remove('source/temp.rst') - shutil.rmtree(os.path.join(SOURCE_PATH, 'generated_temp'), - ignore_errors=True) - - else: - _generate_index(not args.no_api, args.single) - exclude_patterns = _generate_exclude_pattern( - not args.no_api, args.single) - getattr(DocBuilder(args.num_jobs, exclude_patterns), args.command)() + getattr(DocBuilder(args.num_jobs, + args.no_api, + args.single), args.command)() if __name__ == '__main__': diff --git a/doc/source/index.rst.template b/doc/source/index.rst.template index eff1227e98994..7d11218564e32 100644 --- a/doc/source/index.rst.template +++ b/doc/source/index.rst.template @@ -106,8 +106,13 @@ Some other notes See the package overview for more detail about what's in the library. +{% if single_doc_type == 'api' -%} +.. autosummary:: + :toctree: generated_single/ +{% else -%} .. toctree:: :maxdepth: 4 +{% endif %} {% if single_doc -%} {{ single_doc }} From 1ca28a0a9b6ed758772d05afb1a4994cce5a6000 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Sun, 25 Feb 2018 18:28:46 +0100 Subject: [PATCH 07/14] copy existing generated/ docstring page to make sure we have the correct autodoc template --- doc/make.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/doc/make.py b/doc/make.py index 3d4c0b4f4fbc7..eb0b0625269b9 100755 --- a/doc/make.py +++ b/doc/make.py @@ -126,6 +126,29 @@ def _single_doc_type(self): else: return 'api' + def _copy_generated_docstring(self, method): + """Copy existing generated (from api.rst) docstring page because + this is more correct in certain cases (where a custom autodoc + template is used). + + """ + fname = os.path.join(SOURCE_PATH, 'generated', + 'pandas.{}.rst'.format(method)) + temp_dir = os.path.join(SOURCE_PATH, 'generated_single') + + try: + os.makedirs(temp_dir) + except OSError: + pass + + if os.path.exists(fname): + try: + # copying to make sure sphinx always thinks it is new + # and needs to be re-generated (to pick source code changes) + shutil.copy(fname, temp_dir) + except: # noqa + pass + def _generate_index(self): """Create index.rst file with the specified sections.""" if self.single_doc_type == 'rst': @@ -137,6 +160,9 @@ def _generate_index(self): else: single_doc = self.single_doc + if self.single_doc_type == 'api': + self._copy_generated_docstring(single_doc) + with open(os.path.join(SOURCE_PATH, 'index.rst.template')) as f: t = jinja2.Template(f.read()) From 09c508a45c0e614d1f5525276525e34864a54096 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Sun, 25 Feb 2018 18:36:57 +0100 Subject: [PATCH 08/14] fix default to include API docs --- doc/make.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/make.py b/doc/make.py index eb0b0625269b9..83bc1421d9b5a 100755 --- a/doc/make.py +++ b/doc/make.py @@ -328,7 +328,7 @@ def main(): os.environ['PYTHONPATH'] = args.python_path getattr(DocBuilder(args.num_jobs, - args.no_api, + not args.no_api, args.single), args.command)() From 358e08db258c4409863188f22b76f16e43e63568 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Sun, 25 Feb 2018 22:32:35 +0100 Subject: [PATCH 09/14] enable building all api pages with --single api.rst --- doc/make.py | 24 ++++++++++++++---------- doc/source/index.rst.template | 2 +- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/doc/make.py b/doc/make.py index 83bc1421d9b5a..6e398bd0c516a 100755 --- a/doc/make.py +++ b/doc/make.py @@ -85,7 +85,7 @@ def __init__(self, num_jobs=1, include_api=True, single_doc=None): self.exclude_patterns = self._exclude_patterns self._generate_index() - if self.single_doc_type == 'api': + if self.single_doc_type == 'docstring': self._run_os('sphinx-autogen', '-o', 'source/generated_single', 'source/index.rst') @@ -98,7 +98,8 @@ def _exclude_patterns(self): if ((f.endswith('.rst') or f.endswith('.ipynb')) and (f != 'index.rst') and (f != self.single_doc))] - rst_files += ['generated/*.rst'] + if self.single_doc_type != 'api': + rst_files += ['generated/*.rst'] elif not self.include_api: rst_files = ['api.rst', 'generated/*.rst'] else: @@ -112,6 +113,8 @@ def _exclude_patterns(self): @property def _single_doc_type(self): if self.single_doc: + if self.single_doc == 'api.rst': + return 'api' if os.path.exists(os.path.join(SOURCE_PATH, self.single_doc)): return 'rst' try: @@ -124,7 +127,7 @@ def _single_doc_type(self): '"contributing.rst" or a pandas function or ' 'method (e.g. "pandas.DataFrame.head")') else: - return 'api' + return 'docstring' def _copy_generated_docstring(self, method): """Copy existing generated (from api.rst) docstring page because @@ -154,14 +157,15 @@ def _generate_index(self): if self.single_doc_type == 'rst': single_doc = os.path.splitext(os.path.basename(self.single_doc))[0] self.include_api = False - elif self.single_doc_type == 'api' and \ - self.single_doc.startswith('pandas.'): - single_doc = self.single_doc[len('pandas.'):] - else: - single_doc = self.single_doc - - if self.single_doc_type == 'api': + elif self.single_doc_type == 'docstring': + if self.single_doc.startswith('pandas.'): + single_doc = self.single_doc[len('pandas.'):] + else: + single_doc = self.single_doc + self.include_api = False self._copy_generated_docstring(single_doc) + elif self.single_doc_type == 'api': + single_doc = 'api' with open(os.path.join(SOURCE_PATH, 'index.rst.template')) as f: t = jinja2.Template(f.read()) diff --git a/doc/source/index.rst.template b/doc/source/index.rst.template index 7d11218564e32..cb6cce5edaf79 100644 --- a/doc/source/index.rst.template +++ b/doc/source/index.rst.template @@ -106,7 +106,7 @@ Some other notes See the package overview for more detail about what's in the library. -{% if single_doc_type == 'api' -%} +{% if single_doc_type == 'docstring' -%} .. autosummary:: :toctree: generated_single/ {% else -%} From 2cd4a5d9b9168699aeade9d2d1e86414a26f7174 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Sun, 25 Feb 2018 22:36:47 +0100 Subject: [PATCH 10/14] remove generated_single folder in the end to avoid polluting the directory (and otherwise causing sphinx warnings) --- doc/make.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/make.py b/doc/make.py index 6e398bd0c516a..7441a7332ddda 100755 --- a/doc/make.py +++ b/doc/make.py @@ -19,6 +19,8 @@ from contextlib import contextmanager import webbrowser import jinja2 +import shutil + import pandas @@ -243,6 +245,8 @@ def html(self): if self.single_doc is not None: self._open_browser() + shutil.rmtree(os.path.join(SOURCE_PATH, 'generated_single'), + ignore_errors=True) def latex(self, force=False): """Build PDF documentation.""" From 512d3a69adc2be84e332c5a2a416ea4f0cf805e8 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Sun, 25 Feb 2018 23:04:32 +0100 Subject: [PATCH 11/14] fix opening webbrowser (if not single docstring page) --- doc/make.py | 68 +++++++++++++++++++++++++++++------------------------ 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/doc/make.py b/doc/make.py index 7441a7332ddda..eb58e8fe1dcc8 100755 --- a/doc/make.py +++ b/doc/make.py @@ -19,7 +19,6 @@ from contextlib import contextmanager import webbrowser import jinja2 -import shutil import pandas @@ -82,8 +81,10 @@ class DocBuilder: def __init__(self, num_jobs=1, include_api=True, single_doc=None): self.num_jobs = num_jobs self.include_api = include_api - self.single_doc = single_doc - self.single_doc_type = self._single_doc_type + self.single_doc = None + self.single_doc_type = None + if single_doc is not None: + self._process_single_doc(single_doc) self.exclude_patterns = self._exclude_patterns self._generate_index() @@ -99,7 +100,7 @@ def _exclude_patterns(self): rst_files = [f for f in os.listdir(SOURCE_PATH) if ((f.endswith('.rst') or f.endswith('.ipynb')) and (f != 'index.rst') - and (f != self.single_doc))] + and (f != '{0}.rst'.format(self.single_doc)))] if self.single_doc_type != 'api': rst_files += ['generated/*.rst'] elif not self.include_api: @@ -112,16 +113,23 @@ def _exclude_patterns(self): return exclude_patterns - @property - def _single_doc_type(self): - if self.single_doc: - if self.single_doc == 'api.rst': - return 'api' - if os.path.exists(os.path.join(SOURCE_PATH, self.single_doc)): - return 'rst' + def _process_single_doc(self, single_doc): + """Extract self.single_doc (base name) and self.single_doc_type from + passed single_doc kwarg. + + """ + self.include_api = False + + if single_doc == 'api.rst': + self.single_doc_type = 'api' + self.single_doc = 'api' + elif os.path.exists(os.path.join(SOURCE_PATH, single_doc)): + self.single_doc_type = 'rst' + self.single_doc = os.path.splitext(os.path.basename(single_doc))[0] + elif single_doc is not None: try: obj = pandas - for name in self.single_doc.split('.'): + for name in single_doc.split('.'): obj = getattr(obj, name) except AttributeError: raise ValueError('Single document not understood, it should ' @@ -129,16 +137,20 @@ def _single_doc_type(self): '"contributing.rst" or a pandas function or ' 'method (e.g. "pandas.DataFrame.head")') else: - return 'docstring' + self.single_doc_type = 'docstring' + if single_doc.startswith('pandas.'): + self.single_doc = single_doc[len('pandas.'):] + else: + self.single_doc = single_doc - def _copy_generated_docstring(self, method): + def _copy_generated_docstring(self): """Copy existing generated (from api.rst) docstring page because this is more correct in certain cases (where a custom autodoc template is used). """ fname = os.path.join(SOURCE_PATH, 'generated', - 'pandas.{}.rst'.format(method)) + 'pandas.{}.rst'.format(self.single_doc)) temp_dir = os.path.join(SOURCE_PATH, 'generated_single') try: @@ -156,25 +168,15 @@ def _copy_generated_docstring(self, method): def _generate_index(self): """Create index.rst file with the specified sections.""" - if self.single_doc_type == 'rst': - single_doc = os.path.splitext(os.path.basename(self.single_doc))[0] - self.include_api = False - elif self.single_doc_type == 'docstring': - if self.single_doc.startswith('pandas.'): - single_doc = self.single_doc[len('pandas.'):] - else: - single_doc = self.single_doc - self.include_api = False - self._copy_generated_docstring(single_doc) - elif self.single_doc_type == 'api': - single_doc = 'api' + if self.single_doc_type == 'docstring': + self._copy_generated_docstring() with open(os.path.join(SOURCE_PATH, 'index.rst.template')) as f: t = jinja2.Template(f.read()) with open(os.path.join(SOURCE_PATH, 'index.rst'), 'w') as f: f.write(t.render(include_api=self.include_api, - single_doc=single_doc, + single_doc=self.single_doc, single_doc_type=self.single_doc_type)) @staticmethod @@ -229,9 +231,13 @@ def _sphinx_build(self, kind): os.path.join(BUILD_PATH, kind)) def _open_browser(self): - url = os.path.join( - 'file://', DOC_PATH, 'build', 'html', - 'generated_single', '{}.html'.format(self.single_doc)) + base_url = os.path.join('file://', DOC_PATH, 'build', 'html') + if self.single_doc_type == 'docstring': + url = os.path.join( + base_url, + 'generated_single', 'pandas.{}.html'.format(self.single_doc)) + else: + url = os.path.join(base_url, '{}.html'.format(self.single_doc)) webbrowser.open(url, new=2) def html(self): From 348cd4f6fdbc871c4745c8356c64317c91bc1372 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Sun, 25 Feb 2018 23:19:56 +0100 Subject: [PATCH 12/14] remove --docstring leftover --- doc/make.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/doc/make.py b/doc/make.py index eb58e8fe1dcc8..d350c2acd308f 100755 --- a/doc/make.py +++ b/doc/make.py @@ -326,13 +326,6 @@ def main(): type=str, default=os.path.join(DOC_PATH, '..'), help='path') - argparser.add_argument('--docstring', - metavar='FILENAME', - type=str, - nargs='*', - default=None, - help=('method or function name to compile, ' - 'e.g. "DataFrame.join"')) args = argparser.parse_args() if args.command not in cmds: From 2acc63539bbe5ac20e66fa3c4ab5d6b02b6c7638 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Sun, 25 Feb 2018 23:34:34 +0100 Subject: [PATCH 13/14] Update documentation + allow to not specify .rst in --single --- doc/make.py | 8 ++++++-- doc/source/contributing.rst | 27 +++++++++++++++++---------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/doc/make.py b/doc/make.py index d350c2acd308f..2819a62347627 100755 --- a/doc/make.py +++ b/doc/make.py @@ -126,6 +126,10 @@ def _process_single_doc(self, single_doc): elif os.path.exists(os.path.join(SOURCE_PATH, single_doc)): self.single_doc_type = 'rst' self.single_doc = os.path.splitext(os.path.basename(single_doc))[0] + elif os.path.exists( + os.path.join(SOURCE_PATH, '{}.rst'.format(single_doc))): + self.single_doc_type = 'rst' + self.single_doc = single_doc elif single_doc is not None: try: obj = pandas @@ -320,8 +324,8 @@ def main(): metavar='FILENAME', type=str, default=None, - help=('filename of section to compile, ' - 'e.g. "indexing"')) + help=('filename of section or method name to ' + 'compile, e.g. "indexing", "DataFrame.join"')) argparser.add_argument('--python-path', type=str, default=os.path.join(DOC_PATH, '..'), diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst index 258ab874cafcf..e159af9958fde 100644 --- a/doc/source/contributing.rst +++ b/doc/source/contributing.rst @@ -171,7 +171,7 @@ We'll now kick off a three-step process: # Create and activate the build environment conda env create -f ci/environment-dev.yaml conda activate pandas-dev - + # or with older versions of Anaconda: source activate pandas-dev @@ -388,14 +388,11 @@ If you want to do a full clean build, do:: python make.py html You can tell ``make.py`` to compile only a single section of the docs, greatly -reducing the turn-around time for checking your changes. You will be prompted to -delete ``.rst`` files that aren't required. This is okay because the prior -versions of these files can be checked out from git. However, you must make sure -not to commit the file deletions to your Git repository! +reducing the turn-around time for checking your changes. :: - #omit autosummary and API section + # omit autosummary and API section python make.py clean python make.py --no-api @@ -404,10 +401,20 @@ not to commit the file deletions to your Git repository! python make.py clean python make.py --single indexing -For comparison, a full documentation build may take 10 minutes, a ``-no-api`` build -may take 3 minutes and a single section may take 15 seconds. Subsequent builds, which -only process portions you have changed, will be faster. Open the following file in a web -browser to see the full documentation you just built:: + # compile the reference docs for a single function + python make.py clean + python make.py --single DataFrame.join + +For comparison, a full documentation build may take 15 minutes, but a single +section may take 15 seconds. Subsequent builds, which only process portions +you have changed, will be faster. + +You can also specify to use multiple cores to speed up the documentation build:: + + python make.py html --num-jobs 4 + +Open the following file in a web browser to see the full documentation you +just built:: pandas/docs/build/html/index.html From ebe4a8bd43f5efce8d86603a00cd71489d310aee Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Mon, 26 Feb 2018 19:29:55 +0100 Subject: [PATCH 14/14] lint --- doc/source/conf.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index cf4ac6952dc30..835127e5094e4 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -18,7 +18,6 @@ import importlib import warnings -from pandas.compat import u, PY3 try: raw_input # Python 2 @@ -99,8 +98,8 @@ master_doc = 'index' # General information about the project. -project = u('pandas') -copyright = u('2008-2014, the pandas development team') +project = u'pandas' +copyright = u'2008-2014, the pandas development team' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -311,8 +310,8 @@ # file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'pandas.tex', - u('pandas: powerful Python data analysis toolkit'), - u('Wes McKinney\n\& PyData Development Team'), 'manual'), + u'pandas: powerful Python data analysis toolkit', + u'Wes McKinney\n\& PyData Development Team', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of