DOC: Doc build for a single doc made much faster, and clean up

doc/make.py

@@ -15,11 +15,9 @@
 import sys
 import os
 import shutil
-# import subprocess
+import subprocess
 import argparse
-from contextlib import contextmanager
 import webbrowser
-import jinja2
 
 
 DOC_PATH = os.path.dirname(os.path.abspath(__file__))
@@ -28,179 +26,68 @@
 BUILD_DIRS = ['doctrees', 'html', 'latex', 'plots', '_static', '_templates']
 
 
-@contextmanager
-def _maybe_exclude_notebooks():
-    """Skip building the notebooks if pandoc is not installed.
-
-    This assumes that nbsphinx is installed.
-
-    Skip notebook conversion if:
-    1. nbconvert isn't installed, or
-    2. nbconvert is installed, but pandoc isn't
-    """
-    # TODO move to exclude_pattern
-    base = os.path.dirname(__file__)
-    notebooks = [os.path.join(base, 'source', nb)
-                 for nb in ['style.ipynb']]
-    contents = {}
-
-    def _remove_notebooks():
-        for nb in notebooks:
-            with open(nb, 'rt') as f:
-                contents[nb] = f.read()
-            os.remove(nb)
-
-    try:
-        import nbconvert
-    except ImportError:
-        sys.stderr.write('Warning: nbconvert not installed. '
-                         'Skipping notebooks.\n')
-        _remove_notebooks()
-    else:
-        try:
-            nbconvert.utils.pandoc.get_pandoc_version()
-        except nbconvert.utils.pandoc.PandocMissing:
-            sys.stderr.write('Warning: Pandoc is not installed. '
-                             'Skipping notebooks.\n')
-            _remove_notebooks()
-
-    yield
-
-    for nb, content in contents.items():
-        with open(nb, 'wt') as f:
-            f.write(content)
-
-
 class DocBuilder:
-    """Class to wrap the different commands of this script.
+    """
+    Class to wrap the different commands of this script.
 
     All public methods of this class can be called as parameters of the
     script.
     """
-    def __init__(self, num_jobs=1, include_api=True, single_doc=None,
-                 verbosity=0):
+    def __init__(self, num_jobs=0, include_api=True, single_doc=None,
+                 verbosity=0, warnings_are_errors=False):
         self.num_jobs = num_jobs
-        self.include_api = include_api
         self.verbosity = verbosity
-        self.single_doc = None
-        self.single_doc_type = None
-        if single_doc is not None:
-            self._process_single_doc(single_doc)
-        self.exclude_patterns = self._exclude_patterns
-
-        self._generate_index()
-        if self.single_doc_type == 'docstring':
-            self._run_os('sphinx-autogen', '-o',
-                         'source/generated_single', 'source/index.rst')
-
-    @property
-    def _exclude_patterns(self):
-        """Docs source files that will be excluded from building."""
-        # TODO move maybe_exclude_notebooks here
-        if self.single_doc is not None:
-            rst_files = [f for f in os.listdir(SOURCE_PATH)
-                         if ((f.endswith('.rst') or f.endswith('.ipynb'))
-                             and (f != 'index.rst')
-                             and (f != '{0}.rst'.format(self.single_doc)))]
-            if self.single_doc_type != 'api':
-                rst_files += ['generated/*.rst']
-        elif not self.include_api:
-            rst_files = ['api.rst', 'generated/*.rst']
-        else:
-            rst_files = ['generated_single/*.rst']
-
-        exclude_patterns = ','.join(
-            '{!r}'.format(i) for i in ['**.ipynb_checkpoints'] + rst_files)
-
-        return exclude_patterns
+        self.warnings_are_errors = warnings_are_errors
+
+        if single_doc:
+            single_doc = self._process_single_doc(single_doc)
+            include_api = False
+            os.environ['SPHINX_PATTERN'] = single_doc
+        elif not include_api:
+            os.environ['SPHINX_PATTERN'] = '-api'
+
+        self.single_doc_html = None
+        if single_doc and single_doc.endswith('.rst'):
+            self.single_doc_html = os.path.splitext(single_doc)[0] + '.html'
+        elif single_doc:
+            self.single_doc_html = 'generated/pandas.{}.html'.format(
+                single_doc)
 
     def _process_single_doc(self, single_doc):
-        """Extract self.single_doc (base name) and self.single_doc_type from
-        passed single_doc kwarg.
-
         """
-        self.include_api = False
-
-        if single_doc == 'api.rst' or single_doc == 'api':
-            self.single_doc_type = 'api'
-            self.single_doc = 'api'
-        elif os.path.exists(os.path.join(SOURCE_PATH, single_doc)):
-            self.single_doc_type = 'rst'
+        Make sure the provided value for --single is a path to an existing
+        .rst/.ipynb file, or a pandas object that can be imported.
 
-            if 'whatsnew' in single_doc:
-                basename = single_doc
+        For example, categorial.rst or pandas.DataFrame.head. For the latter,
+        return the corresponding file path
+        (e.g. generated/pandas.DataFrame.head.rst).
+        """
+        base_name, extension = os.path.splitext(single_doc)
+        if extension in ('.rst', '.ipynb'):
+            if os.path.exists(os.path.join(SOURCE_PATH, single_doc)):
+                return single_doc
             else:
-                basename = os.path.basename(single_doc)
-            self.single_doc = os.path.splitext(basename)[0]
-        elif os.path.exists(
-                os.path.join(SOURCE_PATH, '{}.rst'.format(single_doc))):
-            self.single_doc_type = 'rst'
-            self.single_doc = single_doc
-        elif single_doc is not None:
+                raise FileNotFoundError('File {} not found'.format(single_doc))
+
+        elif single_doc.startswith('pandas.'):
             try:
                 obj = pandas  # noqa: F821
                 for name in single_doc.split('.'):
                     obj = getattr(obj, name)
             except AttributeError:
-                raise ValueError('Single document not understood, it should '
-                                 'be a file in doc/source/*.rst (e.g. '
-                                 '"contributing.rst" or a pandas function or '
-                                 'method (e.g. "pandas.DataFrame.head")')
+                raise ImportError('Could not import {}'.format(single_doc))
             else:
-                self.single_doc_type = 'docstring'
-                if single_doc.startswith('pandas.'):
-                    self.single_doc = single_doc[len('pandas.'):]
-                else:
-                    self.single_doc = single_doc
-
-    def _copy_generated_docstring(self):
-        """Copy existing generated (from api.rst) docstring page because
-        this is more correct in certain cases (where a custom autodoc
-        template is used).
-
-        """
-        fname = os.path.join(SOURCE_PATH, 'generated',
-                             'pandas.{}.rst'.format(self.single_doc))
-        temp_dir = os.path.join(SOURCE_PATH, 'generated_single')
-
-        try:
-            os.makedirs(temp_dir)
-        except OSError:
-            pass
-
-        if os.path.exists(fname):
-            try:
-                # copying to make sure sphinx always thinks it is new
-                # and needs to be re-generated (to pick source code changes)
-                shutil.copy(fname, temp_dir)
-            except:  # noqa
-                pass
-
-    def _generate_index(self):
-        """Create index.rst file with the specified sections."""
-        if self.single_doc_type == 'docstring':
-            self._copy_generated_docstring()
-
-        with open(os.path.join(SOURCE_PATH, 'index.rst.template')) as f:
-            t = jinja2.Template(f.read())
-
-        with open(os.path.join(SOURCE_PATH, 'index.rst'), 'w') as f:
-            f.write(t.render(include_api=self.include_api,
-                             single_doc=self.single_doc,
-                             single_doc_type=self.single_doc_type))
-
-    @staticmethod
-    def _create_build_structure():
-        """Create directories required to build documentation."""
-        for dirname in BUILD_DIRS:
-            try:
-                os.makedirs(os.path.join(BUILD_PATH, dirname))
-            except OSError:
-                pass
+                return single_doc[len('pandas.'):]
+        else:
+            raise ValueError(('--single={} not understood. Value should be a '
+                              'valid path to a .rst or .ipynb file, or a '
+                              'valid pandas object (e.g. categorical.rst or '
+                              'pandas.DataFrame.head)').format(single_doc))
 
     @staticmethod
     def _run_os(*args):
-        """Execute a command as a OS terminal.
+        """
+        Execute a command as a OS terminal.
 
         Parameters
         ----------
@@ -211,16 +98,11 @@ def _run_os(*args):
         --------
         >>> DocBuilder()._run_os('python', '--version')
         """
-        # TODO check_call should be more safe, but it fails with
-        # exclude patterns, needs investigation
-        # subprocess.check_call(args, stderr=subprocess.STDOUT)
-        exit_status = os.system(' '.join(args))
-        if exit_status:
-            msg = 'Command "{}" finished with exit code {}'
-            raise RuntimeError(msg.format(' '.join(args), exit_status))
+        subprocess.check_call(args, stdout=sys.stdout, stderr=sys.stderr)
 
     def _sphinx_build(self, kind):
-        """Call sphinx to build documentation.
+        """
+        Call sphinx to build documentation.
 
         Attribute `num_jobs` from the class is used.
 
@@ -236,43 +118,44 @@ def _sphinx_build(self, kind):
             raise ValueError('kind must be html or latex, '
                              'not {}'.format(kind))
 
-        self._run_os('sphinx-build',
-                     '-j{}'.format(self.num_jobs),
-                     '-b{}'.format(kind),
-                     '-{}'.format(
-                         'v' * self.verbosity) if self.verbosity else '',
-                     '-d"{}"'.format(os.path.join(BUILD_PATH, 'doctrees')),
-                     '-Dexclude_patterns={}'.format(self.exclude_patterns),
-                     '"{}"'.format(SOURCE_PATH),
-                     '"{}"'.format(os.path.join(BUILD_PATH, kind)))
-
-    def _open_browser(self):
-        base_url = os.path.join('file://', DOC_PATH, 'build', 'html')
-        if self.single_doc_type == 'docstring':
-            url = os.path.join(
-                base_url,
-                'generated_single', 'pandas.{}.html'.format(self.single_doc))
-        else:
-            url = os.path.join(base_url, '{}.html'.format(self.single_doc))
+        self.clean()
+
+        cmd = ['sphinx-build', '-b', kind]
+        if self.num_jobs:
+            cmd += ['-j', str(self.num_jobs)]
+        if self.warnings_are_errors:
+            cmd.append('-W')
+        if self.verbosity:
+            cmd.append('-{}'.format('v' * self.verbosity))
+        cmd += ['-d', os.path.join(BUILD_PATH, 'doctrees'),
+                SOURCE_PATH, os.path.join(BUILD_PATH, kind)]
+        cmd = ['sphinx-build', SOURCE_PATH, os.path.join(BUILD_PATH, kind)]
+        self._run_os(*cmd)
+
+    def _open_browser(self, single_doc_html):
+        """
+        Open a browser tab showing single
+        """
+        url = os.path.join('file://', DOC_PATH, 'build', 'html',
+                           single_doc_html)
         webbrowser.open(url, new=2)
 
     def html(self):
-        """Build HTML documentation."""
-        self._create_build_structure()
-        with _maybe_exclude_notebooks():
-            self._sphinx_build('html')
-            zip_fname = os.path.join(BUILD_PATH, 'html', 'pandas.zip')
-            if os.path.exists(zip_fname):
-                os.remove(zip_fname)
-
-        if self.single_doc is not None:
-            self._open_browser()
-            shutil.rmtree(os.path.join(SOURCE_PATH, 'generated_single'),
-                          ignore_errors=True)
+        """
+        Build HTML documentation.
+        """
+        self._sphinx_build('html')
+        zip_fname = os.path.join(BUILD_PATH, 'html', 'pandas.zip')
+        if os.path.exists(zip_fname):
+            os.remove(zip_fname)
+
+        if self.single_doc_html is not None:
+            self._open_browser(self.single_doc_html)
 
     def latex(self, force=False):
-        """Build PDF documentation."""
-        self._create_build_structure()
+        """
+        Build PDF documentation.
+        """
         if sys.platform == 'win32':
             sys.stderr.write('latex build has not been tested on windows\n')
         else:
@@ -289,18 +172,24 @@ def latex(self, force=False):
                 self._run_os('make')
 
     def latex_forced(self):
-        """Build PDF documentation with retries to find missing references."""
+        """
+        Build PDF documentation with retries to find missing references.
+        """
         self.latex(force=True)
 
     @staticmethod
     def clean():
-        """Clean documentation generated files."""
+        """
+        Clean documentation generated files.
+        """
         shutil.rmtree(BUILD_PATH, ignore_errors=True)
         shutil.rmtree(os.path.join(SOURCE_PATH, 'generated'),
                       ignore_errors=True)
 
     def zip_html(self):
-        """Compress HTML documentation into a zip file."""
+        """
+        Compress HTML documentation into a zip file.
+        """
         zip_fname = os.path.join(BUILD_PATH, 'html', 'pandas.zip')
         if os.path.exists(zip_fname):
             os.remove(zip_fname)
@@ -326,7 +215,7 @@ def main():
                            help='command to run: {}'.format(', '.join(cmds)))
     argparser.add_argument('--num-jobs',
                            type=int,
-                           default=1,
+                           default=0,
                            help='number of jobs used by sphinx-build')
     argparser.add_argument('--no-api',
                            default=False,
@@ -345,6 +234,9 @@ def main():
     argparser.add_argument('-v', action='count', dest='verbosity', default=0,
                            help=('increase verbosity (can be repeated), '
                                  'passed to the sphinx build command'))
+    argparser.add_argument('--warnings-are-errors', '-W',
+                           action='store_true',
+                           help='fail if warnings are raised')
     args = argparser.parse_args()
 
     if args.command not in cmds:
@@ -364,7 +256,7 @@ def main():
     os.environ['MPLBACKEND'] = 'module://matplotlib.backends.backend_agg'
 
     builder = DocBuilder(args.num_jobs, not args.no_api, args.single,
-                         args.verbosity)
+                         args.verbosity, args.warnings_are_errors)
     getattr(builder, args.command)()
 
 

doc/source/conf.py

@@ -13,23 +13,17 @@
 
 import sys
 import os
-import re
 import inspect
 import importlib
 import logging
 import warnings
-
+import jinja2
 from sphinx.ext.autosummary import _import_by_name
 from numpydoc.docscrape import NumpyDocString
 from numpydoc.docscrape_sphinx import SphinxDocString
 
 logger = logging.getLogger(__name__)
 
-try:
-    raw_input          # Python 2
-except NameError:
-    raw_input = input  # Python 3
-
 # https://github.com/sphinx-doc/sphinx/pull/2325/files
 # Workaround for sphinx-build recursion limit overflow:
 # pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL)
@@ -77,18 +71,47 @@
               ]
 
 exclude_patterns = ['**.ipynb_checkpoints']
-
-with open("index.rst") as f:
-    index_rst_lines = f.readlines()
-
-# only include the slow autosummary feature if we're building the API section
-# of the docs
-
-# JP: added from sphinxdocs
-autosummary_generate = False
-
-if any(re.match(r"\s*api\s*", l) for l in index_rst_lines):
-    autosummary_generate = True
+try:
+    import nbconvert
+except ImportError:
+    logger.warn('nbconvert not installed. Skipping notebooks.')
+    exclude_patterns.append('**/*.ipynb')
+else:
+    try:
+        nbconvert.utils.pandoc.get_pandoc_version()
+    except nbconvert.utils.pandoc.PandocMissing:
+        logger.warn('Pandoc not installed. Skipping notebooks.')
+        exclude_patterns.append('**/*.ipynb')
+
+# sphinx_pattern can be '-api' to exclude the API pages,
+# the path to a file, or a Python object
+# (e.g. '10min.rst' or 'pandas.DataFrame.head')
+source_path = os.path.dirname(os.path.abspath(__file__))
+pattern = os.environ.get('SPHINX_PATTERN')
+if pattern:
+    for dirname, dirs, fnames in os.walk(source_path):
+        for fname in fnames:
+            if os.path.splitext(fname)[-1] in ('.rst', '.ipynb'):
+                fname = os.path.relpath(os.path.join(dirname, fname),
+                                        source_path)
+
+                if (fname == 'index.rst'
+                        and os.path.abspath(dirname) == source_path):
+                    continue
+                elif (pattern == '-api'
+                        and (fname == 'api.rst' or dirname == 'generated')):
+                    exclude_patterns.append(fname)
+                elif fname != pattern:
+                    exclude_patterns.append(fname)
+
+with open(os.path.join(source_path, 'index.rst.template')) as f:
+    t = jinja2.Template(f.read())
+with open(os.path.join(source_path, 'index.rst'), 'w') as f:
+    f.write(t.render(include_api=pattern is None,
+                     single_doc=(pattern
+                                 if pattern is not None and pattern != '-api'
+                                 else None)))
+autosummary_generate = True if pattern is None else ['index']
 
 # matplotlib plot directive
 plot_include_source = True
@@ -277,10 +300,11 @@
              "{new}.{method}".format(new=new, method=method))
         )
 
-html_additional_pages = {
-    'generated/' + page[0]: 'api_redirect.html'
-    for page in moved_api_pages
-}
+if pattern is None:
+    html_additional_pages = {
+        'generated/' + page[0]: 'api_redirect.html'
+        for page in moved_api_pages
+    }
 
 
 header = """\
@@ -370,18 +394,17 @@
 # latex_use_modindex = True
 
 
-intersphinx_mapping = {
-    'dateutil': ("https://dateutil.readthedocs.io/en/latest/", None),
-    'matplotlib': ('https://matplotlib.org/', None),
-    'numpy': ('https://docs.scipy.org/doc/numpy/', None),
-    'pandas-gbq': ('https://pandas-gbq.readthedocs.io/en/latest/', None),
-    'py': ('https://pylib.readthedocs.io/en/latest/', None),
-    'python': ('https://docs.python.org/3/', None),
-    'scipy': ('https://docs.scipy.org/doc/scipy/reference/', None),
-    'statsmodels': ('http://www.statsmodels.org/devel/', None),
-}
-import glob
-autosummary_generate = glob.glob("*.rst")
+if pattern is None:
+    intersphinx_mapping = {
+        'dateutil': ("https://dateutil.readthedocs.io/en/latest/", None),
+        'matplotlib': ('https://matplotlib.org/', None),
+        'numpy': ('https://docs.scipy.org/doc/numpy/', None),
+        'pandas-gbq': ('https://pandas-gbq.readthedocs.io/en/latest/', None),
+        'py': ('https://pylib.readthedocs.io/en/latest/', None),
+        'python': ('https://docs.python.org/3/', None),
+        'scipy': ('https://docs.scipy.org/doc/scipy/reference/', None),
+        'statsmodels': ('http://www.statsmodels.org/devel/', None),
+    }
 
 # extlinks alias
 extlinks = {'issue': ('https://github.com/pandas-dev/pandas/issues/%s',

doc/source/index.rst.template

@@ -106,17 +106,21 @@ Some other notes
 See the package overview for more detail about what's in the library.
 
 
-{% if single_doc_type == 'docstring' -%}
+{% if single_doc and single_doc.endswith('.rst') -%}
+.. toctree::
+    :maxdepth: 4
+
+    {{ single_doc[:-4] }}
+{% elif single_doc %}
 .. autosummary::
-    :toctree: generated_single/
+    :toctree: generated/
+
+    {{ single_doc }}
 {% else -%}
 .. toctree::
     :maxdepth: 4
 {% endif %}
 
-    {% if single_doc -%}
-    {{ single_doc }}
-    {% endif -%}
     {% if not single_doc -%}
     What's New <whatsnew/v0.24.0>
     install