PEP 676: Implementation updates (#2208)

Author: AA-Turner

.github/workflows/deploy-gh-pages.yaml

@@ -14,26 +14,16 @@ jobs:
         with:
           fetch-depth: 0  # fetch all history so that last modified date-times are accurate
 
-      - name: 🐍 Set up Python 3.9
+      - name: 🐍 Set up Python 3
         uses: actions/setup-python@v2
         with:
-          python-version: 3.9
-
-      - name: 🧳 Cache pip
-        uses: actions/cache@v2
-        with:
-          # This path is specific to Ubuntu
-          path: ~/.cache/pip
-          # Look to see if there is a cache hit for the corresponding requirements file
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-            ${{ runner.os }}-
+          python-version: 3
+          cache: "pip"
 
       - name: 👷‍ Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          python -m pip install -r requirements.txt
 
       - name: 🔧 Build PEPs
         run: make pages -j$(nproc)

Makefile

@@ -62,7 +62,7 @@ SPHINX_BUILD=$(PYTHON) build.py -j $(SPHINX_JOBS)
 
 # TODO replace `rss:` with this when merged & tested
 pep_rss:
-	$(PYTHON) pep_rss_gen.py
+	$(PYTHON) generate_rss.py
 
 pages: pep_rss
 	$(SPHINX_BUILD) --index-file

build.py

@@ -70,8 +70,6 @@ def create_index_file(html_root: Path, builder: str) -> None:
         warningiserror=args.fail_on_warning,
         parallel=args.jobs,
     )
-    app.builder.copysource = False  # Prevent unneeded source copying - we link direct to GitHub
-    app.builder.search = False  # Disable search
     app.build()
 
     if args.index_file:

conf.py

@@ -43,16 +43,14 @@
 
 # HTML output settings
 html_math_renderer = "maths_to_html"  # Maths rendering
-html_show_copyright = False  # Turn off miscellany
-html_show_sphinx = False
 html_title = "peps.python.org"  # Set <title/>
 
 # Theme settings
 html_theme_path = ["pep_sphinx_extensions"]
 html_theme = "pep_theme"  # The actual theme directory (child of html_theme_path)
 html_use_index = False  # Disable index (we use PEP 0)
-html_sourcelink_suffix = ""  # Fix links to GitHub (don't append .txt)
 html_style = ""  # must be defined here or in theme.conf, but is unused
 html_permalinks = False  # handled in the PEPContents transform
+gettext_auto_build = False  # speed-ups
 
 templates_path = ['pep_sphinx_extensions/pep_theme/templates']  # Theme template relative paths from `confdir`

pep_rss_gen.py renamed to generate_rss.py

@@ -1,13 +1,11 @@
 import datetime
 import email.utils
 from pathlib import Path
-import re
 
-from dateutil import parser
-import docutils.frontend
-import docutils.nodes
-import docutils.parsers.rst
-import docutils.utils
+from docutils import frontend
+from docutils import nodes
+from docutils import utils
+from docutils.parsers import rst
 from feedgen import entry
 from feedgen import feed
 
@@ -44,37 +42,26 @@ def first_line_starting_with(full_path: Path, text: str) -> str:
 
 def pep_creation(full_path: Path) -> datetime.datetime:
     created_str = first_line_starting_with(full_path, "Created:")
-    # bleh, I was hoping to avoid re but some PEPs editorialize on the Created line
-    # (note as of Aug 2020 only PEP 102 has additional content on the Created line)
-    m = re.search(r"(\d+[- ][\w\d]+[- ]\d{2,4})", created_str)
-    if not m:
-        # some older ones have an empty line, that's okay, if it's old we ipso facto don't care about it.
-        # "return None" would make the most sense but datetime objects refuse to compare with that. :-|
-        return datetime.datetime(1900, 1, 1)
-    created_str = m.group(1)
-    try:
-        return parser.parse(created_str, dayfirst=True)
-    except (ValueError, OverflowError):
-        return datetime.datetime(1900, 1, 1)
-
-
-def parse_rst(text: str) -> docutils.nodes.document:
-    rst_parser = docutils.parsers.rst.Parser()
-    components = (docutils.parsers.rst.Parser,)
-    settings = docutils.frontend.OptionParser(components=components).get_default_values()
-    document = docutils.utils.new_document('<rst-doc>', settings=settings)
-    rst_parser.parse(text, document)
+    if full_path.stem == "pep-0102":
+        # remove additional content on the Created line
+        created_str = created_str.split(" ", 1)[0]
+    return datetime.datetime.strptime(created_str, "%d-%b-%Y")
+
+
+def parse_rst(text: str) -> nodes.document:
+    settings = frontend.OptionParser((rst.Parser,)).get_default_values()
+    document = utils.new_document('<rst-doc>', settings=settings)
+    rst.Parser().parse(text, document)
     return document
 
 
 def pep_abstract(full_path: Path) -> str:
     """Return the first paragraph of the PEP abstract"""
     text = full_path.read_text(encoding="utf-8")
-    for node in parse_rst(text):
-        if "<title>Abstract</title>" in str(node):
-            for child in node:
-                if child.tagname == "paragraph":
-                    return child.astext().strip().replace("\n", " ")
+    # TODO replace .traverse with .findall when Sphinx updates to docutils>=0.18.1
+    for node in parse_rst(text).traverse(nodes.section):
+        if node.next_node(nodes.title).astext() == "Abstract":
+            return node.next_node(nodes.paragraph).astext().strip().replace("\n", " ")
     return ""
 
 
@@ -119,7 +106,7 @@ def main():
     Newest Python Enhancement Proposals (PEPs) - Information on new
     language features, and some meta-information like release
     procedure and schedules.
-    """.replace("\n    ", " ").strip()
+    """
 
     # Setup feed generator
     fg = feed.FeedGenerator()
@@ -131,7 +118,7 @@ def main():
     fg.title("Newest Python PEPs")
     fg.link(href="https://www.python.org/dev/peps")
     fg.link(href="https://www.python.org/dev/peps/peps.rss", rel="self")
-    fg.description(desc)
+    fg.description(" ".join(desc.split()))
     fg.lastBuildDate(datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc))
 
     # Add PEP information (ordered by newest first)

pep-0456.txt

@@ -529,7 +529,7 @@ not affect any application code.
 
 The benchmarks were conducted on CPython default branch revision b08868fd5994
 and the PEP repository [pep-456-repos]_. All upstream changes were merged
-into the pep-456 branch. The "performance" CPU governor was configured and
+into the ``pep-456`` branch. The "performance" CPU governor was configured and
 almost all programs were stopped so the benchmarks were able to utilize
 TurboBoost and the CPU caches as much as possible. The raw benchmark results
 of multiple machines and platforms are made available at [benchmarks]_.

pep_sphinx_extensions/__init__.py

@@ -4,11 +4,12 @@
 
 from typing import TYPE_CHECKING
 
+from docutils import nodes
+from docutils.parsers.rst import states
 from docutils.writers.html5_polyglot import HTMLTranslator
-from sphinx.environment import BuildEnvironment
-from sphinx.environment import default_settings
+from sphinx import environment
 
-from pep_sphinx_extensions import config
+from pep_sphinx_extensions.pep_processor.html import pep_html_builder
 from pep_sphinx_extensions.pep_processor.html import pep_html_translator
 from pep_sphinx_extensions.pep_processor.parsing import pep_parser
 from pep_sphinx_extensions.pep_processor.parsing import pep_role
@@ -20,17 +21,16 @@
 # Monkeypatch sphinx.environment.default_settings as Sphinx doesn't allow custom settings or Readers
 # These settings should go in docutils.conf, but are overridden here for now so as not to affect
 # pep2html.py
-default_settings |= {
+environment.default_settings |= {
     "pep_references": True,
     "rfc_references": True,
     "pep_base_url": "",
-    "pep_file_url_template": "pep-%04d.html",
+    "pep_file_url_template": "",
     "_disable_config": True,  # disable using docutils.conf whilst running both PEP generators
 }
 
-# Monkeypatch sphinx.environment.BuildEnvironment.collect_relations, as it takes a long time
-# and we don't use the parent/next/prev functionality
-BuildEnvironment.collect_relations = lambda self: {}
+# TODO replace all inlined PEP and RFC strings with marked-up roles, disable pep_references and rfc_references and remove this monkey-patch
+states.Inliner.pep_reference = lambda s, m, _l: [nodes.reference("", m.group(0), refuri=s.document.settings.pep_url.format(int(m.group("pepnum2"))))]
 
 
 def _depart_maths():
@@ -39,14 +39,17 @@ def _depart_maths():
 
 def _update_config_for_builder(app: Sphinx):
     if app.builder.name == "dirhtml":
-        config.pep_url = f"../{config.pep_stem}"
-        app.env.settings["pep_file_url_template"] = "../pep-%04d"
+        environment.default_settings["pep_url"] = "../pep-{:0>4}"
 
 
 def setup(app: Sphinx) -> dict[str, bool]:
     """Initialize Sphinx extension."""
 
+    environment.default_settings["pep_url"] = "pep-{:0>4}.html"
+
     # Register plugin logic
+    app.add_builder(pep_html_builder.FileBuilder, override=True)
+    app.add_builder(pep_html_builder.DirectoryBuilder, override=True)
     app.add_source_parser(pep_parser.PEPParser)  # Add PEP transforms
     app.add_role("pep", pep_role.PEPRole(), override=True)  # Transform PEP references to links
     app.set_translator("html", pep_html_translator.PEPTranslator)  # Docutils Node Visitor overrides (html builder)

pep_sphinx_extensions/config.py

@@ -0,0 +1,50 @@
+from pathlib import Path
+
+from docutils import nodes
+from docutils.frontend import OptionParser
+from sphinx.builders.html import StandaloneHTMLBuilder
+from sphinx.writers.html import HTMLWriter
+
+from sphinx.builders.dirhtml import DirectoryHTMLBuilder
+
+
+class FileBuilder(StandaloneHTMLBuilder):
+    copysource = False  # Prevent unneeded source copying - we link direct to GitHub
+    search = False  # Disable search
+
+    # Things we don't use but that need to exist:
+    indexer = None
+    relations = {}
+    _script_files = _css_files = []
+
+    def prepare_writing(self, _doc_names: set[str]) -> None:
+        self.docwriter = HTMLWriter(self)
+        _opt_parser = OptionParser([self.docwriter], defaults=self.env.settings, read_config_files=True)
+        self.docsettings = _opt_parser.get_default_values()
+        self.globalcontext = {"docstitle": self.config.html_title, "script_files": [], "css_files": []}
+
+    def get_doc_context(self, docname: str, body: str, _metatags: str) -> dict:
+        """Collect items for the template context of a page."""
+        try:
+            title = self.env.longtitles[docname].astext()
+        except KeyError:
+            title = ""
+
+        # source filename
+        file_is_rst = Path(self.env.srcdir, docname + ".rst").exists()
+        source_name = f"{docname}.rst" if file_is_rst else f"{docname}.txt"
+
+        # local table of contents
+        toc_tree = self.env.tocs[docname].deepcopy()
+        for node in toc_tree.traverse(nodes.reference):
+            node["refuri"] = node["anchorname"] or '#'  # fix targets
+        toc = self.render_partial(toc_tree)["fragment"]
+
+        return {"title": title, "sourcename": source_name, "toc": toc, "body": body}
+
+
+class DirectoryBuilder(FileBuilder):
+    # sync all overwritten things from DirectoryHTMLBuilder
+    name = DirectoryHTMLBuilder.name
+    get_target_uri = DirectoryHTMLBuilder.get_target_uri
+    get_outfilename = DirectoryHTMLBuilder.get_outfilename

pep_sphinx_extensions/pep_processor/html/pep_html_builder.py

@@ -57,26 +57,34 @@ def depart_paragraph(self, _: nodes.paragraph) -> None:
         """Add corresponding end tag from `visit_paragraph`."""
         self.body.append(self.context.pop())
 
+    def visit_footnote_reference(self, node):
+        self.body.append(self.starttag(node, "a", suffix="[",
+            CLASS=f"footnote-reference {self.settings.footnote_references}",
+            href=f"#{node['refid']}"
+        ))
+
+    def depart_footnote_reference(self, node):
+        self.body.append(']</a>')
+
+    def visit_label(self, node):
+        # pass parent node to get id into starttag:
+        self.body.append(self.starttag(node.parent, "dt", suffix="[", CLASS="label"))
+
+        # footnote/citation backrefs:
+        back_refs = node.parent["backrefs"]
+        if self.settings.footnote_backlinks and len(back_refs) == 1:
+            self.body.append(f'<a href="#{back_refs[0]}">')
+            self.context.append(f"</a>]")
+        else:
+            self.context.append("]")
+
     def depart_label(self, node) -> None:
         """PEP link/citation block cleanup with italicised backlinks."""
-        if not self.settings.footnote_backlinks:
-            self.body.append("</span>")
-            self.body.append("</dt>\n<dd>")
-            return
-
-        # If only one reference to this footnote
-        back_references = node.parent["backrefs"]
-        if len(back_references) == 1:
-            self.body.append("</a>")
-
-        # Close the tag
-        self.body.append("</span>")
-
-        # If more than one reference
-        if len(back_references) > 1:
-            back_links = [f"<a href='#{ref}'>{i}</a>" for i, ref in enumerate(back_references, start=1)]
-            back_links_str = ", ".join(back_links)
-            self.body.append(f"<span class='fn-backref''><em> ({back_links_str}) </em></span>")
+        self.body.append(self.context.pop())
+        back_refs = node.parent["backrefs"]
+        if self.settings.footnote_backlinks and len(back_refs) > 1:
+            back_links = ", ".join(f"<a href='#{ref}'>{i}</a>" for i, ref in enumerate(back_refs, start=1))
+            self.body.append(f"<em> ({back_links}) </em>")
 
         # Close the def tags
         self.body.append("</dt>\n<dd>")

pep_sphinx_extensions/pep_processor/html/pep_html_translator.py

@@ -1,15 +1,14 @@
 from sphinx import roles
 
-from pep_sphinx_extensions import config
-
 
 class PEPRole(roles.PEP):
     """Override the :pep: role"""
+    # TODO override the entire thing (internal should be True)
 
     def build_uri(self) -> str:
         """Get PEP URI from role text."""
         pep_str, _, fragment = self.target.partition("#")
-        pep_base = config.pep_url.format(int(pep_str))
+        pep_base = self.inliner.document.settings.pep_url.format(int(pep_str))
         if fragment:
             return f"{pep_base}#{fragment}"
         return pep_base