Merge pull request #7286 from pradyunsg/nicer-nox-setup

Use nox to further automate our release mechanisms

Author: pradyunsg

.azure-pipelines/jobs/package.yml

@@ -15,20 +15,19 @@ jobs:
     inputs:
       versionSpec: '3'
 
-  - bash: pip install twine nox setuptools wheel
-    displayName: Install dependencies
-
-  - bash: nox -s generate_authors
-    displayName: Generate AUTHORS.txt
+  - bash: |
+      git config --global user.email "[email protected]"
+      git config --global user.name "pip"
+    displayName: Setup Git credentials
 
-  - bash: nox -s generate_news -- --yes
-    displayName: Generate NEWS.rst
+  - bash: pip install nox
+    displayName: Install dependencies
 
-  - bash: python setup.py sdist bdist_wheel
-    displayName: Create sdist and wheel
+  - bash: nox -s prepare-release -- 99.9
+    displayName: Prepare dummy release
 
-  - bash: twine check dist/*
-    displayName: Check distributions with twine
+  - bash: nox -s build-release -- 99.9
+    displayName: Generate distributions for the dummy release
 
   - task: PublishBuildArtifacts@1
     displayName: 'Publish Artifact: dist'

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
     args: []
   - id: mypy
     name: mypy, for Py2
-    exclude: noxfile.py|docs|tests
+    exclude: noxfile.py|tools/automation/release|docs|tests
     args: ["-2"]
 
 - repo: https://github.com/pre-commit/pygrep-hooks

docs/html/development/release-process.rst

@@ -80,20 +80,13 @@ Creating a new release
 ----------------------
 
 #. Checkout the current pip ``master`` branch.
-#. Ensure you have the latest ``wheel``, ``setuptools``, ``twine`` and ``nox`` packages installed.
-#. Generate a new ``AUTHORS.txt`` (``nox -s generate_authors``) and commit the
-   results.
-#. Bump the version in ``pip/__init__.py`` to the release version and commit
-   the results. Usually this involves dropping just the ``.devN`` suffix on the
-   version.
-#. Generate a new ``NEWS.rst`` (``nox -s generate_news``) and commit the
-   results.
-#. Create a tag at the current commit, of the form ``YY.N``
-   (``git tag YY.N``).
-#. Checkout the tag (``git checkout YY.N``).
-#. Create the distribution files (``python setup.py sdist bdist_wheel``).
-#. Upload the distribution files to PyPI using twine
-   (``twine upload dist/*``).
+#. Ensure you have the latest ``nox`` and ``twine`` installed.
+#. Prepare for release using ``nox -s prepare-release -- YY.N``.
+   This will update the relevant files and tag the correct commit.
+#. Build the release artifacts using ``nox -s build-release -- YY.N``.
+   This will checkout the tag, generate the distribution files to be
+   uploaded and checkout the master branch again.
+#. Upload the distribution files to PyPI using ``twine upload dist/*``.
 #. Push all of the changes including the tag.
 #. Regenerate the ``get-pip.py`` script in the `get-pip repository`_ (as
    documented there) and commit the results.
@@ -102,11 +95,6 @@ Creating a new release
    adjusting the versions listed in ``Lib/ensurepip/__init__.py``.
 
 
-.. note::
-
-  Steps 3 to 6 are automated in ``nox -s release -- YY.N`` command.
-
-
 .. note::
 
   If the release dropped the support of an obsolete Python version ``M.m``,
@@ -117,6 +105,7 @@ Creating a new release
 
 
 .. note::
+
   If the ``get-pip.py`` script needs to be updated due to changes in pip internals
   and if the last ``M.m/get-pip.py`` published still uses the default template, make
   sure to first duplicate ``templates/default.py`` as ``templates/pre-YY.N.py``
@@ -135,7 +124,7 @@ order to create one of these the changes should already be merged into the
    command ``git checkout -b release/YY.N.Z+1 YY.N``.
 #. Cherry pick the fixed commits off of the ``master`` branch, fixing any
    conflicts.
-#. Follow the steps 3 to 6 from above (or call ``nox -s release -- YY.N.Z+1``)
+#. Run ``nox -s prepare-release -- YY.N.Z+1``.
 #. Merge master into your release branch and drop the news files that have been
    included in your release (otherwise they would also appear in the ``YY.N+1``
    changelog)

noxfile.py

@@ -4,13 +4,17 @@
 # The following comment should be removed at some point in the future.
 # mypy: disallow-untyped-defs=False
 
-import io
+import glob
 import os
 import shutil
-import subprocess
+import sys
 
 import nox
 
+sys.path.append(".")
+from tools.automation import release  # isort:skip  # noqa
+sys.path.pop()
+
 nox.options.reuse_existing_virtualenvs = True
 nox.options.sessions = ["lint"]
 
@@ -27,29 +31,6 @@
 VERSION_FILE = "src/pip/__init__.py"
 
 
-def get_author_list():
-    """Get the list of authors from Git commits.
-    """
-    # subprocess because session.run doesn't give us stdout
-    result = subprocess.run(
-        ["git", "log", "--use-mailmap", "--format=%aN <%aE>"],
-        capture_output=True,
-        encoding="utf-8",
-    )
-
-    # Create a unique list.
-    authors = []
-    seen_authors = set()
-    for author in result.stdout.splitlines():
-        author = author.strip()
-        if author.lower() not in seen_authors:
-            seen_authors.add(author.lower())
-            authors.append(author)
-
-    # Sort our list of Authors by their case insensitive name
-    return sorted(authors, key=lambda x: x.lower())
-
-
 def run_with_protected_pip(session, *arguments):
     """Do a session.run("pip", *arguments), using a "protected" pip.
 
@@ -81,11 +62,6 @@ def should_update_common_wheels():
     return need_to_repopulate
 
 
-def update_version_file(new_version):
-    with open(VERSION_FILE, "w", encoding="utf-8") as f:
-        f.write('__version__ = "{}"\n'.format(new_version))
-
-
 # -----------------------------------------------------------------------------
 # Development Commands
 #   These are currently prototypes to evaluate whether we want to switch over
@@ -174,70 +150,62 @@ def lint(session):
 # -----------------------------------------------------------------------------
 # Release Commands
 # -----------------------------------------------------------------------------
-@nox.session(python=False)
-def generate_authors(session):
-    # Get our list of authors
-    session.log("Collecting author names")
-    authors = get_author_list()
+@nox.session(name="prepare-release")
+def prepare_release(session):
+    version = release.get_version_from_arguments(session.posargs)
+    if not version:
+        session.error("Usage: nox -s prepare-release -- YY.N[.P]")
+
+    session.log("# Ensure nothing is staged")
+    if release.modified_files_in_git("--staged"):
+        session.error("There are files staged in git")
+
+    session.log(f"# Updating {AUTHORS_FILE}")
+    release.generate_authors(AUTHORS_FILE)
+    if release.modified_files_in_git():
+        release.commit_file(
+            session, AUTHORS_FILE, message=f"Update {AUTHORS_FILE}",
+        )
+    else:
+        session.log(f"# No changes to {AUTHORS_FILE}")
 
-    # Write our authors to the AUTHORS file
-    session.log("Writing AUTHORS")
-    with io.open(AUTHORS_FILE, "w", encoding="utf-8") as fp:
-        fp.write(u"\n".join(authors))
-        fp.write(u"\n")
+    session.log("# Generating NEWS")
+    release.generate_news(session, version)
 
+    session.log(f"# Bumping for release {version}")
+    release.update_version_file(version, VERSION_FILE)
+    release.commit_file(session, VERSION_FILE, message="Bump for release")
 
-@nox.session
-def generate_news(session):
-    session.log("Generating NEWS")
-    session.install("towncrier")
+    session.log("# Tagging release")
+    release.create_git_tag(session, version, message=f"Release {version}")
 
-    # You can pass 2 possible arguments: --draft, --yes
-    session.run("towncrier", *session.posargs)
+    session.log("# Bumping for development")
+    next_dev_version = release.get_next_development_version(version)
+    release.update_version_file(next_dev_version, VERSION_FILE)
+    release.commit_file(session, VERSION_FILE, message="Bump for development")
 
 
-@nox.session
-def release(session):
-    assert len(session.posargs) == 1, "A version number is expected"
-    new_version = session.posargs[0]
-    parts = new_version.split('.')
-    # Expect YY.N or YY.N.P
-    assert 2 <= len(parts) <= 3, parts
-    # Only integers
-    parts = list(map(int, parts))
-    session.log("Generating commits for version {}".format(new_version))
-
-    session.log("Checking that nothing is staged")
-    # Non-zero exit code means that something is already staged
-    session.run("git", "diff", "--staged", "--exit-code", external=True)
-
-    session.log(f"Updating {AUTHORS_FILE}")
-    generate_authors(session)
-    if subprocess.run(["git", "diff", "--exit-code"]).returncode:
-        session.run("git", "add", AUTHORS_FILE, external=True)
-        session.run(
-            "git", "commit", "-m", f"Updating {AUTHORS_FILE}",
-            external=True,
-        )
-    else:
-        session.log(f"No update needed for {AUTHORS_FILE}")
+@nox.session(name="build-release")
+def build_release(session):
+    version = release.get_version_from_arguments(session.posargs)
+    if not version:
+        session.error("Usage: nox -s upload-release -- YY.N[.P]")
 
-    session.log("Generating NEWS")
-    session.install("towncrier")
-    session.run("towncrier", "--yes", "--version", new_version)
+    session.log("# Ensure no files in dist/")
+    if release.have_files_in_folder("dist"):
+        session.error("There are files in dist/. Remove them and try again")
 
-    session.log("Updating version")
-    update_version_file(new_version)
-    session.run("git", "add", VERSION_FILE, external=True)
-    session.run("git", "commit", "-m", f"Release {new_version}", external=True)
+    session.log("# Install dependencies")
+    session.install("setuptools", "wheel", "twine")
 
-    session.log("Tagging release")
-    session.run(
-        "git", "tag", "-m", f"Release {new_version}", new_version,
-        external=True,
-    )
+    session.log("# Checkout the tag")
+    session.run("git", "checkout", version, external=True, silent=True)
+
+    session.log("# Build distributions")
+    session.run("python", "setup.py", "sdist", "bdist_wheel", silent=True)
+
+    session.log("# Verify distributions")
+    session.run("twine", "check", *glob.glob("dist/*"), silent=True)
 
-    next_dev_version = f"{parts[0]}.{parts[1] + 1}.dev0"
-    update_version_file(next_dev_version)
-    session.run("git", "add", VERSION_FILE, external=True)
-    session.run("git", "commit", "-m", "Back to development", external=True)
+    session.log("# Checkout the master branch")
+    session.run("git", "checkout", "master", external=True, silent=True)

tools/automation/release/__init__.py

@@ -0,0 +1,116 @@
+"""Helpers for release automation.
+
+These are written according to the order they are called in.
+"""
+
+import io
+import os
+import subprocess
+from typing import List, Optional, Set
+
+from nox.sessions import Session
+
+
+def get_version_from_arguments(arguments: List[str]) -> Optional[str]:
+    """Checks the arguments passed to `nox -s release`.
+
+    If there is only 1 argument that looks like a pip version, returns that.
+    Otherwise, returns None.
+    """
+    if len(arguments) != 1:
+        return None
+
+    version = arguments[0]
+
+    parts = version.split('.')
+    if not 2 <= len(parts) <= 3:
+        # Not of the form: YY.N or YY.N.P
+        return None
+
+    if not all(part.isdigit() for part in parts):
+        # Not all segments are integers.
+        return None
+
+    # All is good.
+    return version
+
+
+def modified_files_in_git(*args: str) -> int:
+    return subprocess.run(
+        ["git", "diff", "--no-patch", "--exit-code", *args],
+        capture_output=True,
+    ).returncode
+
+
+def get_author_list() -> List[str]:
+    """Get the list of authors from Git commits.
+    """
+    # subprocess because session.run doesn't give us stdout
+    result = subprocess.run(
+        ["git", "log", "--use-mailmap", "--format=%aN <%aE>"],
+        capture_output=True,
+        encoding="utf-8",
+    )
+
+    # Create a unique list.
+    authors = []
+    seen_authors: Set[str] = set()
+    for author in result.stdout.splitlines():
+        author = author.strip()
+        if author.lower() not in seen_authors:
+            seen_authors.add(author.lower())
+            authors.append(author)
+
+    # Sort our list of Authors by their case insensitive name
+    return sorted(authors, key=lambda x: x.lower())
+
+
+def generate_authors(filename: str) -> None:
+    # Get our list of authors
+    authors = get_author_list()
+
+    # Write our authors to the AUTHORS file
+    with io.open(filename, "w", encoding="utf-8") as fp:
+        fp.write(u"\n".join(authors))
+        fp.write(u"\n")
+
+
+def commit_file(session: Session, filename: str, *, message: str) -> None:
+    session.run("git", "add", filename, external=True, silent=True)
+    session.run("git", "commit", "-m", message, external=True, silent=True)
+
+
+def generate_news(session: Session, version: str) -> None:
+    session.install("towncrier")
+    session.run("towncrier", "--yes", "--version", version, silent=True)
+
+
+def update_version_file(version: str, filepath: str) -> None:
+    with open(filepath, "w", encoding="utf-8") as f:
+        f.write('__version__ = "{}"\n'.format(version))
+
+
+def create_git_tag(session: Session, tag_name: str, *, message: str) -> None:
+    session.run(
+        "git", "tag", "-m", message, tag_name, external=True, silent=True,
+    )
+
+
+def get_next_development_version(version: str) -> str:
+    major, minor, *_ = map(int, version.split("."))
+
+    # We have at most 4 releases, starting with 0. Once we reach 3, we'd want
+    # to roll-over to the next year's release numbers.
+    if minor == 3:
+        major += 1
+        minor = 0
+    else:
+        minor += 1
+
+    return f"{major}.{minor}.dev0"
+
+
+def have_files_in_folder(folder_name: str) -> bool:
+    if not os.path.exists(folder_name):
+        return False
+    return bool(os.listdir(folder_name))