More robust setup.py code generation

In setup.py, the code generation learns some new tricks:

- Only write generated files if their content would change.
- If the generated files exist and we cannot import stgit, do nothing.
- Modify sys.path to enable importing stgit.

This specifically resolves the problem of installing stgit from a git
URL, e.g. `pip install git+https://github.com/stacked-git/stgit.git`.

Signed-off-by: Peter Grayson <[email protected]>

Author: jpgrayson

build.py

@@ -61,7 +61,7 @@ def main():
             commands.get_commands(allow_cached=False), sys.stdout
         )
     elif options.py_cmd_list:
-        commands.py_commands(commands.get_commands(allow_cached=False), sys.stdout)
+        commands.write_cmdlist_py(sys.stdout)
     elif options.bash_completion:
         write_bash_completion(sys.stdout)
     elif options.fish_completion:

contrib/release/pkgtest.py

@@ -27,6 +27,8 @@
 [x] Test installing to virtualenv
 [ ] Test installing to system (i.e. w/chroot)
 
+[x] Test install from git URL
+
 Test install sdist tarball with pip
 [x] Ensure generated python is in right place
 [x] Ensure generated completions in right place
@@ -207,6 +209,8 @@ def main():
         repo = prepare_test_repo(base, this_repo)
         prepare_virtual_env(base, python, cache_root)
         test_dirty_version(base, repo)
+        test_install_from_git_url(base, repo, cache_root)
+        test_uninstall_from_venv(base)
         test_sdist_creation(base, repo)
         test_create_git_archive(base, repo)
         test_build(base, repo)
@@ -321,6 +325,39 @@ def test_dirty_version(base, repo):
     check_call(['git', 'clean', '-qfx'], cwd=repo)
 
 
+def test_install_from_git_url(base, repo, cache):
+    log_test('test_install_from_git_url')
+    check_call(
+        [
+            venv_py_exe(base),
+            '-m',
+            'pip',
+            'install',
+            # '--no-clean',
+            '--no-index',
+            '--find-links',
+            cache,
+            f'git+file://{str(repo)}',
+        ],
+        cwd=base,
+    )
+    version = version_from_stg_version(
+        check_output([venv_py_exe(base), '-m', 'stgit', '--version'], cwd=base).decode()
+    )
+    log(f'{version=}')
+
+    share_dir = base / 'venv' / 'share' / 'stgit'
+    assert (share_dir / 'completion' / 'stg.fish').is_file()
+    assert (share_dir / 'completion' / 'stgit.bash').is_file()
+    assert (share_dir / 'completion' / 'stgit.zsh').is_file()
+    assert (share_dir / 'templates' / 'patchmail.tmpl').is_file()
+    assert (share_dir / 'contrib').is_dir()
+    assert (share_dir / 'examples').is_dir()
+
+    # Ensure cmdlist.py was generated
+    check_call([venv_py_exe(base), '-c', 'import stgit.commands.cmdlist'], cwd=base)
+
+
 def test_sdist_creation(base, repo):
     log_test('test_sdist_creation')
     dist = base / 'dist'
@@ -556,6 +593,7 @@ def test_develop_mode(base, repo, cache):
 
 
 def test_uninstall_from_venv(base):
+    log_test('test_uninstall_from_venv')
     check_call(
         [venv_py_exe(base), '-m', 'pip', 'uninstall', '--yes', 'stgit'], cwd=base
     )

setup.py

@@ -1,6 +1,9 @@
 #!/usr/bin/env python3
+import io
 import os
+import sys
 from glob import glob
+from importlib import import_module
 from importlib.util import module_from_spec, spec_from_file_location
 
 from setuptools import setup
@@ -50,34 +53,55 @@ def make_release_tree(self, base_dir, files):
 cmdclass = dict(sdist=_stgit_sdist, build_py=_stgit_build_py)
 
 
-def _generate_code():
-    import stgit.commands
-    from stgit.completion.bash import write_bash_completion
-    from stgit.completion.fish import write_fish_completion
-
+# Carefully perform code generation. First check if generated code
+# exists. Only write-out generated code if the content is different.
+# This accommodates several setup.py contexts, including:
+#  - When running from an sdist where code is already generated
+#  - During pip installation where the stgit package is not in sys.path
+#  - In a git worktree where the generated code may change frequently
+#  - When running from a read-only directory/filesystem
+def _maybe_generate_code():
     base = os.path.abspath(os.path.dirname(__file__))
-
-    commands = stgit.commands.get_commands(allow_cached=False)
-    with open(os.path.join(base, 'stgit', 'commands', 'cmdlist.py'), 'w') as f:
-        stgit.commands.py_commands(commands, f)
-
-    completion_dir = os.path.join(base, 'completion')
-    os.makedirs(completion_dir, exist_ok=True)
-    with open(os.path.join(completion_dir, 'stgit.bash'), 'w') as f:
-        write_bash_completion(f)
-    with open(os.path.join(completion_dir, 'stg.fish'), 'w') as f:
-        write_fish_completion(f)
-
-
-# Attempt to generate completion scripts and cmdlist.py. When setup.py
-# is executed in the context of a pip installation, the stgit package
-# will not be available and thus this will fail with an Import error.
-# However, in that case the generated files will already be a part of
-# either the sdist or wheel package that pip is installing.
-try:
-    _generate_code()
-except ImportError:
-    print("Skipping stgit code generation")
+    paths = dict(
+        cmds=os.path.join(base, 'stgit', 'commands', 'cmdlist.py'),
+        bash=os.path.join(base, 'completion', 'stgit.bash'),
+        fish=os.path.join(base, 'completion', 'stg.fish'),
+    )
+
+    existing_content = dict()
+    for k, path in paths.items():
+        if os.path.exists(path):
+            with open(path, 'r') as f:
+                existing_content[k] = f.read()
+        else:
+            existing_content[k] = None
+
+    sys.path.insert(0, base)
+    try:
+        try:
+            gen_funcs = dict(
+                cmds=import_module('stgit.commands').write_cmdlist_py,
+                bash=import_module('stgit.completion.bash').write_bash_completion,
+                fish=import_module('stgit.completion.fish').write_fish_completion,
+            )
+        except ImportError:
+            if all(existing_content.values()):
+                return  # Okay, all generated content exists
+            else:
+                raise RuntimeError('Cannot perform code generation')
+    finally:
+        sys.path.pop(0)
+
+    for k, gen_func in gen_funcs.items():
+        with io.StringIO() as f:
+            gen_func(f)
+            new_content = f.getvalue()
+        if existing_content[k] != new_content:
+            with open(paths[k], 'w') as f:
+                f.write(new_content)
+
+
+_maybe_generate_code()
 
 
 setup(

stgit/commands/__init__.py

@@ -1,3 +1,4 @@
+import importlib
 import os
 
 __copyright__ = """
@@ -20,7 +21,7 @@
 
 def get_command(mod_name):
     """Import and return the given command module."""
-    return __import__(__name__ + '.' + mod_name, globals(), locals(), ['*'])
+    return importlib.import_module(__name__ + '.' + mod_name)
 
 
 _kinds = [
@@ -51,9 +52,7 @@ def get_commands(allow_cached=True):
     one-line command help."""
     if allow_cached:
         try:
-            from stgit.commands.cmdlist import command_list
-
-            return command_list
+            return importlib.import_module('stgit.commands.cmdlist').command_list
         except ImportError:
             # cmdlist.py doesn't exist, so do it the expensive way.
             pass
@@ -63,7 +62,8 @@ def get_commands(allow_cached=True):
     )
 
 
-def py_commands(commands, f):
+def write_cmdlist_py(f):
+    commands = get_commands(allow_cached=False)
     lines = [
         '# This file is autogenerated.',
         '',