Issue #6163: Temporary workaround for legacy setup.py files

.gitignore

@@ -27,6 +27,7 @@ nosetests.xml
 coverage.xml
 *.cover
 tests/data/common_wheels/
+pip-wheel-metadata
 
 # Misc
 *~

news/6163.bugfix

@@ -0,0 +1,3 @@
+When using the setuptools PEP 517 backend as an implicit default fallback,
+inject the source directory as sys.path[0] to more closely match the
+historical direct execution of setup.py scripts.

src/pip/_internal/pyproject.py

@@ -113,9 +113,33 @@ def load_pyproject_toml(
         # In the absence of any explicit backend specification, we
         # assume the setuptools backend, and require wheel and a version
         # of setuptools that supports that backend.
+
+        # Issue #6163 workaround:
+        # A lot of setup.py scripts assume the setup.py directory will be
+        # on sys.path, but the standard setuptools PEP 517 backend doesn't
+        # ensure that. As a temporary workaround, the following changes have
+        # been made to pip:
+        #
+        #   1. The prefix "pip._implicit." is added to the build backend name
+        #      when the backend has been chosen as a fallback by pip rather
+        #      than explicitly by the package developer
+        #   2. The vendored pep517 code has been patched to get the hook
+        #      invocation script to amend sys.path[0] in that case
+        #
+        # These changes can be found by searching for "Issue #6163 workaround"
+        #
+        # To replace the workaround with the real fix once a new version of
+        # setuptools is available with the updated hook:
+        #
+        # 1. Revert all the local changes made to the vendored pep517 code
+        # 2. Change the two references to pip._implicit.setuptools.build_meta
+        #    in this file to instead be to setuptools.build_meta_legacy
+        # 3. Change the minimum setuptools version checks to whichever
+        #    release provides build_meta_legacy
+
         build_system = {
             "requires": ["setuptools>=40.2.0", "wheel"],
-            "build-backend": "setuptools.build_meta",
+            "build-backend": "pip._implicit.setuptools.build_meta",
         }
 
     # If we're using PEP 517, we have build system information (either
@@ -163,7 +187,11 @@ def load_pyproject_toml(
         # execute setup.py, but never considered needing to mention the build
         # tools themselves. The original PEP 518 code had a similar check (but
         # implemented in a different way).
-        backend = "setuptools.build_meta"
+
+        # Issue #6163 workaround:
+        # As above, we add a prefix to the backend name so other parts of the
+        # system can handle the implicit case separately from the explicit one
+        backend = "pip._implicit.setuptools.build_meta"
         check = ["setuptools>=40.2.0", "wheel"]
 
     return (requires, backend, check)

src/pip/_vendor/pep517/_in_process.py

@@ -6,6 +6,10 @@
 - control_dir/input.json:
   - {"kwargs": {...}}
 
+As an interim workaround for https://github.com/pypa/pip/issues/6163, the
+vendored version in pip currently also accepts a PEP517_SYS_PATH_0 environment
+variable, which the wrapper will then insert as sys.path[0]
+
 Results:
 - control_dir/output.json
   - {"return_val": ...}
@@ -21,7 +25,6 @@
 # This is run as a script, not a module, so it can't do a relative import
 import compat
 
-
 class BackendUnavailable(Exception):
     """Raised if we cannot import the backend"""
 
@@ -190,6 +193,13 @@ def main():
         sys.exit("Unknown hook: %s" % hook_name)
     hook = globals()[hook_name]
 
+    # Issue #6163 workaround:
+    # Amend sys.path if the front end has asked the wrapper to do so
+    _path_entry = os.environ.get('PEP517_SYS_PATH_0')
+    if _path_entry:
+        sys.path.insert(0, _path_entry)
+    # End issue #6163 workaround
+
     hook_input = compat.read_json(pjoin(control_dir, 'input.json'))
 
     json_out = {'unsupported': False, 'return_val': None}

src/pip/_vendor/pep517/wrappers.py

@@ -45,6 +45,14 @@ class Pep517HookCaller(object):
     """
     def __init__(self, source_dir, build_backend):
         self.source_dir = abspath(source_dir)
+        # Issue #6163 workaround:
+        # For backwards compatibility with older setup.py scripts that assume
+        # the source directory will be on sys.path, this amends the hook
+        # execution environment in the implicit case
+        __, implicit, build_backend = build_backend.rpartition("pip._implicit.")
+        assert not __ # The implicit marker should only ever be a prefix
+        self._add_source_dir_to_sys_path = implicit
+        # End issue #6163 workaround
         self.build_backend = build_backend
         self._subprocess_runner = default_subprocess_runner
 
@@ -149,10 +157,16 @@ def _call_hook(self, hook_name, kwargs):
                               indent=2)
 
             # Run the hook in a subprocess
+            env = {'PEP517_BUILD_BACKEND': build_backend}
+            # Issue #6163 workaround:
+            # Tell the in-process wrapper to amend sys.path
+            if self._add_source_dir_to_sys_path:
+                env['PEP517_SYS_PATH_0'] = self.source_dir
+            # End issue #6163 workaround
             self._subprocess_runner(
                 [sys.executable, _in_proc_script, hook_name, td],
                 cwd=self.source_dir,
-                extra_environ={'PEP517_BUILD_BACKEND': build_backend}
+                extra_environ=env
             )
 
             data = compat.read_json(pjoin(td, 'output.json'))

tasks/vendoring/patches/pep517.patch

@@ -0,0 +1,74 @@
+diff --git a/src/pip/_vendor/pep517/_in_process.py b/src/pip/_vendor/pep517/_in_process.py
+index d6524b66..e1fb1a14 100644
+--- a/src/pip/_vendor/pep517/_in_process.py
++++ b/src/pip/_vendor/pep517/_in_process.py
+@@ -6,6 +6,10 @@ It expects:
+ - control_dir/input.json:
+   - {"kwargs": {...}}
+
++As an interim workaround for https://github.com/pypa/pip/issues/6163, the
++vendored version in pip currently also accepts a PEP517_SYS_PATH_0 environment
++variable, which the wrapper will then insert as sys.path[0]
++
+ Results:
+ - control_dir/output.json
+   - {"return_val": ...}
+@@ -21,7 +25,6 @@ import sys
+ # This is run as a script, not a module, so it can't do a relative import
+ import compat
+
+-
+ class BackendUnavailable(Exception):
+     """Raised if we cannot import the backend"""
+
+@@ -190,6 +193,13 @@ def main():
+         sys.exit("Unknown hook: %s" % hook_name)
+     hook = globals()[hook_name]
+
++    # Issue #6163 workaround:
++    # Amend sys.path if the front end has asked the wrapper to do so
++    _path_entry = os.environ.get('PEP517_SYS_PATH_0')
++    if _path_entry:
++        sys.path.insert(0, _path_entry)
++    # End issue #6163 workaround
++
+     hook_input = compat.read_json(pjoin(control_dir, 'input.json'))
+
+     json_out = {'unsupported': False, 'return_val': None}
+diff --git a/src/pip/_vendor/pep517/wrappers.py b/src/pip/_vendor/pep517/wrappers.py
+index b14b8991..e256f7c2 100644
+--- a/src/pip/_vendor/pep517/wrappers.py
++++ b/src/pip/_vendor/pep517/wrappers.py
+@@ -45,6 +45,14 @@ class Pep517HookCaller(object):
+     """
+     def __init__(self, source_dir, build_backend):
+         self.source_dir = abspath(source_dir)
++        # Issue #6163 workaround:
++        # For backwards compatibility with older setup.py scripts that assume
++        # the source directory will be on sys.path, this amends the hook
++        # execution environment in the implicit case
++        __, implicit, build_backend = build_backend.rpartition("pip._implicit.")
++        assert not __ # The implicit marker should only ever be a prefix
++        self._add_source_dir_to_sys_path = implicit
++        # End issue #6163 workaround
+         self.build_backend = build_backend
+         self._subprocess_runner = default_subprocess_runner
+
+@@ -149,10 +157,16 @@ class Pep517HookCaller(object):
+                               indent=2)
+
+             # Run the hook in a subprocess
++            env = {'PEP517_BUILD_BACKEND': build_backend}
++            # Issue #6163 workaround:
++            # Tell the in-process wrapper to amend sys.path
++            if self._add_source_dir_to_sys_path:
++                env['PEP517_SYS_PATH_0'] = self.source_dir
++            # End issue #6163 workaround
+             self._subprocess_runner(
+                 [sys.executable, _in_proc_script, hook_name, td],
+                 cwd=self.source_dir,
+-                extra_environ={'PEP517_BUILD_BACKEND': build_backend}
++                extra_environ=env
+             )
+
+             data = compat.read_json(pjoin(td, 'output.json'))

tests/functional/test_pep517.py

@@ -123,3 +123,78 @@ def test_pep517_install_with_no_cache_dir(script, tmpdir, data):
         project_dir,
     )
     result.assert_installed('project', editable=False)
+
+
+def make_pyproject_with_setup(tmpdir, build_system=True, set_backend=True):
+    project_dir = (tmpdir / 'project').mkdir()
+    setup_script = (
+        'from setuptools import setup\n'
+    )
+    expect_script_dir_on_path = True
+    if build_system:
+        buildsys = {
+            'requires': ['setuptools', 'wheel'],
+        }
+        if set_backend:
+            buildsys['build-backend'] = 'setuptools.build_meta'
+            expect_script_dir_on_path = False
+        project_data = pytoml.dumps({'build-system': buildsys})
+    else:
+        project_data = ''
+
+    if expect_script_dir_on_path:
+        setup_script += (
+            'from pep517_test import __version__\n'
+        )
+    else:
+        setup_script += (
+            'try:\n'
+            '    import pep517_test\n'
+            'except ImportError:\n'
+            '    pass\n'
+            'else:\n'
+            '    raise RuntimeError("Source dir incorrectly on sys.path")\n'
+        )
+
+    setup_script += (
+        'setup(name="pep517_test", version="0.1", packages=["pep517_test"])'
+    )
+
+    project_dir.join('pyproject.toml').write(project_data)
+    project_dir.join('setup.py').write(setup_script)
+    package_dir = (project_dir / "pep517_test").mkdir()
+    package_dir.join('__init__.py').write('__version__ = "0.1"')
+    return project_dir, "pep517_test"
+
+
+def test_no_build_system_section(script, tmpdir, data, common_wheels):
+    """Check builds with setup.py, pyproject.toml, but no build-system section.
+    """
+    project_dir, name = make_pyproject_with_setup(tmpdir, build_system=False)
+    result = script.pip(
+        'install', '--no-cache-dir', '--no-index', '-f', common_wheels,
+        project_dir,
+    )
+    result.assert_installed(name, editable=False)
+
+
+def test_no_build_backend_entry(script, tmpdir, data, common_wheels):
+    """Check builds with setup.py, pyproject.toml, but no build-backend-entry.
+    """
+    project_dir, name = make_pyproject_with_setup(tmpdir, set_backend=False)
+    result = script.pip(
+        'install', '--no-cache-dir', '--no-index', '-f', common_wheels,
+        project_dir,
+    )
+    result.assert_installed(name, editable=False)
+
+
+def test_explicit_setuptools_backend(script, tmpdir, data, common_wheels):
+    """Check builds with setup.py, pyproject.toml, and a build-system entry.
+    """
+    project_dir, name = make_pyproject_with_setup(tmpdir)
+    result = script.pip(
+        'install', '--no-cache-dir', '--no-index', '-f', common_wheels,
+        project_dir,
+    )
+    result.assert_installed(name, editable=False)