diff --git a/.gitignore b/.gitignore index dfa41c0224a..b1fd6887efa 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ nosetests.xml coverage.xml *.cover tests/data/common_wheels/ +pip-wheel-metadata # Misc *~ diff --git a/news/6163.bugfix b/news/6163.bugfix new file mode 100644 index 00000000000..45c269cf643 --- /dev/null +++ b/news/6163.bugfix @@ -0,0 +1,3 @@ +When using the setuptools PEP 517 backend as an implicit default fallback, +inject the source directory as sys.path[0] to more closely match the +historical direct execution of setup.py scripts. diff --git a/src/pip/_internal/pyproject.py b/src/pip/_internal/pyproject.py index 1de4b62a75a..bcef5d53283 100644 --- a/src/pip/_internal/pyproject.py +++ b/src/pip/_internal/pyproject.py @@ -113,9 +113,33 @@ def load_pyproject_toml( # In the absence of any explicit backend specification, we # assume the setuptools backend, and require wheel and a version # of setuptools that supports that backend. + + # Issue #6163 workaround: + # A lot of setup.py scripts assume the setup.py directory will be + # on sys.path, but the standard setuptools PEP 517 backend doesn't + # ensure that. As a temporary workaround, the following changes have + # been made to pip: + # + # 1. The prefix "pip._implicit." is added to the build backend name + # when the backend has been chosen as a fallback by pip rather + # than explicitly by the package developer + # 2. The vendored pep517 code has been patched to get the hook + # invocation script to amend sys.path[0] in that case + # + # These changes can be found by searching for "Issue #6163 workaround" + # + # To replace the workaround with the real fix once a new version of + # setuptools is available with the updated hook: + # + # 1. Revert all the local changes made to the vendored pep517 code + # 2. Change the two references to pip._implicit.setuptools.build_meta + # in this file to instead be to setuptools.build_meta_legacy + # 3. Change the minimum setuptools version checks to whichever + # release provides build_meta_legacy + build_system = { "requires": ["setuptools>=40.2.0", "wheel"], - "build-backend": "setuptools.build_meta", + "build-backend": "pip._implicit.setuptools.build_meta", } # If we're using PEP 517, we have build system information (either @@ -163,7 +187,11 @@ def load_pyproject_toml( # execute setup.py, but never considered needing to mention the build # tools themselves. The original PEP 518 code had a similar check (but # implemented in a different way). - backend = "setuptools.build_meta" + + # Issue #6163 workaround: + # As above, we add a prefix to the backend name so other parts of the + # system can handle the implicit case separately from the explicit one + backend = "pip._implicit.setuptools.build_meta" check = ["setuptools>=40.2.0", "wheel"] return (requires, backend, check) diff --git a/src/pip/_vendor/pep517/_in_process.py b/src/pip/_vendor/pep517/_in_process.py index d6524b660a8..e1fb1a1457f 100644 --- a/src/pip/_vendor/pep517/_in_process.py +++ b/src/pip/_vendor/pep517/_in_process.py @@ -6,6 +6,10 @@ - control_dir/input.json: - {"kwargs": {...}} +As an interim workaround for https://github.com/pypa/pip/issues/6163, the +vendored version in pip currently also accepts a PEP517_SYS_PATH_0 environment +variable, which the wrapper will then insert as sys.path[0] + Results: - control_dir/output.json - {"return_val": ...} @@ -21,7 +25,6 @@ # This is run as a script, not a module, so it can't do a relative import import compat - class BackendUnavailable(Exception): """Raised if we cannot import the backend""" @@ -190,6 +193,13 @@ def main(): sys.exit("Unknown hook: %s" % hook_name) hook = globals()[hook_name] + # Issue #6163 workaround: + # Amend sys.path if the front end has asked the wrapper to do so + _path_entry = os.environ.get('PEP517_SYS_PATH_0') + if _path_entry: + sys.path.insert(0, _path_entry) + # End issue #6163 workaround + hook_input = compat.read_json(pjoin(control_dir, 'input.json')) json_out = {'unsupported': False, 'return_val': None} diff --git a/src/pip/_vendor/pep517/wrappers.py b/src/pip/_vendor/pep517/wrappers.py index b14b899150a..e256f7c2ca5 100644 --- a/src/pip/_vendor/pep517/wrappers.py +++ b/src/pip/_vendor/pep517/wrappers.py @@ -45,6 +45,14 @@ class Pep517HookCaller(object): """ def __init__(self, source_dir, build_backend): self.source_dir = abspath(source_dir) + # Issue #6163 workaround: + # For backwards compatibility with older setup.py scripts that assume + # the source directory will be on sys.path, this amends the hook + # execution environment in the implicit case + __, implicit, build_backend = build_backend.rpartition("pip._implicit.") + assert not __ # The implicit marker should only ever be a prefix + self._add_source_dir_to_sys_path = implicit + # End issue #6163 workaround self.build_backend = build_backend self._subprocess_runner = default_subprocess_runner @@ -149,10 +157,16 @@ def _call_hook(self, hook_name, kwargs): indent=2) # Run the hook in a subprocess + env = {'PEP517_BUILD_BACKEND': build_backend} + # Issue #6163 workaround: + # Tell the in-process wrapper to amend sys.path + if self._add_source_dir_to_sys_path: + env['PEP517_SYS_PATH_0'] = self.source_dir + # End issue #6163 workaround self._subprocess_runner( [sys.executable, _in_proc_script, hook_name, td], cwd=self.source_dir, - extra_environ={'PEP517_BUILD_BACKEND': build_backend} + extra_environ=env ) data = compat.read_json(pjoin(td, 'output.json')) diff --git a/tasks/vendoring/patches/pep517.patch b/tasks/vendoring/patches/pep517.patch new file mode 100644 index 00000000000..746a689c9d6 --- /dev/null +++ b/tasks/vendoring/patches/pep517.patch @@ -0,0 +1,74 @@ +diff --git a/src/pip/_vendor/pep517/_in_process.py b/src/pip/_vendor/pep517/_in_process.py +index d6524b66..e1fb1a14 100644 +--- a/src/pip/_vendor/pep517/_in_process.py ++++ b/src/pip/_vendor/pep517/_in_process.py +@@ -6,6 +6,10 @@ It expects: + - control_dir/input.json: + - {"kwargs": {...}} + ++As an interim workaround for https://github.com/pypa/pip/issues/6163, the ++vendored version in pip currently also accepts a PEP517_SYS_PATH_0 environment ++variable, which the wrapper will then insert as sys.path[0] ++ + Results: + - control_dir/output.json + - {"return_val": ...} +@@ -21,7 +25,6 @@ import sys + # This is run as a script, not a module, so it can't do a relative import + import compat + +- + class BackendUnavailable(Exception): + """Raised if we cannot import the backend""" + +@@ -190,6 +193,13 @@ def main(): + sys.exit("Unknown hook: %s" % hook_name) + hook = globals()[hook_name] + ++ # Issue #6163 workaround: ++ # Amend sys.path if the front end has asked the wrapper to do so ++ _path_entry = os.environ.get('PEP517_SYS_PATH_0') ++ if _path_entry: ++ sys.path.insert(0, _path_entry) ++ # End issue #6163 workaround ++ + hook_input = compat.read_json(pjoin(control_dir, 'input.json')) + + json_out = {'unsupported': False, 'return_val': None} +diff --git a/src/pip/_vendor/pep517/wrappers.py b/src/pip/_vendor/pep517/wrappers.py +index b14b8991..e256f7c2 100644 +--- a/src/pip/_vendor/pep517/wrappers.py ++++ b/src/pip/_vendor/pep517/wrappers.py +@@ -45,6 +45,14 @@ class Pep517HookCaller(object): + """ + def __init__(self, source_dir, build_backend): + self.source_dir = abspath(source_dir) ++ # Issue #6163 workaround: ++ # For backwards compatibility with older setup.py scripts that assume ++ # the source directory will be on sys.path, this amends the hook ++ # execution environment in the implicit case ++ __, implicit, build_backend = build_backend.rpartition("pip._implicit.") ++ assert not __ # The implicit marker should only ever be a prefix ++ self._add_source_dir_to_sys_path = implicit ++ # End issue #6163 workaround + self.build_backend = build_backend + self._subprocess_runner = default_subprocess_runner + +@@ -149,10 +157,16 @@ class Pep517HookCaller(object): + indent=2) + + # Run the hook in a subprocess ++ env = {'PEP517_BUILD_BACKEND': build_backend} ++ # Issue #6163 workaround: ++ # Tell the in-process wrapper to amend sys.path ++ if self._add_source_dir_to_sys_path: ++ env['PEP517_SYS_PATH_0'] = self.source_dir ++ # End issue #6163 workaround + self._subprocess_runner( + [sys.executable, _in_proc_script, hook_name, td], + cwd=self.source_dir, +- extra_environ={'PEP517_BUILD_BACKEND': build_backend} ++ extra_environ=env + ) + + data = compat.read_json(pjoin(td, 'output.json')) diff --git a/tests/functional/test_pep517.py b/tests/functional/test_pep517.py index a1a45d27bb9..2203494a35a 100644 --- a/tests/functional/test_pep517.py +++ b/tests/functional/test_pep517.py @@ -123,3 +123,78 @@ def test_pep517_install_with_no_cache_dir(script, tmpdir, data): project_dir, ) result.assert_installed('project', editable=False) + + +def make_pyproject_with_setup(tmpdir, build_system=True, set_backend=True): + project_dir = (tmpdir / 'project').mkdir() + setup_script = ( + 'from setuptools import setup\n' + ) + expect_script_dir_on_path = True + if build_system: + buildsys = { + 'requires': ['setuptools', 'wheel'], + } + if set_backend: + buildsys['build-backend'] = 'setuptools.build_meta' + expect_script_dir_on_path = False + project_data = pytoml.dumps({'build-system': buildsys}) + else: + project_data = '' + + if expect_script_dir_on_path: + setup_script += ( + 'from pep517_test import __version__\n' + ) + else: + setup_script += ( + 'try:\n' + ' import pep517_test\n' + 'except ImportError:\n' + ' pass\n' + 'else:\n' + ' raise RuntimeError("Source dir incorrectly on sys.path")\n' + ) + + setup_script += ( + 'setup(name="pep517_test", version="0.1", packages=["pep517_test"])' + ) + + project_dir.join('pyproject.toml').write(project_data) + project_dir.join('setup.py').write(setup_script) + package_dir = (project_dir / "pep517_test").mkdir() + package_dir.join('__init__.py').write('__version__ = "0.1"') + return project_dir, "pep517_test" + + +def test_no_build_system_section(script, tmpdir, data, common_wheels): + """Check builds with setup.py, pyproject.toml, but no build-system section. + """ + project_dir, name = make_pyproject_with_setup(tmpdir, build_system=False) + result = script.pip( + 'install', '--no-cache-dir', '--no-index', '-f', common_wheels, + project_dir, + ) + result.assert_installed(name, editable=False) + + +def test_no_build_backend_entry(script, tmpdir, data, common_wheels): + """Check builds with setup.py, pyproject.toml, but no build-backend-entry. + """ + project_dir, name = make_pyproject_with_setup(tmpdir, set_backend=False) + result = script.pip( + 'install', '--no-cache-dir', '--no-index', '-f', common_wheels, + project_dir, + ) + result.assert_installed(name, editable=False) + + +def test_explicit_setuptools_backend(script, tmpdir, data, common_wheels): + """Check builds with setup.py, pyproject.toml, and a build-system entry. + """ + project_dir, name = make_pyproject_with_setup(tmpdir) + result = script.pip( + 'install', '--no-cache-dir', '--no-index', '-f', common_wheels, + project_dir, + ) + result.assert_installed(name, editable=False)