diff --git a/.azure-pipelines/steps/run-tests-windows.yml b/.azure-pipelines/steps/run-tests-windows.yml index 39282a3cc80..97cdd6039b4 100644 --- a/.azure-pipelines/steps/run-tests-windows.yml +++ b/.azure-pipelines/steps/run-tests-windows.yml @@ -24,7 +24,7 @@ steps: Set-Acl "R:\Temp" $acl displayName: Set RAMDisk Permissions -- bash: pip install --upgrade 'virtualenv<20' setuptools tox +- bash: pip install --upgrade 'virtualenv' setuptools tox displayName: Install Tox - script: tox -e py -- -m unit -n auto --junit-xml=junit/unit-test.xml diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml index 5b9a9c50c89..ce321aee52f 100644 --- a/.azure-pipelines/steps/run-tests.yml +++ b/.azure-pipelines/steps/run-tests.yml @@ -4,7 +4,7 @@ steps: inputs: versionSpec: '$(python.version)' -- bash: pip install --upgrade 'virtualenv<20' setuptools tox +- bash: pip install --upgrade 'virtualenv' setuptools tox displayName: Install Tox - script: tox -e py -- -m unit -n auto --junit-xml=junit/unit-test.xml diff --git a/src/pip/_internal/utils/virtualenv.py b/src/pip/_internal/utils/virtualenv.py index 4a7812873b3..649de3681c1 100644 --- a/src/pip/_internal/utils/virtualenv.py +++ b/src/pip/_internal/utils/virtualenv.py @@ -4,7 +4,6 @@ import logging import os import re -import site import sys from pip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -90,30 +89,8 @@ def _no_global_under_venv(): return False -def _no_global_under_regular_virtualenv(): - # type: () -> bool - """Check if "no-global-site-packages.txt" exists beside site.py - - This mirrors logic in pypa/virtualenv for determining whether system - site-packages are visible in the virtual environment. - """ - site_mod_dir = os.path.dirname(os.path.abspath(site.__file__)) - no_global_site_packages_file = os.path.join( - site_mod_dir, 'no-global-site-packages.txt', - ) - return os.path.exists(no_global_site_packages_file) - - def virtualenv_no_global(): # type: () -> bool """Returns a boolean, whether running in venv with no system site-packages. """ - # PEP 405 compliance needs to be checked first since virtualenv >=20 would - # return True for both checks, but is only able to use the PEP 405 config. - if _running_under_venv(): - return _no_global_under_venv() - - if _running_under_regular_virtualenv(): - return _no_global_under_regular_virtualenv() - - return False + return _no_global_under_venv() diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 17a72bca82e..bc491463d61 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -1589,7 +1589,9 @@ def test_install_compatible_python_requires(script): @pytest.mark.network -def test_install_pep508_with_url(script): +def test_install_pep508_with_url(script, virtualenv): + virtualenv.user_site_packages = False + res = script.pip( 'install', '--no-index', 'packaging@https://files.pythonhosted.org/packages/2f/2b/' @@ -1601,7 +1603,9 @@ def test_install_pep508_with_url(script): @pytest.mark.network -def test_install_pep508_with_url_in_install_requires(script): +def test_install_pep508_with_url_in_install_requires(script, virtualenv): + virtualenv.user_site_packages = False + pkga_path = create_test_package_with_setup( script, name='pkga', version='1.0', install_requires=[ diff --git a/tests/functional/test_list.py b/tests/functional/test_list.py index 37787246bd0..62dd073aa1b 100644 --- a/tests/functional/test_list.py +++ b/tests/functional/test_list.py @@ -1,5 +1,6 @@ import json import os +import re import pytest @@ -24,8 +25,8 @@ def test_basic_list(simple_script): """ result = simple_script.pip('list') - assert 'simple 1.0' in result.stdout, str(result) - assert 'simple2 3.0' in result.stdout, str(result) + assert re.search(r'simple\s+1\.0', result.stdout) is not None, str(result) + assert re.search(r'simple2\s+3\.0', result.stdout) is not None, str(result) def test_verbose_flag(simple_script): @@ -37,8 +38,8 @@ def test_verbose_flag(simple_script): assert 'Version' in result.stdout, str(result) assert 'Location' in result.stdout, str(result) assert 'Installer' in result.stdout, str(result) - assert 'simple 1.0' in result.stdout, str(result) - assert 'simple2 3.0' in result.stdout, str(result) + assert re.search(r'simple\s+1\.0', result.stdout) is not None, str(result) + assert re.search(r'simple2\s+3\.0', result.stdout) is not None, str(result) def test_columns_flag(simple_script): @@ -49,8 +50,8 @@ def test_columns_flag(simple_script): assert 'Package' in result.stdout, str(result) assert 'Version' in result.stdout, str(result) assert 'simple (1.0)' not in result.stdout, str(result) - assert 'simple 1.0' in result.stdout, str(result) - assert 'simple2 3.0' in result.stdout, str(result) + assert re.search(r'simple\s+1\.0', result.stdout) is not None, str(result) + assert re.search(r'simple2\s+3\.0', result.stdout) is not None, str(result) def test_format_priority(simple_script): @@ -61,16 +62,16 @@ def test_format_priority(simple_script): expect_stderr=True) assert 'simple==1.0' in result.stdout, str(result) assert 'simple2==3.0' in result.stdout, str(result) - assert 'simple 1.0' not in result.stdout, str(result) - assert 'simple2 3.0' not in result.stdout, str(result) + assert re.search(r'simple\s+1\.0', result.stdout) is None, str(result) + assert re.search(r'simple2\s+3\.0', result.stdout) is None, str(result) result = simple_script.pip('list', '--format=freeze', '--format=columns') assert 'Package' in result.stdout, str(result) assert 'Version' in result.stdout, str(result) assert 'simple==1.0' not in result.stdout, str(result) assert 'simple2==3.0' not in result.stdout, str(result) - assert 'simple 1.0' in result.stdout, str(result) - assert 'simple2 3.0' in result.stdout, str(result) + assert re.search(r'simple\s+1\.0', result.stdout) is not None, str(result) + assert re.search(r'simple2\s+3\.0', result.stdout) is not None, str(result) def test_local_flag(simple_script): @@ -91,7 +92,7 @@ def test_local_columns_flag(simple_script): assert 'Package' in result.stdout assert 'Version' in result.stdout assert 'simple (1.0)' not in result.stdout - assert 'simple 1.0' in result.stdout, str(result) + assert re.search(r'simple\s+1\.0', result.stdout) is not None, str(result) @pytest.mark.network @@ -177,7 +178,7 @@ def test_uptodate_columns_flag(script, data): assert 'Location' in result.stdout # editables included assert 'pip-test-package (0.1.1,' not in result.stdout assert 'pip-test-package 0.1.1' in result.stdout, str(result) - assert 'simple2 3.0' in result.stdout, str(result) + assert re.search(r'simple2\s+3\.0', result.stdout) is not None, str(result) @pytest.mark.network diff --git a/tests/lib/venv.py b/tests/lib/venv.py index cc94e29f254..eae5e2c0d36 100644 --- a/tests/lib/venv.py +++ b/tests/lib/venv.py @@ -2,7 +2,7 @@ import compileall import shutil -import sys +import sysconfig import textwrap import six @@ -32,16 +32,15 @@ def __init__(self, location, template=None, venv_type=None): self._create() def _update_paths(self): - home, lib, inc, bin = _virtualenv.path_locations(self.location) - self.bin = Path(bin) - self.site = Path(lib) / 'site-packages' - # Workaround for https://github.com/pypa/virtualenv/issues/306 - if hasattr(sys, "pypy_version_info"): - version_fmt = '{0}' if six.PY3 else '{0}.{1}' - version_dir = version_fmt.format(*sys.version_info) - self.lib = Path(home, 'lib-python', version_dir) - else: - self.lib = Path(lib) + paths = sysconfig.get_paths(vars={ + "installed_base": self.location, + "installed_platbase": self.location, + "base": self.location, + "platbase": self.location, + }) + self.bin = Path(paths["scripts"]) + self.site = Path(paths["purelib"]) + self.lib = Path(paths["stdlib"]) def __repr__(self): return "".format(self.location) @@ -50,10 +49,6 @@ def _create(self, clear=False): if clear: shutil.rmtree(self.location) if self._template: - # On Windows, calling `_virtualenv.path_locations(target)` - # will have created the `target` directory... - if sys.platform == 'win32' and self.location.exists(): - self.location.rmdir() # Clone virtual environment from template. shutil.copytree( self._template.location, self.location, symlinks=True @@ -63,88 +58,51 @@ def _create(self, clear=False): else: # Create a new virtual environment. if self._venv_type == 'virtualenv': - _virtualenv.create_environment( + _virtualenv.cli_run([ self.location, - no_pip=True, - no_wheel=True, - no_setuptools=True, - ) - self._fix_virtualenv_site_module() + "--no-pip", + "--no-wheel", + "--no-setuptools", + ]) elif self._venv_type == 'venv': builder = _venv.EnvBuilder() context = builder.ensure_directories(self.location) builder.create_configuration(context) builder.setup_python(context) self.site.mkdir(parents=True, exist_ok=True) + else: + raise ValueError("venv type must be 'virtualenv' or 'venv'") self.sitecustomize = self._sitecustomize self.user_site_packages = self._user_site_packages - def _fix_virtualenv_site_module(self): - # Patch `site.py` so user site work as expected. - site_py = self.lib / 'site.py' - with open(site_py) as fp: - site_contents = fp.read() - for pattern, replace in ( - ( - # Ensure enabling user site does not result in adding - # the real site-packages' directory to `sys.path`. - ( - '\ndef virtual_addsitepackages(known_paths):\n' - ), - ( - '\ndef virtual_addsitepackages(known_paths):\n' - ' return known_paths\n' - ), - ), - ( - # Fix sites ordering: user site must be added before system. - ( - '\n paths_in_sys = addsitepackages(paths_in_sys)' - '\n paths_in_sys = addusersitepackages(paths_in_sys)\n' - ), - ( - '\n paths_in_sys = addusersitepackages(paths_in_sys)' - '\n paths_in_sys = addsitepackages(paths_in_sys)\n' - ), - ), - ): - assert pattern in site_contents - site_contents = site_contents.replace(pattern, replace) - with open(site_py, 'w') as fp: - fp.write(site_contents) - # Make sure bytecode is up-to-date too. - assert compileall.compile_file(str(site_py), quiet=1, force=True) - def _customize_site(self): - contents = '' - if self._venv_type == 'venv': - # Enable user site (before system). - contents += textwrap.dedent( - ''' - import os, site, sys - - if not os.environ.get('PYTHONNOUSERSITE', False): - - site.ENABLE_USER_SITE = True - - # First, drop system-sites related paths. - original_sys_path = sys.path[:] - known_paths = set() - for path in site.getsitepackages(): - site.addsitedir(path, known_paths=known_paths) - system_paths = sys.path[len(original_sys_path):] - for path in system_paths: - if path in original_sys_path: - original_sys_path.remove(path) - sys.path = original_sys_path - - # Second, add user-site. - site.addsitedir(site.getusersitepackages()) - - # Third, add back system-sites related paths. - for path in site.getsitepackages(): - site.addsitedir(path) - ''').strip() + # Enable user site (before system). + contents = textwrap.dedent( + ''' + import os, site, sys + + if not os.environ.get('PYTHONNOUSERSITE', False): + + site.ENABLE_USER_SITE = True + + # First, drop system-sites related paths. + original_sys_path = sys.path[:] + known_paths = set() + for path in site.getsitepackages(): + site.addsitedir(path, known_paths=known_paths) + system_paths = sys.path[len(original_sys_path):] + for path in system_paths: + if path in original_sys_path: + original_sys_path.remove(path) + sys.path = original_sys_path + + # Second, add user-site. + site.addsitedir(site.getusersitepackages()) + + # Third, add back system-sites related paths. + for path in site.getsitepackages(): + site.addsitedir(path) + ''').strip() if self._sitecustomize is not None: contents += '\n' + self._sitecustomize sitecustomize = self.site / "sitecustomize.py" @@ -176,11 +134,13 @@ def user_site_packages(self): @user_site_packages.setter def user_site_packages(self, value): self._user_site_packages = value - if self._venv_type == 'virtualenv': - marker = self.lib / "no-global-site-packages.txt" - if self._user_site_packages: - marker.unlink() - else: - marker.touch() - elif self._venv_type == 'venv': - self._customize_site() + self._customize_site() + + pyvenv_cfg = self.location.joinpath("pyvenv.cfg") + modified_lines = [] + for line in pyvenv_cfg.read_text().splitlines(): + k, v = line.split("=", 1) + if k.strip() == "include-system-site-packages": + line = "{}= {}".format(k, "true" if value else "false") + modified_lines.append(line) + pyvenv_cfg.write_text("\n".join(modified_lines)) diff --git a/tests/unit/test_build_env.py b/tests/unit/test_build_env.py index 1f3b88b4d87..a102f0439a7 100644 --- a/tests/unit/test_build_env.py +++ b/tests/unit/test_build_env.py @@ -168,6 +168,8 @@ def test_build_env_overlay_prefix_has_priority(script): assert result.stdout.strip() == '2.0', str(result) +@pytest.mark.skipif("sys.version_info < (3,)", + reason="Incompatible with the latest virtualenv") @pytest.mark.incompatible_with_test_venv def test_build_env_isolation(script): diff --git a/tests/unit/test_utils_virtualenv.py b/tests/unit/test_utils_virtualenv.py index 625539d7617..ea66e803bdd 100644 --- a/tests/unit/test_utils_virtualenv.py +++ b/tests/unit/test_utils_virtualenv.py @@ -1,5 +1,4 @@ import logging -import site import sys import pytest @@ -7,62 +6,9 @@ from pip._internal.utils import virtualenv -@pytest.mark.parametrize("real_prefix, base_prefix, expected", [ - (None, None, False), # Python 2 base interpreter - (None, sys.prefix, False), # Python 3 base interpreter - (None, "not_sys_prefix", True), # PEP405 venv - (sys.prefix, None, True), # Unknown case - (sys.prefix, sys.prefix, True), # Unknown case - (sys.prefix, "not_sys_prefix", True), # Unknown case - ("not_sys_prefix", None, True), # Python 2 virtualenv - ("not_sys_prefix", sys.prefix, True), # Python 3 virtualenv - ("not_sys_prefix", "not_sys_prefix", True), # Unknown case -]) -def test_running_under_virtualenv( - monkeypatch, real_prefix, base_prefix, expected): - # Use raising=False to prevent AttributeError on missing attribute - if real_prefix is None: - monkeypatch.delattr(sys, "real_prefix", raising=False) - else: - monkeypatch.setattr(sys, "real_prefix", real_prefix, raising=False) - if base_prefix is None: - monkeypatch.delattr(sys, "base_prefix", raising=False) - else: - monkeypatch.setattr(sys, "base_prefix", base_prefix, raising=False) - assert virtualenv.running_under_virtualenv() == expected - - -@pytest.mark.parametrize( - "under_virtualenv, no_global_file, expected", [ - (False, False, False), - (False, True, False), - (True, False, False), - (True, True, True), - ], -) -def test_virtualenv_no_global_with_regular_virtualenv( - monkeypatch, - tmpdir, - under_virtualenv, - no_global_file, - expected, -): - monkeypatch.setattr(virtualenv, '_running_under_venv', lambda: False) - - monkeypatch.setattr(site, '__file__', tmpdir / 'site.py') - monkeypatch.setattr( - virtualenv, '_running_under_regular_virtualenv', - lambda: under_virtualenv, - ) - if no_global_file: - (tmpdir / 'no-global-site-packages.txt').touch() - - assert virtualenv.virtualenv_no_global() == expected - - @pytest.mark.parametrize( "pyvenv_cfg_lines, under_venv, expected, expect_warning", [ - (None, False, False, False), + (None, False, True, True), (None, True, True, True), # this has a warning. ( [ diff --git a/tools/requirements/tests.txt b/tools/requirements/tests.txt index ef87225d6c4..968a76fa491 100644 --- a/tools/requirements/tests.txt +++ b/tools/requirements/tests.txt @@ -13,6 +13,6 @@ pytest-xdist pyyaml scripttest setuptools>=39.2.0 # Needed for `setuptools.wheel.Wheel` support. -https://github.com/pypa/virtualenv/archive/legacy.zip#egg=virtualenv +virtualenv werkzeug==0.16.0 wheel