diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 921d5e8a6..ac3858966 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.5.0 + rev: v3.0.0 hooks: - id: check-ast - id: check-builtin-literals @@ -15,10 +15,6 @@ repos: rev: v2.0.1 hooks: - id: add-trailing-comma -- repo: https://github.com/asottile/yesqa - rev: v1.1.0 - hooks: - - id: yesqa - repo: https://github.com/asottile/pyupgrade rev: v2.4.1 hooks: diff --git a/src/virtualenv/__main__.py b/src/virtualenv/__main__.py index 5d47db8cd..ce03e4063 100644 --- a/src/virtualenv/__main__.py +++ b/src/virtualenv/__main__.py @@ -39,7 +39,18 @@ def __str__(self): " creator {}".format(ensure_text(str(self.session.creator))), ] if self.session.seeder.enabled: - lines += (" seeder {}".format(ensure_text(str(self.session.seeder))),) + lines += ( + " seeder {}".format(ensure_text(str(self.session.seeder))), + " added seed packages: {}".format( + ", ".join( + sorted( + "==".join(i.stem.split("-")) + for i in self.session.creator.purelib.iterdir() + if i.suffix == ".dist-info" + ), + ), + ), + ) if self.session.activators: lines.append(" activators {}".format(",".join(i.__class__.__name__ for i in self.session.activators))) return os.linesep.join(lines) diff --git a/src/virtualenv/seed/embed/base_embed.py b/src/virtualenv/seed/embed/base_embed.py index bffd49478..55998a3c5 100644 --- a/src/virtualenv/seed/embed/base_embed.py +++ b/src/virtualenv/seed/embed/base_embed.py @@ -12,20 +12,19 @@ @add_metaclass(ABCMeta) class BaseEmbed(Seeder): - packages = ["pip", "setuptools", "wheel"] + def validate_version(self): + pass + + packages = {} def __init__(self, options): super(BaseEmbed, self).__init__(options, enabled=options.no_seed is False) self.download = options.download self.extra_search_dir = [i.resolve() for i in options.extra_search_dir if i.exists()] - def latest_is_none(key): - value = getattr(options, key) - return None if value == "latest" else value - - self.pip_version = latest_is_none("pip") - self.setuptools_version = latest_is_none("setuptools") - self.wheel_version = latest_is_none("wheel") + self.pip_version = options.pip + self.setuptools_version = options.setuptools + self.wheel_version = options.wheel self.no_pip = options.no_pip self.no_setuptools = options.no_setuptools @@ -68,13 +67,13 @@ def add_parser_arguments(cls, parser, interpreter, app_data): help="a path containing wheels the seeder may also use beside bundled (can be set 1+ times)", default=[], ) - for package in cls.packages: + for package, default in cls.packages.items(): parser.add_argument( "--{}".format(package), dest=package, metavar="version", help="{} version to install, bundle for bundled".format(package), - default="latest", + default=default, ) for package in cls.packages: parser.add_argument( diff --git a/src/virtualenv/seed/embed/pip_invoke.py b/src/virtualenv/seed/embed/pip_invoke.py index 74c96f1bb..5fdb46d4d 100644 --- a/src/virtualenv/seed/embed/pip_invoke.py +++ b/src/virtualenv/seed/embed/pip_invoke.py @@ -6,7 +6,7 @@ from virtualenv.discovery.cached_py_info import LogCmd from virtualenv.info import PY3 from virtualenv.seed.embed.base_embed import BaseEmbed -from virtualenv.seed.embed.wheels.acquire import get_bundled_wheel, pip_wheel_env_run +from virtualenv.seed.embed.wheels.acquire import Version, get_bundled_wheel, pip_wheel_env_run from virtualenv.util.subprocess import Popen from virtualenv.util.zipapp import ensure_file_on_disk @@ -17,6 +17,13 @@ class PipInvoke(BaseEmbed): + + packages = { + "pip": Version.latest, + "setuptools": Version.latest, + "wheel": Version.latest, + } + def __init__(self, options): super(PipInvoke, self).__init__(options) @@ -38,10 +45,12 @@ def get_pip_install_cmd(self, exe, version): cmd.append("--no-index") pkg_versions = self.package_version() for key, ver in pkg_versions.items(): - cmd.append("{}{}".format(key, "=={}".format(ver) if ver is not None else "")) + cmd.append("{}{}".format(key, "" if ver == Version.latest else "=={}".format(ver))) with ExitStack() as stack: folders = set() - for context in (ensure_file_on_disk(get_bundled_wheel(p, version), self.app_data) for p in pkg_versions): + for context in ( + ensure_file_on_disk(get_bundled_wheel(p, version).path, self.app_data) for p in pkg_versions + ): folders.add(stack.enter_context(context).parent) for folder in folders: cmd.extend(["--find-links", str(folder)]) diff --git a/src/virtualenv/seed/embed/wheels/acquire.py b/src/virtualenv/seed/embed/wheels/acquire.py index 91b630dd4..940af8241 100644 --- a/src/virtualenv/seed/embed/wheels/acquire.py +++ b/src/virtualenv/seed/embed/wheels/acquire.py @@ -4,142 +4,134 @@ import logging import os import sys -from collections import defaultdict from contextlib import contextmanager -from copy import copy +from operator import attrgetter from shutil import copy2 -from zipfile import ZipFile from virtualenv.info import IS_ZIPAPP +from virtualenv.seed.embed.wheels.util import Wheel from virtualenv.util.path import Path -from virtualenv.util.six import ensure_str, ensure_text +from virtualenv.util.six import ensure_str from virtualenv.util.subprocess import Popen, subprocess from virtualenv.util.zipapp import ensure_file_on_disk from . import BUNDLE_SUPPORT, MAX +from .periodic_update import periodic_update BUNDLE_FOLDER = Path(os.path.abspath(__file__)).parent class WheelDownloadFail(ValueError): - def __init__(self, packages, for_py_version, exit_code, out, err): - self.packages = packages + def __init__(self, distribution, version, for_py_version, exit_code, out, err): + self.distribution = distribution + self.version = version self.for_py_version = for_py_version self.exit_code = exit_code self.out = out.strip() self.err = err.strip() -def get_wheels(for_py_version, wheel_cache_dir, extra_search_dir, packages, app_data, download): +class Version: + #: the version bundled with virtualenv + bundle = "bundle" + + #: the latest version available locally + latest = "latest" + + #: periodically check that newer versions are available, otherwise use bundled + periodic_update = "periodic-update" + + #: custom version handlers + non_version = bundle, latest, periodic_update + + @staticmethod + def of_version(value): + return None if value in Version.non_version else value + + +def get_wheel(distribution, version, for_py_version, search_dirs, download, cache_dir, app_data): + """ + Get a wheel with the given distribution-version-for_py_version trio, by using the extra search dir + download + """ # not all wheels are compatible with all python versions, so we need to py version qualify it - processed = copy(packages) + of_version = Version.of_version(version) # 1. acquire from bundle - acquire_from_bundle(processed, for_py_version, wheel_cache_dir) + wheel = from_bundle(distribution, of_version, for_py_version, cache_dir) + # 2. acquire from extra search dir - acquire_from_dir(processed, for_py_version, wheel_cache_dir, extra_search_dir) - # 3. download from the internet - if download and processed: - download_wheel(processed, for_py_version, wheel_cache_dir, app_data) - - # in the end just get the wheels - wheels = _get_wheels(wheel_cache_dir, packages) - return {p: next(iter(ver_to_files))[1] for p, ver_to_files in wheels.items()} - - -def acquire_from_bundle(packages, for_py_version, to_folder): - for pkg, version in list(packages.items()): - bundle = get_bundled_wheel(pkg, for_py_version) - if bundle is not None: - pkg_version = bundle.stem.split("-")[1] - exact_version_match = version == pkg_version - if exact_version_match: - del packages[pkg] - if version is None or exact_version_match: - bundled_wheel_file = to_folder / bundle.name - if not bundled_wheel_file.exists(): - logging.debug("get bundled wheel %s", bundle) - if IS_ZIPAPP: - from virtualenv.util.zipapp import extract - - extract(bundle, bundled_wheel_file) - else: - copy2(str(bundle), str(bundled_wheel_file)) - - -def get_bundled_wheel(package, version_release): - return BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(version_release, {}) or BUNDLE_SUPPORT[MAX]).get(package) - - -def acquire_from_dir(packages, for_py_version, to_folder, extra_search_dir): - if not packages: - return - for search_dir in extra_search_dir: - wheels = _get_wheels(search_dir, packages) - for pkg, ver_wheels in wheels.items(): - stop = False - for _, filename in ver_wheels: - dest = to_folder / filename.name + if version == Version.latest or wheel is None: + found_wheel = from_dir(distribution, of_version, for_py_version, cache_dir, search_dirs) + if found_wheel is not None and (wheel is None or found_wheel.version_tuple >= wheel.version_tuple): + wheel = found_wheel + + # 3. trigger periodic update + if version == Version.periodic_update: + wheel = periodic_update(distribution, for_py_version, wheel, cache_dir, app_data) + + # 4. download from the internet + if download: + download_wheel(distribution, of_version, for_py_version, cache_dir, app_data) + wheel = _get_wheels(cache_dir, distribution, of_version)[0] # get latest from cache post download + + return wheel + + +def from_bundle(distribution, version, for_py_version, wheel_cache_dir): + """ + Load the bundled wheel to a cache directory. + """ + bundle = get_bundled_wheel(distribution, for_py_version) + if bundle is None: + return None + if version is None or version == bundle.version: + bundled_wheel_file = wheel_cache_dir / bundle.path.name + logging.debug("use bundled wheel %s", bundle) + if not bundled_wheel_file.exists(): + logging.debug("copy bundled wheel to %s", bundled_wheel_file) + if IS_ZIPAPP: + from virtualenv.util.zipapp import extract + + extract(bundle, bundled_wheel_file) + else: + copy2(str(bundle), str(bundled_wheel_file)) + return Wheel(bundled_wheel_file) + + +def get_bundled_wheel(distribution, for_py_version): + path = BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(for_py_version, {}) or BUNDLE_SUPPORT[MAX]).get(distribution) + if path is None: + return None + return Wheel.from_path(path) + + +def from_dir(distribution, version, for_py_version, cache_dir, directories): + """ + Load a compatible wheel from a given folder. + """ + for folder in directories: + for wheel in _get_wheels(folder, distribution, version): + dest = cache_dir / wheel.name + if wheel.support_py(for_py_version): + logging.debug("load extra search dir wheel %s", wheel) if not dest.exists(): - if wheel_support_py(filename, for_py_version): - logging.debug("get extra search dir wheel %s", filename) - copy2(str(filename), str(dest)) - stop = True - else: - stop = True - if stop and packages[pkg] is not None: - del packages[pkg] - break - - -def wheel_support_py(filename, py_version): - name = "{}.dist-info/METADATA".format("-".join(filename.stem.split("-")[0:2])) - with ZipFile(ensure_text(str(filename)), "r") as zip_file: - metadata = zip_file.read(name).decode("utf-8") - marker = "Requires-Python:" - requires = next((i[len(marker) :] for i in metadata.splitlines() if i.startswith(marker)), None) - if requires is None: # if it does not specify a python requires the assumption is compatible - return True - py_version_int = tuple(int(i) for i in py_version.split(".")) - for require in (i.strip() for i in requires.split(",")): - # https://www.python.org/dev/peps/pep-0345/#version-specifiers - for operator, check in [ - ("!=", lambda v: py_version_int != v), - ("==", lambda v: py_version_int == v), - ("<=", lambda v: py_version_int <= v), - (">=", lambda v: py_version_int >= v), - ("<", lambda v: py_version_int < v), - (">", lambda v: py_version_int > v), - ]: - if require.startswith(operator): - ver_str = require[len(operator) :].strip() - version = tuple((int(i) if i != "*" else None) for i in ver_str.split("."))[0:2] - if not check(version): - return False - break - return True - - -def _get_wheels(from_folder, packages): - wheels = defaultdict(list) + copy2(str(wheel.path), str(dest)) + return Wheel(dest) + return None + + +def _get_wheels(from_folder, distribution, version): + wheels = [] for filename in from_folder.iterdir(): - if filename.suffix == ".whl": - data = filename.stem.split("-") - if len(data) >= 2: - pkg, version = data[0:2] - if pkg in packages: - pkg_version = packages[pkg] - if pkg_version is None or pkg_version == version: - wheels[pkg].append((version, filename)) - for versions in wheels.values(): - versions.sort( - key=lambda a: tuple(int(i) if i.isdigit() else i for i in a[0].split(".")), reverse=True, - ) - return wheels - - -def download_wheel(packages, for_py_version, to_folder, app_data): - to_download = list(p if v is None else "{}=={}".format(p, v) for p, v in packages.items()) - logging.debug("download wheels %s", to_download) + wheel = Wheel.from_path(filename) + if wheel and wheel.distribution == distribution: + if version is None or wheel.version == version: + wheels.append(wheel) + return sorted(wheels, key=attrgetter("version_tuple", "distribution"), reverse=True) + + +def download_wheel(distribution, version, for_py_version, to_folder, app_data): + to_download = "{}{}".format(distribution, "" if version is None else "=={}".format(version)) + logging.debug("download wheel %s", to_download) cmd = [ sys.executable, "-m", @@ -152,15 +144,15 @@ def download_wheel(packages, for_py_version, to_folder, app_data): for_py_version, "-d", str(to_folder), + to_download, ] - cmd.extend(to_download) # pip has no interface in python - must be a new sub-process with pip_wheel_env_run("{}.{}".format(*sys.version_info[0:2]), app_data) as env: process = Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) out, err = process.communicate() if process.returncode != 0: - raise WheelDownloadFail(packages, for_py_version, process.returncode, out, err) + raise WheelDownloadFail(distribution, version, for_py_version, process.returncode, out, err) @contextmanager @@ -172,7 +164,7 @@ def pip_wheel_env_run(version, app_data): for k, v in {"PIP_USE_WHEEL": "1", "PIP_USER": "0", "PIP_NO_INPUT": "1"}.items() }, ) - with ensure_file_on_disk(get_bundled_wheel("pip", version), app_data) as pip_wheel_path: + with ensure_file_on_disk(get_bundled_wheel("pip", version).path, app_data) as pip_wheel_path: # put the bundled wheel onto the path, and use it to do the bootstrap operation env[str("PYTHONPATH")] = str(pip_wheel_path) yield env diff --git a/src/virtualenv/seed/embed/wheels/periodic_update.py b/src/virtualenv/seed/embed/wheels/periodic_update.py new file mode 100644 index 000000000..e1a5037d2 --- /dev/null +++ b/src/virtualenv/seed/embed/wheels/periodic_update.py @@ -0,0 +1,213 @@ +""" +Periodically update bundled versions. +""" + +import calendar +import json +import logging +import subprocess +import sys +import tempfile +from contextlib import contextmanager +from datetime import datetime, timedelta +from shutil import copy2 + +from six.moves.urllib.request import urlopen + +from virtualenv.seed.embed.wheels.util import Wheel +from virtualenv.util.lock import ReentrantFileLock, Timeout +from virtualenv.util.path import Path, safe_delete +from virtualenv.util.subprocess import DETACHED_PROCESS, Popen + + +def periodic_update(distribution, for_py_version, wheel, cache_dir, app_data): + if distribution != "pip": + raise RuntimeError("only pip may be periodically updated") + + needs_update = False + with update_log_for_distribution(cache_dir.parent, distribution, no_block=True) as update_log: + if update_log.needs_update: + update_log.started = datetime.now() + update_log.update() + needs_update = True + if needs_update: + trigger_update(distribution, for_py_version, wheel, app_data) + # TODO: only upgrade when an hour passed since periodic update - to keep most CI stable + for version in update_log.versions: + if version.filename is None or version.filename == wheel.name: + continue + updated_wheel = Wheel(cache_dir / version.filename) + # use it only if released long enough time ago - use version number to approximate release + # only use if it has been released for at least 28 days + if datetime.now() - version.release_date > timedelta(days=28): + logging.debug("using periodically updated wheel %s", updated_wheel) + return updated_wheel + return wheel + + +@contextmanager +def update_log_for_distribution(folder, distribution, no_block=False): + root_lock, lock_name = ReentrantFileLock(folder), "{}.update.lock".format(distribution) + try: + with root_lock.lock_for_key(lock_name, no_block=no_block): + update_log = UpdateLog.from_path(folder / "{}.update.json".format(distribution)) + yield update_log + except Timeout: + return + + +DATETIME_FMT = "%Y-%m-%dT%H:%M:%SZ" + + +def dump_datetime(value): + if value is None: + return None + return datetime.strftime(value, DATETIME_FMT) + + +def load_datetime(value): + if value is None: + return None + return datetime.strptime(value, DATETIME_FMT) + + +class NewVersion(object): + def __init__(self, filename, release_date): + self.filename = filename + self.release_date = release_date + + @classmethod + def from_dict(cls, dictionary): + return cls(filename=dictionary["filename"], release_date=load_datetime(dictionary["release_date"])) + + def to_dict(self): + return { + "filename": self.filename, + "release_date": dump_datetime(self.release_date), + } + + +class UpdateLog(object): + def __init__(self, path, started, completed, versions): + self.path = path + self.started = started + self.completed = completed + self.versions = versions + + @classmethod + def from_dict(cls, path, dictionary): + return cls( + path, + load_datetime(dictionary.get("started")), + load_datetime(dictionary.get("completed")), + [NewVersion.from_dict(v) for v in dictionary.get("versions", [])], + ) + + @classmethod + def from_path(cls, path): + content = {} + if path.exists(): + try: + with open(str(path), "rt") as file_handler: + content = json.load(file_handler) + except (IOError, ValueError): + pass + return cls.from_dict(path, content) + + def to_dict(self): + return { + "started": dump_datetime(self.started), + "completed": dump_datetime(self.completed), + "versions": [r.to_dict() for r in self.versions], + } + + def update(self): + with open(str(self.path), "wt") as file_handler: + json.dump(self.to_dict(), file_handler, sort_keys=True, indent=4) + + @property + def needs_update(self): + now = datetime.now() + if self.completed is None: # never completed + return self._check_start(now) + else: + if now - self.completed <= timedelta(days=14): + return False + return self._check_start(now) + + def _check_start(self, now): + return self.started is None or now - self.started >= timedelta(hours=1) + + +def trigger_update(distribution, for_py_version, wheel, app_data): + cmd = [ + sys.executable, + "-c", + "from virtualenv.seed.embed.wheels.periodic_update import do_update;" + "do_update({!r}, {!r}, {!r}, {!r})".format(distribution, for_py_version, str(wheel.path), str(app_data.path)), + ] + kwargs = {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE} + if sys.platform == "win32": + kwargs["creation_flags"] = DETACHED_PROCESS + process = Popen(cmd, **kwargs) + logging.info( + "triggered periodic upgrade of %s==%s (for python %s) via background process having PID %d", + distribution, + wheel.version, + for_py_version, + process.pid, + ) + process.communicate() # on purpose not called to make it detached + + +def do_update(distribution, for_py_version, wheel_filename, app_data): + temp_dir = Path(tempfile.mkdtemp()) + try: + copy2(wheel_filename, temp_dir) + local_wheel = Wheel(Path(wheel_filename)) + from .acquire import download_wheel + + with update_log_for_distribution(local_wheel.path.parent.parent, distribution) as u_log: + + download_wheel(distribution, None, for_py_version, temp_dir, ReentrantFileLock(app_data)) + + new_wheels = [f for f in temp_dir.iterdir() if f.name != local_wheel.name] + if new_wheels: + new_wheel = new_wheels[0] + dest = local_wheel.path.parent / new_wheel.name + if not dest.exists(): + copy2(new_wheel, dest) + else: + dest = local_wheel.path + release_date = _get_release_date(dest) + if u_log.updated is not None: + u_log.previous_updated = u_log.updated + u_log.previous_updated_release_date = u_log.updated_release_date + u_log.updated = dest.name + u_log.updated_release_date = release_date + u_log.completed = datetime.now() + u_log.update() + + finally: + safe_delete(temp_dir) + + +def _get_release_date(dest): + wheel = Wheel(dest) + # the most accurate is to ask PyPi - https://pypi.org/pypi/pip/json + try: + with urlopen("https://pypi.org/pypi/{}/json".format(wheel.distribution)) as file_handler: + content = json.load(file_handler) + return datetime.strptime(content["releases"][wheel.version][0]["upload_time"], "%Y-%m-%dT%H:%M:%S") + except Exception: # noqa + pass + # otherwise can approximate from the version number https://pip.pypa.io/en/latest/development/release-process/ + released = datetime(year=2000 + wheel.version_tuple[0], month=wheel.version_tuple[1] * 3 + 1, day=1) + released += timedelta(days=calendar.monthrange(released.year, released.month)[1]) + return released + + +__all__ = ( + "periodic_update", + "do_update", +) diff --git a/src/virtualenv/seed/embed/wheels/util.py b/src/virtualenv/seed/embed/wheels/util.py new file mode 100644 index 000000000..b4ed44c49 --- /dev/null +++ b/src/virtualenv/seed/embed/wheels/util.py @@ -0,0 +1,72 @@ +from zipfile import ZipFile + +from virtualenv.util.six import ensure_text + + +class Wheel(object): + def __init__(self, path): + # https://www.python.org/dev/peps/pep-0427/#file-name-convention + # The wheel filename is {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl + self.path = path + self._parts = path.stem.split("-") + + @classmethod + def from_path(cls, path): + if path.suffix == ".whl" and len(path.stem.split("-")) >= 5: + return cls(path) + return None + + @property + def distribution(self): + return self._parts[0] + + @property + def version(self): + return self._parts[1] + + @property + def version_tuple(self): + result = [] + for part in self.version.split(".")[0:3]: + try: + result.append(int(part)) + except ValueError: + break + return tuple(result) + + @property + def name(self): + return self.path.name + + def support_py(self, py_version): + name = "{}.dist-info/METADATA".format("-".join(self.path.stem.split("-")[0:2])) + with ZipFile(ensure_text(str(self.path)), "r") as zip_file: + metadata = zip_file.read(name).decode("utf-8") + marker = "Requires-Python:" + requires = next((i[len(marker) :] for i in metadata.splitlines() if i.startswith(marker)), None) + if requires is None: # if it does not specify a python requires the assumption is compatible + return True + py_version_int = tuple(int(i) for i in py_version.split(".")) + for require in (i.strip() for i in requires.split(",")): + # https://www.python.org/dev/peps/pep-0345/#version-specifiers + for operator, check in [ + ("!=", lambda v: py_version_int != v), + ("==", lambda v: py_version_int == v), + ("<=", lambda v: py_version_int <= v), + (">=", lambda v: py_version_int >= v), + ("<", lambda v: py_version_int < v), + (">", lambda v: py_version_int > v), + ]: + if require.startswith(operator): + ver_str = require[len(operator) :].strip() + version = tuple((int(i) if i != "*" else None) for i in ver_str.split("."))[0:2] + if not check(version): + return False + break + return True + + def __repr__(self): + return "{}({})".format(self.__class__.__name__, self.path) + + def __str__(self): + return str(self.path) diff --git a/src/virtualenv/seed/via_app_data/via_app_data.py b/src/virtualenv/seed/via_app_data/via_app_data.py index de3757d7c..3eeffeca8 100644 --- a/src/virtualenv/seed/via_app_data/via_app_data.py +++ b/src/virtualenv/seed/via_app_data/via_app_data.py @@ -8,18 +8,23 @@ from virtualenv.info import fs_supports_symlink from virtualenv.seed.embed.base_embed import BaseEmbed -from virtualenv.seed.embed.wheels.acquire import WheelDownloadFail, get_wheels -from virtualenv.util.path import safe_delete +from virtualenv.seed.embed.wheels.acquire import Version, WheelDownloadFail, get_wheel from .pip_install.copy import CopyPipInstall from .pip_install.symlink import SymlinkPipInstall class FromAppData(BaseEmbed): + packages = { + "pip": Version.periodic_update, + "setuptools": Version.latest, + "wheel": Version.latest, + } + def __init__(self, options): super(FromAppData, self).__init__(options) self.symlinks = options.symlink_app_data - self.base_cache = self.app_data / "seed-app-data" / "v1.0.1" + self.base_cache = self.app_data / "seed-app-data" / "v1.0.2" @classmethod def add_parser_arguments(cls, parser, interpreter, app_data): @@ -40,13 +45,13 @@ def run(self, creator): return base_cache = self.base_cache / creator.interpreter.version_release_str with self._get_seed_wheels(creator, base_cache) as name_to_whl: - pip_version = name_to_whl["pip"].stem.split("-")[1] if "pip" in name_to_whl else None + pip_version = name_to_whl["pip"].version_tuple if "pip" in name_to_whl else None installer_class = self.installer_class(pip_version) def _install(name, wheel): logging.debug("install %s from wheel %s via %s", name, wheel, installer_class.__name__) - image_folder = base_cache.path / "image" / installer_class.__name__ / wheel.stem - installer = installer_class(wheel, creator, image_folder) + image_folder = base_cache.path / "image" / installer_class.__name__ / wheel.path.stem + installer = installer_class(wheel.path, creator, image_folder) if not installer.has_image(): installer.build_image() installer.install(creator.interpreter.version_info) @@ -61,33 +66,25 @@ def _install(name, wheel): def _get_seed_wheels(self, creator, base_cache): with base_cache.lock_for_key("wheels"): wheels_to = base_cache.path / "wheels" - if wheels_to.exists(): - safe_delete(wheels_to) wheels_to.mkdir(parents=True, exist_ok=True) name_to_whl, lock, fail = {}, Lock(), {} - def _get(package, version): - wheel_loader = partial( - get_wheels, - creator.interpreter.version_release_str, - wheels_to, - self.extra_search_dir, - {package: version}, - self.app_data, - ) + def _get(pkg, version): + for_py_version = creator.interpreter.version_release_str + loader = partial(get_wheel, pkg, version, for_py_version, self.extra_search_dir) failure, result = None, None # fallback to download in case the exact version is not available for download in [True] if self.download else [False, True]: failure = None try: - result = wheel_loader(download) + result = loader(download, wheels_to, self.app_data) if result: break except Exception as exception: failure = exception if failure: if isinstance(failure, WheelDownloadFail): - msg = "failed to download {}".format(package) + msg = "failed to download {}".format(pkg) if version is not None: msg += " version {}".format(version) msg += ", pip download exit code {}".format(failure.exit_code) @@ -99,10 +96,10 @@ def _get(package, version): msg = repr(failure) logging.error(msg) with lock: - fail[package] = version + fail[pkg] = version else: with lock: - name_to_whl.update(result) + name_to_whl[pkg] = result package_versions = self.package_version() threads = list(Thread(target=_get, args=(pkg, v)) for pkg, v in package_versions.items()) @@ -114,11 +111,10 @@ def _get(package, version): raise RuntimeError("seed failed due to failing to download wheels {}".format(", ".join(fail.keys()))) yield name_to_whl - def installer_class(self, pip_version): - if self.symlinks and pip_version: + def installer_class(self, pip_version_tuple): + if self.symlinks and pip_version_tuple: # symlink support requires pip 19.3+ - pip_version_int = tuple(int(i) for i in pip_version.split(".")[0:2]) - if pip_version_int >= (19, 3): + if pip_version_tuple >= (19, 3): return SymlinkPipInstall return CopyPipInstall diff --git a/src/virtualenv/util/lock.py b/src/virtualenv/util/lock.py index 0c5e72fc7..eb7a78f95 100644 --- a/src/virtualenv/util/lock.py +++ b/src/virtualenv/util/lock.py @@ -74,7 +74,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): self._release(self._lock) - def _lock_file(self, lock): + def _lock_file(self, lock, no_block=False): # multiple processes might be trying to get a first lock... so we cannot check if this directory exist without # a lock, but that lock might then become expensive, and it's not clear where that lock should live. # Instead here we just ignore if we fail to create the directory. @@ -85,6 +85,8 @@ def _lock_file(self, lock): try: lock.acquire(0.0001) except Timeout: + if no_block: + raise logging.debug("lock file %s present, will block until released", lock.lock_file) lock.release() # release the acquire try from above lock.acquire() @@ -94,13 +96,19 @@ def _release(lock): lock.release() @contextmanager - def lock_for_key(self, name): + def lock_for_key(self, name, no_block=False): lock = self._create_lock(name) try: try: - self._lock_file(lock) + self._lock_file(lock, no_block) yield finally: self._release(lock) finally: self._del_lock(lock) + + +__all__ = ( + "Timeout", + "ReentrantFileLock", +) diff --git a/src/virtualenv/util/subprocess/__init__.py b/src/virtualenv/util/subprocess/__init__.py index 6cb0a2810..95fc0dd81 100644 --- a/src/virtualenv/util/subprocess/__init__.py +++ b/src/virtualenv/util/subprocess/__init__.py @@ -12,6 +12,8 @@ else: Popen = subprocess.Popen +DETACHED_PROCESS = 0x00000008 + def run_cmd(cmd): try: @@ -29,4 +31,5 @@ def run_cmd(cmd): "subprocess", "Popen", "run_cmd", + "DETACHED_PROCESS", ) diff --git a/tests/unit/seed/embed/wheels/test_acquire.py b/tests/unit/seed/embed/wheels/test_acquire.py index 49f743336..b49b42d74 100644 --- a/tests/unit/seed/embed/wheels/test_acquire.py +++ b/tests/unit/seed/embed/wheels/test_acquire.py @@ -1,11 +1,11 @@ -from virtualenv.seed.embed.wheels.acquire import get_bundled_wheel, wheel_support_py +from virtualenv.seed.embed.wheels.acquire import get_bundled_wheel def test_wheel_support_no_python_requires(mocker): - wheel = get_bundled_wheel(package="setuptools", version_release=None) + wheel = get_bundled_wheel("setuptools", for_py_version=None) zip_mock = mocker.MagicMock() - mocker.patch("virtualenv.seed.embed.wheels.acquire.ZipFile", new=zip_mock) + mocker.patch("virtualenv.seed.embed.wheels.util.ZipFile", new=zip_mock) zip_mock.return_value.__enter__.return_value.read = lambda name: b"" - supports = wheel_support_py(wheel, "3.8") + supports = wheel.support_py("3.8") assert supports is True