From 128f6105c6862ea57b06b6ea261436695b75c59d Mon Sep 17 00:00:00 2001 From: Alessio Ababilov Date: Sun, 1 Sep 2013 11:19:43 +0300 Subject: [PATCH] Handle multiple requirements for one package Sometimes a developer needs to install dependencies for several packages, and each of them has its own requirements.txt file. Often, the same package is referenced in several files, possibly with different version requirements. pip should not print "Double requirement given". Instead, it should aggregate requirements (e.g., prettytable>=0.6 and prettytable<0.8 turns into prettytable>=0.6,<0.8). --- pip/cmdoptions.py | 6 ++ pip/commands/install.py | 5 +- pip/commands/uninstall.py | 6 +- pip/commands/wheel.py | 5 +- pip/req.py | 176 +++++++++++++++++++++++++++++++++----- tests/unit/test_req.py | 30 ++++++- 6 files changed, 203 insertions(+), 25 deletions(-) diff --git a/pip/cmdoptions.py b/pip/cmdoptions.py index c6904dadda1..b9e857b1533 100644 --- a/pip/cmdoptions.py +++ b/pip/cmdoptions.py @@ -169,6 +169,12 @@ def make_option_group(group, parser): default=False, help="Don't clean up build directories.") +ignore_incompatibles = make_option( + '--ignore-incompatibles', + action='store_true', + default=False, + help="Ignore the incompatible requirements to the same package.") + ########## # groups # diff --git a/pip/commands/install.py b/pip/commands/install.py index 11099c5a228..851e958466f 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -145,6 +145,7 @@ def __init__(self, *args, **kw): help="Include pre-release and development versions. By default, pip only finds stable versions.") cmd_opts.add_option(cmdoptions.no_clean) + cmd_opts.add_option(cmdoptions.ignore_incompatibles) index_opts = cmdoptions.make_option_group(cmdoptions.index_group, self.parser) @@ -219,7 +220,8 @@ def run(self, options, args): ignore_dependencies=options.ignore_dependencies, force_reinstall=options.force_reinstall, use_user_site=options.use_user_site, - target_dir=temp_target_dir) + target_dir=temp_target_dir, + ignore_incompatibles=options.ignore_incompatibles) for name in args: requirement_set.add_requirement( InstallRequirement.from_line(name, None)) @@ -240,6 +242,7 @@ def run(self, options, args): 'to %(name)s (see "pip help %(name)s")' % opts) logger.warn(msg) return + requirement_set.process_multiple_requirements() try: if not options.no_download: diff --git a/pip/commands/uninstall.py b/pip/commands/uninstall.py index 388053b20f2..3159dc40c12 100644 --- a/pip/commands/uninstall.py +++ b/pip/commands/uninstall.py @@ -1,6 +1,7 @@ from pip.req import InstallRequirement, RequirementSet, parse_requirements from pip.basecommand import Command from pip.exceptions import InstallationError +from pip import cmdoptions class UninstallCommand(Command): @@ -34,6 +35,7 @@ def __init__(self, *args, **kw): dest='yes', action='store_true', help="Don't ask for confirmation of uninstall deletions.") + self.cmd_opts.add_option(cmdoptions.ignore_incompatibles) self.parser.insert_option_group(0, self.cmd_opts) @@ -41,7 +43,8 @@ def run(self, options, args): requirement_set = RequirementSet( build_dir=None, src_dir=None, - download_dir=None) + download_dir=None, + ignore_incompatibles=options.ignore_incompatibles) for name in args: requirement_set.add_requirement( InstallRequirement.from_line(name)) @@ -51,4 +54,5 @@ def run(self, options, args): if not requirement_set.has_requirements: raise InstallationError('You must give at least one requirement ' 'to %(name)s (see "pip help %(name)s")' % dict(name=self.name)) + requirement_set.process_multiple_requirements() requirement_set.uninstall(auto_confirm=options.yes) diff --git a/pip/commands/wheel.py b/pip/commands/wheel.py index a29cc2d2059..48c93088de1 100644 --- a/pip/commands/wheel.py +++ b/pip/commands/wheel.py @@ -75,6 +75,7 @@ def __init__(self, *args, **kw): help="Include pre-release and development versions. By default, pip only finds stable versions.") cmd_opts.add_option(cmdoptions.no_clean) + cmd_opts.add_option(cmdoptions.ignore_incompatibles) index_opts = cmdoptions.make_option_group(cmdoptions.index_group, self.parser) @@ -125,7 +126,8 @@ def run(self, options, args): download_dir=None, download_cache=options.download_cache, ignore_dependencies=options.ignore_dependencies, - ignore_installed=True) + ignore_installed=True, + ignore_incompatibles=options.ignore_incompatibles) #parse args and/or requirements files for name in args: @@ -149,6 +151,7 @@ def run(self, options, args): 'to %(name)s (see "pip help %(name)s")' % opts) logger.error(msg) return + requirement_set.process_multiple_requirements() try: #build wheels diff --git a/pip/req.py b/pip/req.py index 31505082f93..c10ffdc5159 100644 --- a/pip/req.py +++ b/pip/req.py @@ -9,6 +9,8 @@ import textwrap import zipfile +from pip.vendor.six import iteritems + from distutils.util import change_root from pip.locations import (bin_py, running_under_virtualenv,PIP_DELETE_MARKER_FILENAME, write_delete_marker_file) @@ -852,7 +854,8 @@ class RequirementSet(object): def __init__(self, build_dir, src_dir, download_dir, download_cache=None, upgrade=False, ignore_installed=False, as_egg=False, target_dir=None, - ignore_dependencies=False, force_reinstall=False, use_user_site=False): + ignore_dependencies=False, force_reinstall=False, use_user_site=False, + ignore_incompatibles=False): self.build_dir = build_dir self.src_dir = src_dir self.download_dir = download_dir @@ -861,8 +864,9 @@ def __init__(self, build_dir, src_dir, download_dir, download_cache=None, self.ignore_installed = ignore_installed self.force_reinstall = force_reinstall self.requirements = Requirements() - # Mapping of alias: real_name - self.requirement_aliases = {} + self.multiple_requirements = {} + self.ignore_incompatibles = ignore_incompatibles + self.incompatibles = set() self.unnamed_requirements = [] self.ignore_dependencies = ignore_dependencies self.successfully_downloaded = [] @@ -887,20 +891,152 @@ def add_requirement(self, install_req): #url or path requirement w/o an egg fragment self.unnamed_requirements.append(install_req) else: - if self.has_requirement(name): - raise InstallationError( - 'Double requirement given: %s (already in %s, name=%r)' - % (install_req, self.get_requirement(name), name)) - self.requirements[name] = install_req - ## FIXME: what about other normalizations? E.g., _ vs. -? - if name.lower() != name: - self.requirement_aliases[name.lower()] = name + req_key = install_req.req.key + try: + previous_req = self.requirements[req_key] + except KeyError: + pass + else: + self.multiple_requirements.setdefault( + req_key, [previous_req]).append(install_req) + self.requirements[req_key] = install_req + + def process_multiple_requirements(self): + self.incompatibles = set() + for req_key, req_list in iteritems(self.multiple_requirements): + joined_req = self.aggregate_requirement(req_list) + self.find_conflicts(joined_req, req_list) + self.requirements[req_key] = joined_req + if self.incompatibles and not self.ignore_incompatibles: + raise InstallationError("Incompatible requirements found") + + @staticmethod + def error_requirement(req): + logger.error("\t%s: %s" % + (req.comes_from or "command line", + req.url or str(req.req))) + + def incompatible_requirement(self, chosen, conflicting): + if chosen.req.key not in self.incompatibles: + self.incompatibles.add(chosen.req.key) + logger.error("%s: incompatible requirements" % chosen.req.key) + if self.ignore_incompatibles: + logger.error("Choosing:") + self.error_requirement(chosen) + if self.ignore_incompatibles: + logger.error("Conflicting:") + self.error_requirement(conflicting) + + def aggregate_requirement(self, req_list): + """Aggregate requirement list for one package together. + + Possible returns: + * ==A - exact version (even when there are conflicts) + * >=?A,<=?B,(!=C)+ - line segment (no conflicts detected) + * >=?A,(!=C)+ - more than (also when conflicts detected) + + :param:req_list list of pip.req.InstallRequirement + :return: pip.req.InstallRequirement + """ + if len(req_list) == 1: + return req_list[0] + req_strict = None + lower_bound_str = None + lower_bound_version = None + lower_bound_req = None + upper_bound_str = None + upper_bound_version = None + upper_bound_req = None + conflicts = [] + for req in req_list: + for spec in req.req.specs: + if spec[0] == "==": + return req + spec_str = "%s%s" % spec + if spec[0] == "!=": + conflicts.append(spec_str) + continue + version = pkg_resources.parse_version(spec[1]) + # strict_check is < or >, not <= or >= + strict_check = len(spec[0]) == 1 + if spec[0][0] == ">": + if (not lower_bound_version or (version > lower_bound_version) or + (strict_check and version == lower_bound_version)): + lower_bound_version = version + lower_bound_str = spec_str + lower_bound_req = req + else: + if (not upper_bound_version or (version < upper_bound_version) or + (strict_check and version == upper_bound_version)): + upper_bound_version = version + upper_bound_str = spec_str + upper_bound_req = req + req_key = req_list[0].req.key + if lower_bound_version and upper_bound_version: + bad_bounds = False + if lower_bound_version > upper_bound_version: + upper_bound_str = None + if lower_bound_version == upper_bound_version: + if lower_bound_str[1] == "=" and upper_bound_str[1] == "=": + return pip.req.InstallRequirement.from_line( + "%s==%s" % (req_key, upper_bound_str[2:]), + "aggregated requirements") + else: + upper_bound_str = None + req_specs = [] + if lower_bound_str: + req_specs.append(lower_bound_str) + if upper_bound_str: + req_specs.append(upper_bound_str) + req_specs.extend(conflicts) + return pip.req.InstallRequirement.from_line( + "%s%s" % (req_key, ",".join(req_specs)), + "aggregated requirements") + + def find_conflicts(self, joined_req, req_list): + segment_ok = False + lower_version = None + lower_strict = False + exact_version = None + conflicts = [] + for parsed, trans, op, ver in joined_req.req.index: + if op[0] == ">": + lower_version = parsed + lower_strict = len(op) == 1 + elif op[0] == "<": + segment_ok = True + elif op[0] == "=": + exact_version = parsed + else: + conflicts.append(parsed) + if exact_version: + for req in req_list: + if not exact_version in req.req: + self.incompatible_requirement(joined_req, req) + else: + for req in req_list: + for parsed, trans, op, ver in req.req.index: + if not segment_ok and op[0] == "<": + # analyse lower bound: x >= A or x > A + if (lower_version > parsed or ( + lower_version == parsed and + (lower_strict or len(op) != 2))): + self.incompatible_requirement(joined_req, req) + break + + @staticmethod + def canonical_name(name): + # make happy some tests: "distribute" is parsed as "setuptools" in + # test_from_setuptools_7_to_setuptools_7_with_distribute_7_installed + if name.isalpha(): + return name.lower() + try: + return pkg_resources.Requirement.parse(name).key + except ValueError: + return name def has_requirement(self, project_name): - for name in project_name, project_name.lower(): - if name in self.requirements or name in self.requirement_aliases: - return True - return False + return self.canonical_name(project_name) in self.requirements @property def has_requirements(self): @@ -928,12 +1064,10 @@ def is_download(self): return False def get_requirement(self, project_name): - for name in project_name, project_name.lower(): - if name in self.requirements: - return self.requirements[name] - if name in self.requirement_aliases: - return self.requirements[self.requirement_aliases[name]] - raise KeyError("No project with the name %r" % project_name) + try: + return self.requirements[self.canonical_name(project_name)] + except KeyError: + raise KeyError("No project with the name %r" % project_name) def uninstall(self, auto_confirm=False): for req in self.requirements.values(): diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index 67c373b3ec6..970f4463090 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -6,7 +6,7 @@ import pip.wheel -from pkg_resources import Distribution +from pkg_resources import Distribution, Requirement from mock import Mock, patch from pip.exceptions import PreviousBuildDirError from pip.index import PackageFinder @@ -52,6 +52,34 @@ def test_no_reuse_existing_build_dir(self, data): finder ) + def test_process_multiple_requirements(self): + test_list = [ + (["a>1"], "a>1", None), + (["a<1"], "a<1", None), + (["a>1", "a<2"], "a>1,<2", None), + (["a>=1", "a>1"], "a>1", None), + (["a<=1", "a<1"], "a<1", None), + (["a<=1", "a>=1"], "a==1", None), + (["a>1", "a>2", "a>4.3", "a>4.0", "a>0"], "a>4.3", None), + (["a>1", "a!=7"], "a>1,!=7", None), + (["MixedCase>1", "mixedcase>2"], "mixedcase>2", None), + (["x>1", "x<=1"], "x>1", "x"), + (["x>1", "x<0"], "x>1", "x"), + (["x>1", "x==0"], "x==0", "x"), + (["x==1", "x==0"], "x==1", "x"), + ] + for (req_lines, req_result, req_incompat) in test_list: + reqset = self.basic_reqset() + reqset.ignore_incompatibles = True + for req in req_lines: + reqset.add_requirement(InstallRequirement.from_line(req)) + reqset.process_multiple_requirements() + reqs = reqset.requirements.values() + assert len(reqs) == 1 + assert reqs[0].req == Requirement.parse(req_result) + if req_incompat: + assert req_incompat in reqset.incompatibles + def test_url_with_query(): """InstallRequirement should strip the fragment, but not the query."""