From 16f5e8f763956695c8c37e819b1c05fac99d4383 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Wed, 5 Oct 2011 19:46:03 +0200 Subject: [PATCH] Extended PEP 381 mirror and package verification features. This includes the following modifications: - The PyPI mirrors are now used by default. You can disable them by specifying `--no-mirrors`. Each mirror is now tried one by when when retrieving a distribution instead of naively randomizing the list of index URLs. Distributions from mirrors whose signature don't match are ignored. - Added `--refresh-serverkey` option to `install` command to retrieve the public PyPI server key used to verify the mirror packages, caching it as ~/.pip/serverkey.pub. Same thing happens (once) it doesn't exist. - Added ability to verify the SSL certificate of PyPI. This requires the ssl module added in Python 2.6. For Python < 2.6 the standalone version at http://pypi.python.org/pypi/ssl can be installed instead. Using https://pypi.python.org/ is the default now. - Added OrderedDict for Python < 3.1 used in PackageFinder when collecting packages links for easier handling. - A bunch of PEP 8 related fixes. --- pip/backwardcompat.py | 147 +++++++++++ pip/basecommand.py | 49 +++- pip/baseparser.py | 6 + pip/commands/install.py | 25 +- pip/commands/search.py | 2 +- pip/download.py | 244 +++++++++++++++--- pip/index.py | 541 +++++++++++++++++++++++++++++++--------- pip/locations.py | 2 + pip/req.py | 298 +++++++++++++--------- tests/test_pip.py | 2 + 10 files changed, 1030 insertions(+), 286 deletions(-) diff --git a/pip/backwardcompat.py b/pip/backwardcompat.py index bb88b4b92ec..084247e3d11 100644 --- a/pip/backwardcompat.py +++ b/pip/backwardcompat.py @@ -3,6 +3,7 @@ import sys import os import shutil +import base64 __all__ = ['any', 'WindowsError', 'md5', 'copytree'] @@ -60,11 +61,21 @@ def b(s): def u(s): return s.decode('utf-8') + def _ord(x): + return x + def console_to_str(s): return s.decode(console_encoding) bytes = bytes string_types = (str,) raw_input = input + + def decode_base64(source): + source = source.encode("ascii") # ensure bytes + return base64.decodebytes(source) + + _long = lambda x: x + else: from cStringIO import StringIO from urllib2 import URLError, HTTPError @@ -84,6 +95,9 @@ def b(s): def u(s): return s + def _ord(x): + return ord(x) + def console_to_str(s): return s bytes = str @@ -92,6 +106,8 @@ def console_to_str(s): cmp = cmp raw_input = raw_input BytesIO = StringIO + decode_base64 = base64.decodestring + _long = lambda x: long(x) try: from email.parser import FeedParser @@ -122,3 +138,134 @@ def product(*args, **kwds): result = [x+[y] for x in result for y in pool] for prod in result: yield tuple(prod) + +try: + from collections import OrderedDict +except ImportError: + # Copyright (c) 2009 Raymond Hettinger + # + # Permission is hereby granted, free of charge, to any person + # obtaining a copy of this software and associated documentation files + # (the "Software"), to deal in the Software without restriction, + # including without limitation the rights to use, copy, modify, merge, + # publish, distribute, sublicense, and/or sell copies of the Software, + # and to permit persons to whom the Software is furnished to do so, + # subject to the following conditions: + # + # The above copyright notice and this permission notice shall be + # included in all copies or substantial portions of the Software. + # + # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + # OTHER DEALINGS IN THE SOFTWARE. + from UserDict import DictMixin + + class OrderedDict(dict, DictMixin): + + def __init__(self, *args, **kwds): + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + try: + self.__end + except AttributeError: + self.clear() + self.update(*args, **kwds) + + def clear(self): + self.__end = end = [] + end += [None, end, end] # sentinel node for doubly linked list + self.__map = {} # key --> [key, prev, next] + dict.clear(self) + + def __setitem__(self, key, value): + if key not in self: + end = self.__end + curr = end[1] + curr[2] = end[1] = self.__map[key] = [key, curr, end] + dict.__setitem__(self, key, value) + + def __delitem__(self, key): + dict.__delitem__(self, key) + key, prev, next = self.__map.pop(key) + prev[2] = next + next[1] = prev + + def __iter__(self): + end = self.__end + curr = end[2] + while curr is not end: + yield curr[0] + curr = curr[2] + + def __reversed__(self): + end = self.__end + curr = end[1] + while curr is not end: + yield curr[0] + curr = curr[1] + + def popitem(self, last=True): + if not self: + raise KeyError('dictionary is empty') + if last: + key = reversed(self).next() + else: + key = iter(self).next() + value = self.pop(key) + return key, value + + def __reduce__(self): + items = [[k, self[k]] for k in self] + tmp = self.__map, self.__end + del self.__map, self.__end + inst_dict = vars(self).copy() + self.__map, self.__end = tmp + if inst_dict: + return (self.__class__, (items,), inst_dict) + return self.__class__, (items,) + + def keys(self): + return list(self) + + setdefault = DictMixin.setdefault + update = DictMixin.update + pop = DictMixin.pop + values = DictMixin.values + items = DictMixin.items + iterkeys = DictMixin.iterkeys + itervalues = DictMixin.itervalues + iteritems = DictMixin.iteritems + + def __repr__(self): + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, self.items()) + + def copy(self): + return self.__class__(self) + + @classmethod + def fromkeys(cls, iterable, value=None): + d = cls() + for key in iterable: + d[key] = value + return d + + def __eq__(self, other): + if isinstance(other, OrderedDict): + if len(self) != len(other): + return False + for p, q in zip(self.items(), other.items()): + if p != q: + return False + return True + return dict.__eq__(self, other) + + def __ne__(self, other): + return not self == other + diff --git a/pip/basecommand.py b/pip/basecommand.py index 3d442eb05c5..e70be911f9c 100644 --- a/pip/basecommand.py +++ b/pip/basecommand.py @@ -7,12 +7,14 @@ import time from pip import commands -from pip.log import logger -from pip.baseparser import parser, ConfigOptionParser, UpdatingDefaultsHelpFormatter +from pip.backwardcompat import StringIO, walk_packages, u +from pip.baseparser import (parser, ConfigOptionParser, + UpdatingDefaultsHelpFormatter) from pip.download import urlopen -from pip.exceptions import (BadCommand, InstallationError, UninstallationError, - CommandError) -from pip.backwardcompat import StringIO, walk_packages +from pip.exceptions import (BadCommand, InstallationError, + UninstallationError, CommandError) +from pip.log import logger +from pip.locations import serverkey_file __all__ = ['command_dict', 'Command', 'load_all_commands', 'load_command', 'command_names'] @@ -48,11 +50,31 @@ def merge_options(self, initial_options, options): for attr in ['log', 'proxy', 'require_venv', 'log_explicit_levels', 'log_file', 'timeout', 'default_vcs', 'skip_requirements_regex', - 'no_input']: - setattr(options, attr, getattr(initial_options, attr) or getattr(options, attr)) + 'no_input', 'refresh_serverkey']: + setattr(options, attr, + getattr(initial_options, attr) or getattr(options, attr)) options.quiet += initial_options.quiet options.verbose += initial_options.verbose + def refresh_serverkey(self, url='https://pypi.python.org/serverkey'): + serverkey_cache = open(serverkey_file, 'wb') + try: + try: + content = urlopen(url).read() + serverkey_cache.write(content) + except Exception: + e = sys.exc_info()[1] + raise + raise InstallationError('Could not refresh local cache (%s) ' + 'of PyPI server key (%s): %s' % + (serverkey_file, url, e)) + else: + logger.notify('Refreshed local cache (%s) of ' + 'PyPI server key (%s):\n\n%s' % + (serverkey_file, url, u(content))) + finally: + serverkey_cache.close() + def setup_logging(self): pass @@ -60,10 +82,10 @@ def main(self, complete_args, args, initial_options): options, args = self.parser.parse_args(args) self.merge_options(initial_options, options) - level = 1 # Notify + level = 1 # Notify level += options.verbose level -= options.quiet - level = logger.level_for_integer(4-level) + level = logger.level_for_integer(4 - level) complete_log = [] logger.consumers.extend( [(level, sys.stdout), @@ -76,9 +98,13 @@ def main(self, complete_args, args, initial_options): if options.require_venv: # If a venv is required check if it can really be found if not os.environ.get('VIRTUAL_ENV'): - logger.fatal('Could not find an activated virtualenv (required).') + logger.fatal('Could not find an activated ' + 'virtualenv (required).') sys.exit(3) + if not os.path.exists(serverkey_file) or options.refresh_serverkey: + self.refresh_serverkey() + if options.log: log_fp = open_logfile(options.log, 'a') logger.consumers.append((logger.DEBUG, log_fp)) @@ -155,7 +181,7 @@ def open_logfile(filename, mode='a'): log_fp = open(filename, mode) if exists: - log_fp.write('%s\n' % ('-'*60)) + log_fp.write('%s\n' % ('-' * 60)) log_fp.write('%s run on %s\n' % (sys.argv[0], time.strftime('%c'))) return log_fp @@ -178,4 +204,3 @@ def load_all_commands(): def command_names(): names = set((pkg[1] for pkg in walk_packages(path=commands.__path__))) return list(names) - diff --git a/pip/baseparser.py b/pip/baseparser.py index 6af54c7d476..6eeb6012207 100644 --- a/pip/baseparser.py +++ b/pip/baseparser.py @@ -199,5 +199,11 @@ def get_default_values(self): type='str', default='', help=optparse.SUPPRESS_HELP) +parser.add_option( + '--refresh-serverkey', + dest='refresh_serverkey', + action='store_true', + default=False, + help="Refresh the cached version of PyPI's server key") parser.disable_interspersed_args() diff --git a/pip/commands/install.py b/pip/commands/install.py index d4d9e6383d6..90db34577cf 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -1,3 +1,4 @@ +import optparse import os import sys from pip.req import InstallRequirement, RequirementSet @@ -34,8 +35,8 @@ def __init__(self): action='append', default=[], metavar='FILENAME', - help='Install all the packages listed in the given requirements file. ' - 'This option can be used multiple times.') + help='Install all the packages listed in the given requirements ' + 'file. This option can be used multiple times.') self.parser.add_option( '-f', '--find-links', dest='find_links', @@ -47,7 +48,7 @@ def __init__(self): '-i', '--index-url', '--pypi-url', dest='index_url', metavar='URL', - default='http://pypi.python.org/simple/', + default='https://pypi.python.org/simple/', help='Base URL of Python Package Index (default %default)') self.parser.add_option( '--extra-index-url', @@ -62,19 +63,25 @@ def __init__(self): action='store_true', default=False, help='Ignore package index (only looking at --find-links URLs instead)') + self.parser.add_option( + '--no-mirrors', + dest='use_mirrors', + action='store_false', + default=False, + help='Ignore the PyPI mirrors') self.parser.add_option( '-M', '--use-mirrors', dest='use_mirrors', action='store_true', - default=False, - help='Use the PyPI mirrors as a fallback in case the main index is down.') + default=True, + help=optparse.SUPPRESS_HELP) self.parser.add_option( '--mirrors', dest='mirrors', metavar='URL', action='append', default=[], - help='Specific mirror URLs to query when --use-mirrors is used') + help='Specific mirror URLs to use instead of querying the DNS for list of mirrors') self.parser.add_option( '-b', '--build', '--build-dir', '--build-directory', @@ -196,9 +203,11 @@ def run(self, options, args): InstallRequirement.from_line(name, None)) for name in options.editables: requirement_set.add_requirement( - InstallRequirement.from_editable(name, default_vcs=options.default_vcs)) + InstallRequirement.from_editable( + name, default_vcs=options.default_vcs)) for filename in options.requirements: - for req in parse_requirements(filename, finder=finder, options=options): + for req in parse_requirements( + filename, finder=finder, options=options): requirement_set.add_requirement(req) if not requirement_set.has_requirements: opts = {'name': self.name} diff --git a/pip/commands/search.py b/pip/commands/search.py index f7f82d964b4..f1e6aafe136 100644 --- a/pip/commands/search.py +++ b/pip/commands/search.py @@ -25,7 +25,7 @@ def __init__(self): '--index', dest='index', metavar='URL', - default='http://pypi.python.org/pypi', + default='https://pypi.python.org/pypi', help='Base URL of Python Package Index (default %default)') def run(self, options, args): diff --git a/pip/download.py b/pip/download.py index b11d1e8c4e7..d3ccdc27e95 100644 --- a/pip/download.py +++ b/pip/download.py @@ -4,14 +4,16 @@ import os import re import shutil +import socket import sys import tempfile from pip.backwardcompat import (md5, copytree, xmlrpclib, urllib, urllib2, - urlparse, string_types, HTTPError) + urlparse, string_types) from pip.exceptions import InstallationError from pip.util import (splitext, rmtree, format_size, display_path, backup_dir, ask, - unpack_file, create_download_cache_folder, cache_download) + unpack_file, create_download_cache_folder, + cache_download) from pip.vcs import vcs from pip.log import logger @@ -25,6 +27,50 @@ xmlrpclib_transport = xmlrpclib.Transport() +CACERT_ROOT_CRT = """\ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE-----""" + + def get_file_content(url, comes_from=None): """Gets the content of a file; it may be a filename, file: URL, or http: URL. Returns (location, content)""" @@ -55,7 +101,8 @@ def get_file_content(url, comes_from=None): content = f.read() except IOError: e = sys.exc_info()[1] - raise InstallationError('Could not open requirements file: %s' % str(e)) + raise InstallationError( + 'Could not open requirements file: %s' % str(e)) else: f.close() return url, content @@ -79,6 +126,8 @@ def __call__(self, url): auth. """ + if url.startswith('https://pypi.python.org'): + verify_pypi_ssl_certificate() url, username, password = self.extract_credentials(url) if username is None: try: @@ -103,13 +152,14 @@ def get_request(self, url): def get_response(self, url, username=None, password=None): """ - does the dirty work of actually getting the rsponse object using urllib2 - and its HTTP auth builtins. + does the dirty work of actually getting the rsponse object using + urllib2 and its HTTP auth builtins. """ scheme, netloc, path, query, frag = urlparse.urlsplit(url) req = self.get_request(url) - stored_username, stored_password = self.passman.find_user_password(None, netloc) + stored_username, stored_password = \ + self.passman.find_user_password(None, netloc) # see if we have a password stored if stored_username is None: if username is None and self.prompting: @@ -117,10 +167,12 @@ def get_response(self, url, username=None, password=None): password = urllib.quote(getpass.getpass('Password: ')) if username and password: self.passman.add_password(None, netloc, username, password) - stored_username, stored_password = self.passman.find_user_password(None, netloc) + stored_username, stored_password = \ + self.passman.find_user_password(None, netloc) authhandler = urllib2.HTTPBasicAuthHandler(self.passman) opener = urllib2.build_opener(authhandler) - # FIXME: should catch a 401 and offer to let the user reenter credentials + # FIXME: should catch a 401 and offer to let the + # user reenter credentials return opener.open(req) def setup(self, proxystr='', prompting=True): @@ -132,8 +184,8 @@ def setup(self, proxystr='', prompting=True): self.prompting = prompting proxy = self.get_proxy(proxystr) if proxy: - proxy_support = urllib2.ProxyHandler({"http": proxy, "ftp": proxy}) - opener = urllib2.build_opener(proxy_support, urllib2.CacheFTPHandler) + handler = urllib2.ProxyHandler({"http": proxy, "ftp": proxy}) + opener = urllib2.build_opener(handler, urllib2.CacheFTPHandler) urllib2.install_opener(opener) def parse_credentials(self, netloc): @@ -322,10 +374,10 @@ def is_file_url(link): def _check_md5(download_hash, link): - download_hash = download_hash.hexdigest() - if download_hash != link.md5_hash: - logger.fatal("MD5 hash of the package %s (%s) doesn't match the expected hash %s!" - % (link, download_hash, link.md5_hash)) + digest = download_hash.hexdigest() + if digest != link.md5_hash: + logger.fatal("MD5 hash of the package %s (%s) doesn't match the " + "expected hash %s!" % (link, digest, link.md5_hash)) raise InstallationError('Bad MD5 hash for package %s' % link) @@ -351,15 +403,17 @@ def _download_url(resp, link, temp_location): except (ValueError, KeyError, TypeError): total_length = 0 downloaded = 0 - show_progress = total_length > 40*1000 or not total_length + show_progress = total_length > 40 * 1000 or not total_length show_url = link.show_url try: if show_progress: ## FIXME: the URL can get really long in this message: if total_length: - logger.start_progress('Downloading %s (%s): ' % (show_url, format_size(total_length))) + logger.start_progress('Downloading %s (%s): ' % + (show_url, format_size(total_length))) else: - logger.start_progress('Downloading %s (unknown size): ' % show_url) + logger.start_progress('Downloading %s (unknown size): ' % + show_url) else: logger.notify('Downloading %s' % show_url) logger.debug('Downloading from URL %s' % link) @@ -373,7 +427,9 @@ def _download_url(resp, link, temp_location): if not total_length: logger.show_progress('%s' % format_size(downloaded)) else: - logger.show_progress('%3i%% %s' % (100*downloaded/total_length, format_size(downloaded))) + logger.show_progress('%3i%% %s' % + (100 * downloaded / total_length, + format_size(downloaded))) if link.md5_hash: download_hash.update(chunk) fp.write(chunk) @@ -397,8 +453,9 @@ def _copy_file(filename, location, content_type, link): os.remove(download_location) elif response == 'b': dest_file = backup_dir(download_location) - logger.warn('Backing up %s to %s' - % (display_path(download_location), display_path(dest_file))) + logger.warn('Backing up %s to %s' % + (display_path(download_location), + display_path(dest_file))) shutil.move(download_location, dest_file) if copy: shutil.copy(filename, download_location) @@ -412,14 +469,13 @@ def unpack_http_url(link, location, download_cache, only_download): target_file = None download_hash = None if download_cache: - target_file = os.path.join(download_cache, - urllib.quote(target_url, '')) + cache_filename = list(filter(None, target_url.split('/')))[-1] + target_file = os.path.join(download_cache, cache_filename) if not os.path.isdir(download_cache): create_download_cache_folder(download_cache) - if (target_file - and os.path.exists(target_file) - and os.path.exists(target_file + '.content-type')): - fp = open(target_file+'.content-type') + if (target_file and os.path.exists(target_file) + and os.path.exists(target_file + '.content-type')): + fp = open(target_file + '.content-type') content_type = fp.read().strip() fp.close() if link.md5_hash: @@ -427,11 +483,11 @@ def unpack_http_url(link, location, download_cache, only_download): temp_location = target_file logger.notify('Using download cache from %s' % target_file) else: - resp = _get_response_from_url(target_url, link) - content_type = resp.info()['content-type'] + response = _get_response_from_url(target_url, link) + content_type = response.info()['content-type'] filename = link.filename # fallback # Have a look at the Content-Disposition header for a better guess - content_disposition = resp.info().get('content-disposition') + content_disposition = response.info().get('content-disposition') if content_disposition: type, params = cgi.parse_header(content_disposition) # We use ``or`` here because we don't want to use an "empty" value @@ -442,12 +498,12 @@ def unpack_http_url(link, location, download_cache, only_download): ext = mimetypes.guess_extension(content_type) if ext: filename += ext - if not ext and link.url != geturl(resp): - ext = os.path.splitext(geturl(resp))[1] + if not ext and link.url != geturl(response): + ext = os.path.splitext(geturl(response))[1] if ext: filename += ext temp_location = os.path.join(temp_dir, filename) - download_hash = _download_url(resp, link, temp_location) + download_hash = _download_url(response, link, temp_location) if link.md5_hash: _check_md5(download_hash, link) if only_download: @@ -479,3 +535,127 @@ def _get_response_from_url(target_url, link): class Urllib2HeadRequest(urllib2.Request): def get_method(self): return "HEAD" + + +def valid_ipv6_addr(addr): + try: + addr = socket.inet_pton(socket.AF_INET6, addr) + except socket.error: # not a valid address + return False + return True + + +def verify_pypi_ssl_certificate(): + """ + This opens a socket to PyPI and checks if the SSL cert is correct. + """ + try: + import ssl + except ImportError: + logger.fatal('WARNING! Could not import the ssl module needed to ' + 'verify the SSL certificate of PyPI. Try installing ' + 'it by running (requires compiler): pip install ssl') + else: + ca_cert_dir = tempfile.mkdtemp('-ca-cert', 'pip-') + try: + try: + # write cacert root cert to temporary file + ca_cert_path = os.path.join(ca_cert_dir, 'root.crt') + ca_cert_file = open(ca_cert_path, 'w') + try: + ca_cert_file.write(CACERT_ROOT_CRT) + finally: + ca_cert_file.close() + + hostname = 'pypi.python.org' + + # replace host name with IP + addr = socket.getaddrinfo(hostname, 443)[0][4][0] + + # create socket and connect to server + # server address is specified later in connect() method + if valid_ipv6_addr(addr): + sock = socket.socket(socket.AF_INET6) + else: + sock = socket.socket() + sock.connect((addr, 443)) + + # wrap socket to add SSL support + sock = ssl.wrap_socket(sock, + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=ca_cert_path) + + # manual check of hostname + match_hostname(sock.getpeercert(), hostname) + + # log success + logger.info('Successfully verified SSL certificate of PyPI.') + + except (socket.error, socket.timeout, ValueError, + ssl.SSLError, CertificateError): + message = ('Could not verify the SSL certificate of ' + 'pypi.python.org! This might be caused by a ' + 'hickup while transmitting it, an out-of-date ' + 'root cert included in pip or an attempted man-' + 'in-the-middle attack.') + raise InstallationError(message) + + finally: + shutil.rmtree(ca_cert_dir) + + +class CertificateError(ValueError): + pass + + +def _dnsname_to_pat(dn): + pats = [] + for frag in dn.split(r'.'): + if frag == '*': + # When '*' is a fragment by itself, it matches a non-empty dotless + # fragment. + pats.append('[^.]+') + else: + # Otherwise, '*' matches any dotless fragment. + frag = re.escape(frag) + pats.append(frag.replace(r'\*', '[^.]*')) + return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) + + +def match_hostname(cert, hostname): + """ + Verify that *cert* (in decoded format as returned by + SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 rules + are mostly followed, but IP addresses are not accepted for *hostname*. + + CertificateError is raised on failure. On success, the function + returns nothing. + """ + if not cert: + raise ValueError("empty or no certificate") + dnsnames = [] + san = cert.get('subjectAltName', ()) + for key, value in san: + if key == 'DNS': + if _dnsname_to_pat(value).match(hostname): + return + dnsnames.append(value) + if not san: + # The subject is only checked when subjectAltName is empty + for sub in cert.get('subject', ()): + for key, value in sub: + # XXX according to RFC 2818, the most specific Common Name + # must be used. + if key == 'commonName': + if _dnsname_to_pat(value).match(hostname): + return + dnsnames.append(value) + if len(dnsnames) > 1: + raise CertificateError("hostname %r doesn't match either of %s" % + (hostname, ', '.join(map(repr, dnsnames)))) + elif len(dnsnames) == 1: + raise CertificateError("hostname %r doesn't match %r" % + (hostname, dnsnames[0])) + else: + raise CertificateError("no appropriate commonName or " + "subjectAltName fields were found") diff --git a/pip/index.py b/pip/index.py index 4381b110332..466d552faa9 100644 --- a/pip/index.py +++ b/pip/index.py @@ -1,5 +1,6 @@ -"""Routines related to PyPI, indexes""" - +""" +Routines related to PyPI, indexes, PEP381 mirrors +""" import sys import os import re @@ -12,16 +13,19 @@ import socket import string import zlib +from pip.locations import serverkey_file from pip.log import logger -from pip.util import Inf -from pip.util import normalize_name, splitext -from pip.exceptions import DistributionNotFound +from pip.util import Inf, normalize_name, splitext +from pip.exceptions import DistributionNotFound, InstallationError from pip.backwardcompat import (WindowsError, BytesIO, - Queue, httplib, urlparse, - URLError, HTTPError, u, - product, url2pathname) -from pip.backwardcompat import Empty as QueueEmpty -from pip.download import urlopen, path_to_url2, url_to_path, geturl, Urllib2HeadRequest + Queue, urlparse, + URLError, HTTPError, b, u, + product, url2pathname, + OrderedDict, _ord as ord, + decode_base64, _long, + Empty as QueueEmpty) +from pip.download import (urlopen, path_to_url2, url_to_path, geturl, + Urllib2HeadRequest) __all__ = ['PackageFinder'] @@ -30,14 +34,15 @@ class PackageFinder(object): - """This finds packages. + """ + This finds packages. This is meant to match easy_install's technique for looking for packages, by reading pages and looking for appropriate links """ - def __init__(self, find_links, index_urls, - use_mirrors=False, mirrors=None, main_mirror_url=None): + def __init__(self, find_links, index_urls, use_mirrors=False, + mirrors=None, main_mirror_url=None): self.find_links = find_links self.index_urls = index_urls self.dependency_links = [] @@ -46,9 +51,15 @@ def __init__(self, find_links, index_urls, self.logged_links = set() if use_mirrors: self.mirror_urls = self._get_mirror_urls(mirrors, main_mirror_url) - logger.info('Using PyPI mirrors: %s' % ', '.join(self.mirror_urls)) + logger.info('Using PyPI mirrors: %s' % + ', '.join([url.url for url in self.mirror_urls])) else: - self.mirror_urls = [] + self.mirror_urls = () + serverkey_cache = open(serverkey_file, 'rb') + try: + self.serverkey = load_key(serverkey_cache.read()) + finally: + serverkey_cache.close() def add_dependency_links(self, links): ## FIXME: this shouldn't be global list this, it should only @@ -61,138 +72,221 @@ def add_dependency_links(self, links): def _sort_locations(locations): """ Sort locations into "files" (archives) and "urls", and return - a pair of lists (files,urls) + a pair of lists (files, urls) """ files = [] urls = [] # puts the url for the given file path into the appropriate # list - def sort_path(path): - url = path_to_url2(path) - if mimetypes.guess_type(url, strict=False)[0] == 'text/html': + def sort_path(url, path): + new_url = path_to_url2(path) + mimetype = mimetypes.guess_type(new_url, strict=False)[0] + url.url = new_url + if mimetype == 'text/html': urls.append(url) else: files.append(url) for url in locations: - if url.startswith('file:'): - path = url_to_path(url) + if isinstance(url, Link): + url = url.copy() + else: + url = Link(url) + if url.url.startswith('file:'): + path = url_to_path(url.url) if os.path.isdir(path): path = os.path.realpath(path) for item in os.listdir(path): - sort_path(os.path.join(path, item)) + sort_path(url, os.path.join(path, item)) elif os.path.isfile(path): - sort_path(path) + sort_path(url, path) else: urls.append(url) return files, urls + def make_package_url(self, url, name): + """ + For maximum compatibility with easy_install, ensure the path + ends in a trailing slash. Although this isn't in the spec + (and PyPI can handle it without the slash) some other index + implementations might break if they relied on easy_install's + behavior. + """ + if isinstance(url, Link): + package_url = url.copy() + else: + package_url = Link(url) + new_url = posixpath.join(package_url.url, name) + if not new_url.endswith('/'): + new_url = new_url + '/' + package_url.url = new_url + return package_url + + def verify(self, requirement, url): + """ + Verifies the URL for the given requirement using the PEP381 + verification code. + """ + if url.comes_from: + try: + data = b(url.comes_from.content) + if data and requirement.serversig: + return verify(self.serverkey, data, requirement.serversig) + except ValueError: + return False + return False + def find_requirement(self, req, upgrade): url_name = req.url_name # Only check main index if index URL is given: main_index_url = None if self.index_urls: # Check that we have the url_name correctly spelled: - main_index_url = Link(posixpath.join(self.index_urls[0], url_name)) - # This will also cache the page, so it's okay that we get it again later: + main_index_url = self.make_package_url(self.index_urls[0], url_name) + # This will also cache the page, + # so it's okay that we get it again later: page = self._get_page(main_index_url, req) if page is None: - url_name = self._find_url_name(Link(self.index_urls[0]), url_name, req) or req.url_name + url_name = self._find_url_name( + Link(self.index_urls[0]), url_name, req) or req.url_name # Combine index URLs with mirror URLs here to allow # adding more index URLs from requirements files - all_index_urls = self.index_urls + self.mirror_urls - - def mkurl_pypi_url(url): - loc = posixpath.join(url, url_name) - # For maximum compatibility with easy_install, ensure the path - # ends in a trailing slash. Although this isn't in the spec - # (and PyPI can handle it without the slash) some other index - # implementations might break if they relied on easy_install's behavior. - if not loc.endswith('/'): - loc = loc + '/' - return loc + + locations = [] + indexes_package_urls = [] + mirrors_package_urls = [] if url_name is not None: - locations = [ - mkurl_pypi_url(url) - for url in all_index_urls] + self.find_links - else: - locations = list(self.find_links) - locations.extend(self.dependency_links) + indexes_package_urls = [self.make_package_url(url, url_name) + for url in self.index_urls] + locations.extend(indexes_package_urls) + mirrors_package_urls = [self.make_package_url(url, url_name) + for url in self.mirror_urls] + locations.extend(mirrors_package_urls) + + locations.extend(self.find_links + self.dependency_links) + for version in req.absolute_versions: if url_name is not None and main_index_url is not None: - locations = [ - posixpath.join(main_index_url.url, version)] + locations + version_url = posixpath.join(main_index_url.url, version) + locations = [version_url] + locations file_locations, url_locations = self._sort_locations(locations) - locations = [Link(url) for url in url_locations] + locations = [] + for url in url_locations: + if isinstance(url, Link): + locations.append(url) + else: + locations.append(Link(url)) logger.debug('URLs to search for versions for %s:' % req) for location in locations: logger.debug('* %s' % location) + found_versions = [] - found_versions.extend( - self._package_versions( - [Link(url, '-f') for url in self.find_links], req.name.lower())) + found_versions.extend(self._package_versions( + [Link(url, '-f') for url in self.find_links], req.name.lower())) + page_versions = [] for page in self._get_pages(locations, req): logger.debug('Analyzing links from page %s' % page.url) logger.indent += 2 try: - page_versions.extend(self._package_versions(page.links, req.name.lower())) + page_versions.extend(self._package_versions( + page.links, req.name.lower())) finally: logger.indent -= 2 + dependency_versions = list(self._package_versions( [Link(url) for url in self.dependency_links], req.name.lower())) if dependency_versions: - logger.info('dependency_links found: %s' % ', '.join([link.url for parsed, link, version in dependency_versions])) + dependency_urls = [link.url for _, link, _ in dependency_versions] + logger.info('dependency_links found: %s' % + ', '.join(dependency_urls)) + file_versions = list(self._package_versions( [Link(url) for url in file_locations], req.name.lower())) - if not found_versions and not page_versions and not dependency_versions and not file_versions: - logger.fatal('Could not find any downloads that satisfy the requirement %s' % req) - raise DistributionNotFound('No distributions at all found for %s' % req) + if (not found_versions and not page_versions and + not dependency_versions and not file_versions): + logger.fatal('Could not find any downloads that satisfy ' + 'the requirement %s' % req) + raise DistributionNotFound('No distributions at all found for %s' + % req) + if req.satisfied_by is not None: - found_versions.append((req.satisfied_by.parsed_version, Inf, req.satisfied_by.version)) + found_versions.append((req.satisfied_by.parsed_version, + Inf, req.satisfied_by.version)) + if file_versions: file_versions.sort(reverse=True) - logger.info('Local files found: %s' % ', '.join([url_to_path(link.url) for parsed, link, version in file_versions])) + file_urls = [url_to_path(link.url) for _, link, _ in file_versions] + logger.info('Local files found: %s' % ', '.join(file_urls)) found_versions = file_versions + found_versions + all_versions = found_versions + page_versions + dependency_versions - applicable_versions = [] - for (parsed_version, link, version) in all_versions: + + applicable_versions = OrderedDict() + for parsed_version, link, version in all_versions: if version not in req.req: - logger.info("Ignoring link %s, version %s doesn't match %s" - % (link, version, ','.join([''.join(s) for s in req.req.specs]))) + req_specs = [''.join(s) for s in req.req.specs] + logger.info("Ignoring link %s, version %s doesn't match %s" % + (link, version, ','.join(req_specs))) continue - applicable_versions.append((link, version)) - applicable_versions = sorted(applicable_versions, key=lambda v: pkg_resources.parse_version(v[1]), reverse=True) - existing_applicable = bool([link for link, version in applicable_versions if link is Inf]) + if link.comes_from in mirrors_package_urls: + link.is_mirror = True + applicable_versions.setdefault(version, []).append(link) + + for version in applicable_versions: + random.shuffle(applicable_versions[version]) + + applicable_versions = OrderedDict(sorted(applicable_versions.items(), + key=lambda v: pkg_resources.parse_version(v[0]), reverse=True)) + + existing_applicable = bool([link for link in [links + for links in applicable_versions.items()] + if link is Inf]) if not upgrade and existing_applicable: - if applicable_versions[0][1] is Inf: - logger.info('Existing installed version (%s) is most up-to-date and satisfies requirement' - % req.satisfied_by.version) + if Inf in applicable_versions: + logger.info('Existing installed version (%s) is most ' + 'up-to-date and satisfies requirement' % + req.satisfied_by.version) else: - logger.info('Existing installed version (%s) satisfies requirement (most up-to-date version is %s)' - % (req.satisfied_by.version, applicable_versions[0][1])) + logger.info('Existing installed version (%s) satisfies ' + 'requirement (most up-to-date version is %s)' % + (req.satisfied_by.version, + applicable_versions[0][1])) return None + if not applicable_versions: - logger.fatal('Could not find a version that satisfies the requirement %s (from versions: %s)' - % (req, ', '.join([version for parsed_version, link, version in found_versions]))) - raise DistributionNotFound('No distributions matching the version for %s' % req) - if applicable_versions[0][0] is Inf: - # We have an existing version, and its the best version - logger.info('Installed version (%s) is most up-to-date (past versions: %s)' - % (req.satisfied_by.version, ', '.join([version for link, version in applicable_versions[1:]]) or 'none')) + show_versions = [version for _, _, version in found_versions] + logger.fatal('Could not find a version that satisfies ' + 'the requirement %s (from versions: %s)' % + (req, ', '.join(show_versions))) + raise DistributionNotFound('No distributions matching ' + 'the version for %s' % req) + + newest = list(applicable_versions.keys())[0] + if Inf in applicable_versions: + # We have an existing version, and it's the best version + show_versions = [vers for vers in applicable_versions.keys()[1:]] + logger.info('Installed version (%s) is most up-to-date ' + '(past versions: %s)' % + (req.satisfied_by.version, + ', '.join(show_versions) or 'none')) return None + if len(applicable_versions) > 1: logger.info('Using version %s (newest of versions: %s)' % - (applicable_versions[0][1], ', '.join([version for link, version in applicable_versions]))) - return applicable_versions[0][0] + (newest, ', '.join(applicable_versions.keys()))) + + return applicable_versions[newest] def _find_url_name(self, index_url, url_name, req): - """Finds the true URL name of a package, when the given name isn't quite correct. - This is usually used to implement case-insensitivity.""" + """ + Finds the true URL name of a package, when the given name isn't + quite correct. This is usually used to implement case-insensitivity. + """ if not index_url.url.endswith('/'): # Vaguely part of the PyPI API... weird but true. ## FIXME: bad to modify this? @@ -205,7 +299,8 @@ def _find_url_name(self, index_url, url_name, req): for link in page.links: base = posixpath.basename(link.path.rstrip('/')) if norm_name == normalize_name(base): - logger.notify('Real name of requirement %s is %s' % (url_name, base)) + logger.notify('Real name of requirement %s is %s' % + (url_name, base)) return base return None @@ -219,12 +314,13 @@ def _get_pages(self, locations, req): seen = set() threads = [] for i in range(min(10, len(locations))): - t = threading.Thread(target=self._get_queued_page, args=(req, pending_queue, done, seen)) - t.setDaemon(True) - threads.append(t) - t.start() - for t in threads: - t.join() + thread = threading.Thread(target=self._get_queued_page, + args=(req, pending_queue, done, seen)) + thread.setDaemon(True) + threads.append(thread) + thread.start() + for thread in threads: + thread.join() return done _log_lock = threading.Lock() @@ -250,7 +346,10 @@ def _get_queued_page(self, req, pending_queue, done, seen): _py_version_re = re.compile(r'-py([123]\.[0-9])$') def _sort_links(self, links): - "Returns elements of links in order, non-egg links first, egg links second, while eliminating duplicates" + """ + Returns elements of links in order, non-egg links first, + egg links second, while eliminating duplicates + """ eggs, no_eggs = [], [] seen = set() for link in links: @@ -290,19 +389,22 @@ def _link_package_versions(self, link, search_name): ext = '.tar' + ext if ext not in ('.tar.gz', '.tar.bz2', '.tar', '.tgz', '.zip'): if link not in self.logged_links: - logger.debug('Skipping link %s; unknown archive format: %s' % (link, ext)) + logger.debug('Skipping link %s; unknown archive ' + 'format: %s' % (link, ext)) self.logged_links.add(link) return [] version = self._egg_info_matches(egg_info, search_name, link) if version is None: - logger.debug('Skipping link %s; wrong project name (not %s)' % (link, search_name)) + logger.debug('Skipping link %s; wrong project name (not %s)' % + (link, search_name)) return [] match = self._py_version_re.search(version) if match: version = version[:match.start()] py_version = match.group(1) if py_version != sys.version[:3]: - logger.debug('Skipping %s because Python version is incorrect' % link) + logger.debug('Skipping %s because Python ' + 'version is incorrect' % link) return [] logger.debug('Found link %s, version: %s' % (link, version)) return [(pkg_resources.parse_version(version), @@ -326,7 +428,8 @@ def _get_page(self, link, req): return HTMLPage.get_page(link, req, cache=self.cache) def _get_mirror_urls(self, mirrors=None, main_mirror_url=None): - """Retrieves a list of URLs from the main mirror DNS entry + """ + Retrieves a list of URLs from the main mirror DNS entry unless a list of mirror URLs are passed. """ if not mirrors: @@ -343,7 +446,7 @@ def _get_mirror_urls(self, mirrors=None, main_mirror_url=None): mirror_url = "%s/simple/" % mirror_url mirror_urls.add(mirror_url) - return list(mirror_urls) + return tuple(Link(url, is_mirror=True) for url in mirror_urls) class PageCache(object): @@ -369,7 +472,7 @@ def set_is_archive(self, url, value=True): self._archives[url] = value def add_page_failure(self, url, level): - self._failures[url] = self._failures.get(url, 0)+level + self._failures[url] = self._failures.get(url, 0) + level def add_page(self, urls, page): for url in urls: @@ -384,7 +487,8 @@ class HTMLPage(object): _download_re = re.compile(r'\s*download\s+url', re.I) ## These aren't so aweful: _rel_re = re.compile("""<[^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*>""", re.I) - _href_re = re.compile('href=(?:"([^"]*)"|\'([^\']*)\'|([^>\\s\\n]*))', re.I|re.S) + _href_re = re.compile('href=(?:"([^"]*)"|\'([^\']*)\'|([^>\\s\\n]*))', + re.I | re.S) _base_re = re.compile(r"""]+)""", re.I) def __init__(self, content, url, headers=None): @@ -406,7 +510,8 @@ def get_page(cls, link, req, cache=None, skip_archives=True): from pip.vcs import VcsSupport for scheme in VcsSupport.schemes: if url.lower().startswith(scheme) and url[len(scheme)] in '+:': - logger.debug('Cannot look at %(scheme)s URL %(link)s' % locals()) + logger.debug('Cannot look at %s URL %s' % + (scheme, link)) return None if cache is not None: @@ -425,16 +530,20 @@ def get_page(cls, link, req, cache=None, skip_archives=True): if content_type.lower().startswith('text/html'): break else: - logger.debug('Skipping page %s because of Content-Type: %s' % (link, content_type)) + logger.debug('Skipping page %s because of ' + 'Content-Type: %s' % + (link, content_type)) if cache is not None: cache.set_is_archive(url) return None logger.debug('Getting page %s' % url) # Tack index.html onto file:// URLs that point to directories - (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(url) + parsed_url = urlparse.urlparse(url) + scheme, netloc, path, params, query, fragment = parsed_url if scheme == 'file' and os.path.isdir(url2pathname(path)): - # add trailing slash if not present so urljoin doesn't trim final segment + # add trailing slash if not present so urljoin + # doesn't trim final segment if not url.endswith('/'): url += '/' url = urlparse.urljoin(url, 'index.html') @@ -453,16 +562,18 @@ def get_page(cls, link, req, cache=None, skip_archives=True): if encoding == 'deflate': contents = zlib.decompress(contents) inst = cls(u(contents), real_url, headers) - except (HTTPError, URLError, socket.timeout, socket.error, OSError, WindowsError): + except (HTTPError, URLError, socket.timeout, + socket.error, OSError, WindowsError): e = sys.exc_info()[1] desc = str(e) if isinstance(e, socket.timeout): log_meth = logger.info - level =1 + level = 1 desc = 'timed out' elif isinstance(e, URLError): log_meth = logger.info - if hasattr(e, 'reason') and isinstance(e.reason, socket.timeout): + if (hasattr(e, 'reason') and + isinstance(e.reason, socket.timeout)): desc = 'timed out' level = 1 else: @@ -475,7 +586,8 @@ def get_page(cls, link, req, cache=None, skip_archives=True): log_meth = logger.info level = 1 log_meth('Could not fetch URL %s: %s' % (link, desc)) - log_meth('Will skip URL %s when looking for download links for %s' % (link.url, req)) + log_meth('Will skip URL %s when looking for ' + 'download links for %s' % (link.url, req)) if cache is not None: cache.add_page_failure(url, level) return None @@ -494,7 +606,8 @@ def _get_content_type(url): req = Urllib2HeadRequest(url, headers={'Host': netloc}) resp = urlopen(req) try: - if hasattr(resp, 'code') and resp.code != 200 and scheme not in ('ftp', 'ftps'): + if (hasattr(resp, 'code') and + resp.code != 200 and scheme not in ('ftp', 'ftps')): ## FIXME: doesn't handle redirects return '' return resp.info().get('content-type', '') @@ -513,7 +626,9 @@ def base_url(self): @property def links(self): - """Yields all links in the page""" + """ + Yields all links in the page + """ for match in self._href_re.finditer(self.content): url = match.group(1) or match.group(2) or match.group(3) url = self.clean_link(urlparse.urljoin(self.base_url, url)) @@ -526,7 +641,9 @@ def rel_links(self): yield url def explicit_rel_links(self, rels=('homepage', 'download')): - """Yields all links with the given relations""" + """ + Yields all links with the given relations + """ for match in self._rel_re.finditer(self.content): found_rels = match.group(1).lower().split() for rel in rels: @@ -555,21 +672,26 @@ def scraped_rel_links(self): url = self.clean_link(urlparse.urljoin(self.base_url, url)) yield Link(url, self) - _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) + _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%~_\\|-]', re.I) def clean_link(self, url): - """Makes sure a link is fully encoded. That is, if a ' ' shows up in + """ + Makes sure a link is fully encoded. That is, if a ' ' shows up in the link, it will be rewritten to %20 (while not over-quoting - % or other characters).""" - return self._clean_re.sub( - lambda match: '%%%2x' % ord(match.group(0)), url) + % or other characters). + """ + def replacer(match): + matched_group = match.group(0) + return '%%%2x' % ord(matched_group) + return self._clean_re.sub(replacer, url.strip()) class Link(object): - def __init__(self, url, comes_from=None): + def __init__(self, url, comes_from=None, is_mirror=False): self.url = url self.comes_from = comes_from + self.is_mirror = is_mirror def __str__(self): if self.comes_from: @@ -634,6 +756,10 @@ def md5_hash(self): def show_url(self): return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) + def copy(self): + return self.__class__(self.url, comes_from=self.comes_from, + is_mirror=self.is_mirror) + def get_requirement_from_url(url): """Get a requirement from the URL, if possible. This looks for #egg @@ -686,14 +812,203 @@ def get_mirrors(hostname=None): def string_range(last): - """Compute the range of string between "a" and last. + """ + Compute the range of string between "a" and last. This works for simple "a to z" lists, but also for "a to zz" lists. """ for k in range(len(last)): - for x in product(string.ascii_lowercase, repeat=k+1): + for x in product(string.ascii_lowercase, repeat=k + 1): result = ''.join(x) yield result if result == last: return + +# Distribute and use freely; there are no restrictions on further +# dissemination and usage except those imposed by the laws of your +# country of residence. This software is provided "as is" without +# warranty of fitness for use or suitability for any purpose, express +# or implied. Use at your own risk or not at all. +""" +Verify a DSA signature, for use with PyPI mirrors. + +Verification should use the following steps: +1. Download the DSA key from http://pypi.python.org/serverkey, as key_string +2. key = load_key(key_string) +3. Download the package page, from /simple//, as data +4. Download the package signature, from /serversig/, as sig +5. Check verify(key, data, sig) +""" + +try: + from M2Crypto import EVP, DSA, BIO + + def load_key(string): + """ + load_key(string) -> key + + Convert a PEM format public DSA key into + an internal representation. + """ + return DSA.load_pub_key_bio(BIO.MemoryBuffer(string)) + + def verify(key, data, sig): + """ + verify(key, data, sig) -> bool + + Verify autenticity of the signature created by key for + data. data is the bytes that got signed; signature is the + bytes that represent the signature, using the sha1+DSA + algorithm. key is an internal representation of the DSA key + as returned from load_key.""" + md = EVP.MessageDigest('sha1') + md.update(data) + digest = md.final() + return key.verify_asn1(digest, sig) + +except ImportError: + + # DSA signature algorithm, taken from pycrypto 2.0.1 + # The license terms are the same as the ones for this module. + def _inverse(u, v): + """ + _inverse(u:long, u:long):long + Return the inverse of u mod v. + """ + u3, v3 = _long(u), _long(v) + u1, v1 = _long(1), _long(0) + while v3 > 0: + q = u3 // v3 + u1, v1 = v1, u1 - v1 * q + u3, v3 = v3, u3 - v3 * q + while u1 < 0: + u1 = u1 + v + return u1 + + def _verify(key, M, sig): + p, q, g, y = key + r, s = sig + if r <= 0 or r >= q or s <= 0 or s >= q: + return False + w = _inverse(s, q) + u1, u2 = (M * w) % q, (r * w) % q + v1 = pow(g, u1, p) + v2 = pow(y, u2, p) + v = (v1 * v2) % p + v = v % q + return v == r + + # END OF pycrypto + + def _bytes2int(b): + value = 0 + for c in b: + value = value * 256 + ord(c) + return value + + _SEQUENCE = 0x30 # cons + _INTEGER = 2 # prim + _BITSTRING = 3 # prim + _OID = 6 # prim + + def _asn1parse(string): + tag = ord(string[0]) + assert tag & 31 != 31 # only support one-byte tags + length = ord(string[1]) + assert length != 128 # indefinite length not supported + pos = 2 + if length > 128: + # multi-byte length + val = 0 + length -= 128 + val = _bytes2int(string[pos:pos + length]) + pos += length + length = val + data = string[pos:pos + length] + rest = string[pos + length:] + assert pos + length <= len(string) + if tag == _SEQUENCE: + result = [] + while data: + value, data = _asn1parse(data) + result.append(value) + elif tag == _INTEGER: + assert ord(data[0]) < 128 # negative numbers not supported + result = 0 + for c in data: + result = result * 256 + ord(c) + elif tag == _BITSTRING: + result = data + elif tag == _OID: + result = data + else: + raise ValueError("Unsupported tag %x" % tag) + return (tag, result), rest + + def load_key(string): + """ + load_key(string) -> key + + Convert a PEM format public DSA key into + an internal representation.""" + lines = [line.strip() for line in string.splitlines()] + assert lines[0] == b("-----BEGIN PUBLIC KEY-----") + assert lines[-1] == b("-----END PUBLIC KEY-----") + data = decode_base64(''.join([u(line) for line in lines[1:-1]])) + spki, rest = _asn1parse(data) + assert not rest + # SubjectPublicKeyInfo ::= SEQUENCE { + # algorithm AlgorithmIdentifier, + # subjectPublicKey BIT STRING } + assert spki[0] == _SEQUENCE + algoid, key = spki[1] + assert key[0] == _BITSTRING + key = key[1] + # AlgorithmIdentifier ::= SEQUENCE { + # algorithm OBJECT IDENTIFIER, + # parameters ANY DEFINED BY algorithm OPTIONAL } + assert algoid[0] == _SEQUENCE + algorithm, parameters = algoid[1] + # dsaEncryption + # assert algorithm[0] == _OID and algorithm[1] == '*\x86H\xce8\x04\x01' + # Dss-Parms ::= SEQUENCE { + # p INTEGER, + # q INTEGER, + # g INTEGER } + assert parameters[0] == _SEQUENCE + p, q, g = parameters[1] + assert p[0] == q[0] == g[0] == _INTEGER + p, q, g = p[1], q[1], g[1] + # Parse bit string value as integer + # assert key[0] == '\0' # number of bits multiple of 8 + y, rest = _asn1parse(key[1:]) + assert not rest + assert y[0] == _INTEGER + y = y[1] + return p, q, g, y + + def verify(key, data, sig): + """ + verify(key, data, sig) -> bool + + Verify autenticity of the signature created by key for + data. data is the bytes that got signed; signature is the + bytes that represent the signature, using the sha1+DSA + algorithm. key is an internal representation of the DSA key + as returned from load_key.""" + from hashlib import sha1 + sha = sha1() + sha.update(data) + data = sha.digest() + data = _bytes2int(data) + # Dss-Sig-Value ::= SEQUENCE { + # r INTEGER, + # s INTEGER } + sig, rest = _asn1parse(sig) + assert not rest + assert sig[0] == _SEQUENCE + r, s = sig[1] + assert r[0] == s[0] == _INTEGER + sig = r[1], s[1] + return _verify(key, data, sig) diff --git a/pip/locations.py b/pip/locations.py index b439bd375fe..c87792be822 100644 --- a/pip/locations.py +++ b/pip/locations.py @@ -48,3 +48,5 @@ def running_under_virtualenv(): # Forcing to use /usr/local/bin for standard Mac OS X framework installs if sys.platform[:6] == 'darwin' and sys.prefix[:16] == '/System/Library/': bin_py = '/usr/local/bin' + +serverkey_file = os.path.join(default_storage_dir, 'serverkey.pub') diff --git a/pip/req.py b/pip/req.py index 4f2d5fad222..558dead433e 100644 --- a/pip/req.py +++ b/pip/req.py @@ -1,28 +1,30 @@ -import sys import os -import shutil import re -import zipfile -import pkg_resources +import shutil +import socket +import sys import tempfile -from pip.locations import bin_py, running_under_virtualenv +import zipfile +from pkg_resources import (get_distribution, Requirement, PY_MAJOR, + VersionConflict, DistributionNotFound) + +from pip import call_subprocess +from pip.index import Link +from pip.log import logger +from pip.locations import bin_py, running_under_virtualenv, build_prefix from pip.exceptions import InstallationError, UninstallationError from pip.vcs import vcs -from pip.log import logger -from pip.util import display_path, rmtree -from pip.util import ask, backup_dir -from pip.util import is_installable_dir, is_local, dist_is_local -from pip.util import renames, normalize_path, egg_link_path -from pip.util import make_path_relative -from pip import call_subprocess from pip.backwardcompat import (any, copytree, urlparse, urllib, ConfigParser, string_types, HTTPError, - FeedParser, get_python_version, - b) -from pip.index import Link -from pip.locations import build_prefix + FeedParser, get_python_version, b, + WindowsError, URLError) + + +from pip.util import (display_path, rmtree, ask, backup_dir, + is_installable_dir, is_local, dist_is_local, renames, + normalize_path, egg_link_path, make_path_relative) from pip.download import (get_file_content, is_url, url_to_path, - path_to_url, is_archive_file, + path_to_url, is_archive_file, urlopen, unpack_vcs_link, is_vcs_url, is_file_url, unpack_file_url, unpack_http_url) @@ -35,7 +37,7 @@ class InstallRequirement(object): def __init__(self, req, comes_from, source_dir=None, editable=False, url=None, update=True): if isinstance(req, string_types): - req = pkg_resources.Requirement.parse(req) + req = Requirement.parse(req) self.req = req self.comes_from = comes_from self.source_dir = source_dir @@ -56,6 +58,8 @@ def __init__(self, req, comes_from, source_dir=None, editable=False, self.install_succeeded = None # UninstallPathSet of uninstalled distribution (for possible rollback) self.uninstalled = None + # The server signature from PyPI + self._serversig = None @classmethod def from_editable(cls, editable_req, comes_from=None, default_vcs=None): @@ -68,7 +72,8 @@ def from_editable(cls, editable_req, comes_from=None, default_vcs=None): @classmethod def from_line(cls, name, comes_from=None): - """Creates an InstallRequirement from a name, which might be a + """ + Creates an InstallRequirement from a name, which might be a requirement, directory containing 'setup.py', filename, or URL. """ url = None @@ -79,17 +84,21 @@ def from_line(cls, name, comes_from=None): if is_url(name): link = Link(name) - elif os.path.isdir(path) and (os.path.sep in name or name.startswith('.')): + elif os.path.isdir(path) and ( + os.path.sep in name or name.startswith('.')): if not is_installable_dir(path): - raise InstallationError("Directory %r is not installable. File 'setup.py' not found.", name) + raise InstallationError("Directory %r is not installable. " + "File 'setup.py' not found.", name) link = Link(path_to_url(name)) elif is_archive_file(path): if not os.path.isfile(path): - logger.warn('Requirement %r looks like a filename, but the file does not exist', name) + logger.warn('Requirement %r looks like a filename, ' + 'but the file does not exist', name) link = Link(path_to_url(name)) - # If the line has an egg= definition, but isn't editable, pull the requirement out. - # Otherwise, assume the name is the req for the non URL/path/archive case. + # If the line has an egg= definition, but isn't editable, + # pull the requirement out. Otherwise, assume the name is + # the req for the non URL/path/archive case. if link and req is None: url = link.url_fragment req = link.egg_fragment @@ -195,6 +204,22 @@ def url_name(self): def setup_py(self): return os.path.join(self.source_dir, 'setup.py') + @property + def serversig(self): + if self._serversig is None: + if self.req is not None: + sig_url = 'https://pypi.python.org/serversig/%s/' % self.url_name + try: + self._serversig = urlopen(sig_url).read() + except (HTTPError, URLError, socket.timeout, + socket.error, OSError, WindowsError): + # return empty string in case this was just a + # temporary connection failure + return '' + else: + self._serversig = '' + return self._serversig + def run_egg_info(self, force_root_egg_info=False): assert self.source_dir if self.name: @@ -223,7 +248,7 @@ def run_egg_info(self, force_root_egg_info=False): finally: logger.indent -= 2 if not self.req: - self.req = pkg_resources.Requirement.parse( + self.req = Requirement.parse( "%(Name)s==%(Version)s" % self.pkg_info()) self.correct_build_location() @@ -411,8 +436,7 @@ def uninstall(self, auto_confirm=False): pip_egg_info_path = os.path.join(dist.location, dist.egg_name()) + '.egg-info' # workaround for http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=618367 - debian_egg_info_path = pip_egg_info_path.replace( - '-py%s' % pkg_resources.PY_MAJOR, '') + debian_egg_info_path = pip_egg_info_path.replace('-py%s' % PY_MAJOR, '') easy_install_egg = dist.egg_name() + '.egg' develop_egg_link = egg_link_path(dist) @@ -473,11 +497,12 @@ def uninstall(self, auto_confirm=False): config.readfp(FakeFile(dist.get_metadata_lines('entry_points.txt'))) if config.has_section('console_scripts'): for name, value in config.items('console_scripts'): - paths_to_remove.add(os.path.join(bin_py, name)) + this_bin = os.path.join(bin_py, name) + paths_to_remove.add(this_bin) if sys.platform == 'win32': - paths_to_remove.add(os.path.join(bin_py, name) + '.exe') - paths_to_remove.add(os.path.join(bin_py, name) + '.exe.manifest') - paths_to_remove.add(os.path.join(bin_py, name) + '-script.py') + paths_to_remove.add(this_bin + '.exe') + paths_to_remove.add(this_bin + '.exe.manifest') + paths_to_remove.add(this_bin + '-script.py') paths_to_remove.remove(auto_confirm) self.uninstalled = paths_to_remove @@ -486,15 +511,15 @@ def rollback_uninstall(self): if self.uninstalled: self.uninstalled.rollback() else: - logger.error("Can't rollback %s, nothing uninstalled." - % (self.project_name,)) + logger.error("Can't rollback %s, nothing uninstalled." % + self.project_name) def commit_uninstall(self): if self.uninstalled: self.uninstalled.commit() else: - logger.error("Can't commit %s, nothing uninstalled." - % (self.project_name,)) + logger.error("Can't commit %s, nothing uninstalled." % + self.project_name) def archive(self, build_dir): assert self.source_dir @@ -524,7 +549,7 @@ def archive(self, build_dir): dirname = os.path.join(dirpath, dirname) name = self._clean_zip_name(dirname, dir) zipdir = zipfile.ZipInfo(self.name + '/' + name + '/') - zipdir.external_attr = 0x1ED << 16 # 0o755 + zipdir.external_attr = 0x1ED << 16 # 0o755 zip.writestr(zipdir, '') for filename in filenames: if filename == PIP_DELETE_MARKER_FILENAME: @@ -537,9 +562,9 @@ def archive(self, build_dir): logger.notify('Saved %s' % display_path(archive_path)) def _clean_zip_name(self, name, prefix): - assert name.startswith(prefix+os.path.sep), ( + assert name.startswith(prefix + os.path.sep), ( "name %r doesn't start with prefix %r" % (name, prefix)) - name = name[len(prefix)+1:] + name = name[len(prefix) + 1:] name = name.replace(os.path.sep, '/') return name @@ -597,7 +622,7 @@ def install(self, install_options, global_options=()): new_lines.append(make_path_relative(filename, egg_info_dir)) f.close() f = open(os.path.join(egg_info_dir, 'installed-files.txt'), 'w') - f.write('\n'.join(new_lines)+'\n') + f.write('\n'.join(new_lines) + '\n') f.close() finally: if os.path.exists(record_filename): @@ -651,11 +676,11 @@ def check_if_exists(self): if self.req is None: return False try: - self.satisfied_by = pkg_resources.get_distribution(self.req) - except pkg_resources.DistributionNotFound: + self.satisfied_by = get_distribution(self.req) + except DistributionNotFound: return False - except pkg_resources.VersionConflict: - self.conflicts_with = pkg_resources.get_distribution(self.req.project_name) + except VersionConflict: + self.conflicts_with = get_distribution(self.req.project_name) return True @property @@ -780,6 +805,8 @@ def __init__(self, build_dir, src_dir, download_dir, download_cache=None, self.build_dir = build_dir self.src_dir = src_dir self.download_dir = download_dir + if download_cache is not None: + download_cache = os.path.expanduser(download_cache) self.download_cache = download_cache self.upgrade = upgrade self.ignore_installed = ignore_installed @@ -894,141 +921,169 @@ def locate_files(self): % (req_to_install, req_to_install.source_dir)) def prepare_files(self, finder, force_root_egg_info=False, bundle=False): - """Prepare process. Create temp directories, download and/or unpack files.""" + """ + Prepare process. Create temp directories, download + and/or unpack files. + """ unnamed = list(self.unnamed_requirements) reqs = list(self.requirements.values()) while reqs or unnamed: if unnamed: - req_to_install = unnamed.pop(0) + requirement = unnamed.pop(0) else: - req_to_install = reqs.pop(0) + requirement = reqs.pop(0) install = True - if not self.ignore_installed and not req_to_install.editable: - req_to_install.check_if_exists() - if req_to_install.satisfied_by: + if not self.ignore_installed and not requirement.editable: + requirement.check_if_exists() + if requirement.satisfied_by: if self.upgrade: - req_to_install.conflicts_with = req_to_install.satisfied_by - req_to_install.satisfied_by = None + requirement.conflicts_with = requirement.satisfied_by + requirement.satisfied_by = None else: install = False - if req_to_install.satisfied_by: + if requirement.satisfied_by: logger.notify('Requirement already satisfied ' '(use --upgrade to upgrade): %s' - % req_to_install) - if req_to_install.editable: - logger.notify('Obtaining %s' % req_to_install) + % requirement) + if requirement.editable: + logger.notify('Obtaining %s' % requirement) elif install: - if req_to_install.url and req_to_install.url.lower().startswith('file:'): - logger.notify('Unpacking %s' % display_path(url_to_path(req_to_install.url))) + if requirement.url and requirement.url.lower().startswith('file:'): + logger.notify('Unpacking %s' % + display_path(url_to_path(requirement.url))) else: - logger.notify('Downloading/unpacking %s' % req_to_install) + logger.notify('Downloading/unpacking %s' % requirement) logger.indent += 2 try: is_bundle = False - if req_to_install.editable: - if req_to_install.source_dir is None: - location = req_to_install.build_location(self.src_dir) - req_to_install.source_dir = location + if requirement.editable: + if requirement.source_dir is None: + location = requirement.build_location(self.src_dir) + requirement.source_dir = location else: - location = req_to_install.source_dir + location = requirement.source_dir if not os.path.exists(self.build_dir): _make_build_dir(self.build_dir) - req_to_install.update_editable(not self.is_download) + requirement.update_editable(not self.is_download) if self.is_download: - req_to_install.run_egg_info() - req_to_install.archive(self.download_dir) + requirement.run_egg_info() + requirement.archive(self.download_dir) else: - req_to_install.run_egg_info() + requirement.run_egg_info() elif install: ##@@ if filesystem packages are not marked ##editable in a req, a non deterministic error ##occurs when the script attempts to unpack the ##build directory - location = req_to_install.build_location(self.build_dir, not self.is_download) + location = requirement.build_location( + self.build_dir, not self.is_download) ## FIXME: is the existance of the checkout good enough to use it? I don't think so. unpack = True if not os.path.exists(os.path.join(location, 'setup.py')): ## FIXME: this won't upgrade when there's an existing package unpacked in `location` - if req_to_install.url is None: - url = finder.find_requirement(req_to_install, upgrade=self.upgrade) + if requirement.url is None: + urls = finder.find_requirement(requirement, upgrade=self.upgrade) else: - ## FIXME: should req_to_install.url already be a link? - url = Link(req_to_install.url) - assert url - if url: - try: - self.unpack_url(url, location, self.is_download) - except HTTPError: - e = sys.exc_info()[1] - logger.fatal('Could not install requirement %s because of error %s' - % (req_to_install, e)) - raise InstallationError( - 'Could not install requirement %s because of HTTP error %s for URL %s' - % (req_to_install, e, url)) + ## FIXME: should requirement.url already be a link? + urls = [Link(requirement.url)] + if urls: + # Trying each of the returned URLs one by one + for url in urls: + if url.is_mirror: + if finder.verify(requirement, url): + logger.info('Successfully verified ' + 'mirror URL %s for requirement ' + '%s.' % (url, requirement)) + else: + logger.warn('Could not verify mirror ' + 'URL %s for requirement %s. ' + 'Skipping.' % (url, requirement)) + continue + try: + self.unpack_url(url, location, self.is_download) + except HTTPError: + e = sys.exc_info()[1] + logger.fatal('Could not install ' + 'requirement %s because of error %s' % + (requirement, e)) + raise InstallationError('Could not ' + 'install requirement %s because of ' + 'HTTP error %s for URL %s' % + (requirement, e, url)) + else: + # stop trying after successful retrieval + break + else: + raise InstallationError('Could not install ' + 'requirement %s because no valid URLs ' + 'were found.' % requirement) else: unpack = False if unpack: - is_bundle = req_to_install.is_bundle + is_bundle = requirement.is_bundle url = None if is_bundle: - req_to_install.move_bundle_files(self.build_dir, self.src_dir) - for subreq in req_to_install.bundle_requirements(): + requirement.move_bundle_files(self.build_dir, self.src_dir) + for subreq in requirement.bundle_requirements(): reqs.append(subreq) self.add_requirement(subreq) elif self.is_download: - req_to_install.source_dir = location + requirement.source_dir = location if url and url.scheme in vcs.all_schemes: - req_to_install.run_egg_info() - req_to_install.archive(self.download_dir) + requirement.run_egg_info() + requirement.archive(self.download_dir) else: - req_to_install.source_dir = location - req_to_install.run_egg_info() + requirement.source_dir = location + requirement.run_egg_info() if force_root_egg_info: # We need to run this to make sure that the .egg-info/ # directory is created for packing in the bundle - req_to_install.run_egg_info(force_root_egg_info=True) - req_to_install.assert_source_matches_version() + requirement.run_egg_info(force_root_egg_info=True) + requirement.assert_source_matches_version() #@@ sketchy way of identifying packages not grabbed from an index - if bundle and req_to_install.url: - self.copy_to_build_dir(req_to_install) + if bundle and requirement.url: + self.copy_to_build_dir(requirement) install = False - # req_to_install.req is only avail after unpack for URL pkgs + # requirement.req is only avail after unpack for URL pkgs # repeat check_if_exists to uninstall-on-upgrade (#14) - req_to_install.check_if_exists() - if req_to_install.satisfied_by: + requirement.check_if_exists() + if requirement.satisfied_by: if self.upgrade or self.ignore_installed: - req_to_install.conflicts_with = req_to_install.satisfied_by - req_to_install.satisfied_by = None + requirement.conflicts_with = requirement.satisfied_by + requirement.satisfied_by = None else: install = False if not is_bundle and not self.is_download: ## FIXME: shouldn't be globally added: - finder.add_dependency_links(req_to_install.dependency_links) + finder.add_dependency_links(requirement.dependency_links) ## FIXME: add extras in here: if not self.ignore_dependencies: - for req in req_to_install.requirements(): + for req in requirement.requirements(): try: - name = pkg_resources.Requirement.parse(req).project_name + name = Requirement.parse(req).project_name except ValueError: e = sys.exc_info()[1] ## FIXME: proper warning - logger.error('Invalid requirement: %r (%s) in requirement %s' % (req, e, req_to_install)) + logger.error('Invalid requirement: %r (%s) ' + 'in requirement %s' % + (req, e, requirement)) continue if self.has_requirement(name): ## FIXME: check for conflict continue - subreq = InstallRequirement(req, req_to_install) + subreq = InstallRequirement(req, requirement) reqs.append(subreq) self.add_requirement(subreq) - if req_to_install.name not in self.requirements: - self.requirements[req_to_install.name] = req_to_install + if requirement.name not in self.requirements: + self.requirements[requirement.name] = requirement else: - self.reqs_to_cleanup.append(req_to_install) + self.reqs_to_cleanup.append(requirement) if install: - self.successfully_downloaded.append(req_to_install) - if bundle and (req_to_install.url and req_to_install.url.startswith('file:///')): - self.copy_to_build_dir(req_to_install) + self.successfully_downloaded.append(requirement) + if bundle and (requirement.url and + requirement.url.startswith('file:///')): + self.copy_to_build_dir(requirement) finally: logger.indent -= 2 @@ -1074,23 +1129,26 @@ def unpack_url(self, link, location, only_download=False): elif is_file_url(link): return unpack_file_url(link, location) else: - if self.download_cache: - self.download_cache = os.path.expanduser(self.download_cache) - return unpack_http_url(link, location, self.download_cache, only_download) + return unpack_http_url(link, location, + self.download_cache, only_download) def install(self, install_options, global_options=()): - """Install everything in this set (after having downloaded and unpacked the packages)""" + """ + Install everything in this set + (after having downloaded and unpacked the packages) + """ to_install = [r for r in self.requirements.values() if self.upgrade or not r.satisfied_by] if to_install: - logger.notify('Installing collected packages: %s' % ', '.join([req.name for req in to_install])) + logger.notify('Installing collected packages: %s' % + ', '.join([req.name for req in to_install])) logger.indent += 2 try: for requirement in to_install: if requirement.conflicts_with: - logger.notify('Found existing installation: %s' - % requirement.conflicts_with) + logger.notify('Found existing installation: %s' % + requirement.conflicts_with) logger.indent += 2 try: requirement.uninstall(auto_confirm=True) @@ -1181,9 +1239,9 @@ def bundle_requirements(self): return ''.join(parts) def _clean_zip_name(self, name, prefix): - assert name.startswith(prefix+os.path.sep), ( + assert name.startswith(prefix + os.path.sep), ( "name %r doesn't start with prefix %r" % (name, prefix)) - name = name[len(prefix)+1:] + name = name[len(prefix) + 1:] name = name.replace(os.path.sep, '/') return name diff --git a/tests/test_pip.py b/tests/test_pip.py index 92cfff7d764..7e7d57bcddb 100644 --- a/tests/test_pip.py +++ b/tests/test_pip.py @@ -23,6 +23,7 @@ site_packages_suffix = site.USER_SITE[len(site.USER_BASE) + 1:] + def path_to_url(path): """ Convert a path to URI. The path will be made absolute and @@ -282,6 +283,7 @@ def __init__(self, environ=None, use_distribute=None): environ['PIP_NO_INPUT'] = '1' environ['PIP_LOG_FILE'] = str(self.root_path/'pip-log.txt') + environ['PIP_USE_MIRRORS'] = 'false' super(TestPipEnvironment, self).__init__( self.root_path, ignore_hidden=False,