Extended PEP 381 mirror and package verification features.

pip/backwardcompat.py

@@ -3,6 +3,7 @@
 import sys
 import os
 import shutil
+import base64
 
 __all__ = ['any', 'WindowsError', 'md5', 'copytree']
 
@@ -60,11 +61,21 @@ def b(s):
     def u(s):
         return s.decode('utf-8')
 
+    def _ord(x):
+        return x
+
     def console_to_str(s):
         return s.decode(console_encoding)
     bytes = bytes
     string_types = (str,)
     raw_input = input
+
+    def decode_base64(source):
+        source = source.encode("ascii")  # ensure bytes
+        return base64.decodebytes(source)
+
+    _long = lambda x: x
+
 else:
     from cStringIO import StringIO
     from urllib2 import URLError, HTTPError
@@ -84,6 +95,9 @@ def b(s):
     def u(s):
         return s
 
+    def _ord(x):
+        return ord(x)
+
     def console_to_str(s):
         return s
     bytes = str
@@ -92,6 +106,8 @@ def console_to_str(s):
     cmp = cmp
     raw_input = raw_input
     BytesIO = StringIO
+    decode_base64 = base64.decodestring
+    _long = lambda x: long(x)
 
 try:
     from email.parser import FeedParser
@@ -122,3 +138,134 @@ def product(*args, **kwds):
         result = [x+[y] for x in result for y in pool]
     for prod in result:
         yield tuple(prod)
+
+try:
+    from collections import OrderedDict
+except ImportError:
+    # Copyright (c) 2009 Raymond Hettinger
+    #
+    # Permission is hereby granted, free of charge, to any person
+    # obtaining a copy of this software and associated documentation files
+    # (the "Software"), to deal in the Software without restriction,
+    # including without limitation the rights to use, copy, modify, merge,
+    # publish, distribute, sublicense, and/or sell copies of the Software,
+    # and to permit persons to whom the Software is furnished to do so,
+    # subject to the following conditions:
+    #
+    #     The above copyright notice and this permission notice shall be
+    #     included in all copies or substantial portions of the Software.
+    #
+    #     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    #     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+    #     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    #     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+    #     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+    #     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+    #     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+    #     OTHER DEALINGS IN THE SOFTWARE.
+    from UserDict import DictMixin
+
+    class OrderedDict(dict, DictMixin):
+
+        def __init__(self, *args, **kwds):
+            if len(args) > 1:
+                raise TypeError('expected at most 1 arguments, got %d' % len(args))
+            try:
+                self.__end
+            except AttributeError:
+                self.clear()
+            self.update(*args, **kwds)
+
+        def clear(self):
+            self.__end = end = []
+            end += [None, end, end]         # sentinel node for doubly linked list
+            self.__map = {}                 # key --> [key, prev, next]
+            dict.clear(self)
+
+        def __setitem__(self, key, value):
+            if key not in self:
+                end = self.__end
+                curr = end[1]
+                curr[2] = end[1] = self.__map[key] = [key, curr, end]
+            dict.__setitem__(self, key, value)
+
+        def __delitem__(self, key):
+            dict.__delitem__(self, key)
+            key, prev, next = self.__map.pop(key)
+            prev[2] = next
+            next[1] = prev
+
+        def __iter__(self):
+            end = self.__end
+            curr = end[2]
+            while curr is not end:
+                yield curr[0]
+                curr = curr[2]
+
+        def __reversed__(self):
+            end = self.__end
+            curr = end[1]
+            while curr is not end:
+                yield curr[0]
+                curr = curr[1]
+
+        def popitem(self, last=True):
+            if not self:
+                raise KeyError('dictionary is empty')
+            if last:
+                key = reversed(self).next()
+            else:
+                key = iter(self).next()
+            value = self.pop(key)
+            return key, value
+
+        def __reduce__(self):
+            items = [[k, self[k]] for k in self]
+            tmp = self.__map, self.__end
+            del self.__map, self.__end
+            inst_dict = vars(self).copy()
+            self.__map, self.__end = tmp
+            if inst_dict:
+                return (self.__class__, (items,), inst_dict)
+            return self.__class__, (items,)
+
+        def keys(self):
+            return list(self)
+
+        setdefault = DictMixin.setdefault
+        update = DictMixin.update
+        pop = DictMixin.pop
+        values = DictMixin.values
+        items = DictMixin.items
+        iterkeys = DictMixin.iterkeys
+        itervalues = DictMixin.itervalues
+        iteritems = DictMixin.iteritems
+
+        def __repr__(self):
+            if not self:
+                return '%s()' % (self.__class__.__name__,)
+            return '%s(%r)' % (self.__class__.__name__, self.items())
+
+        def copy(self):
+            return self.__class__(self)
+
+        @classmethod
+        def fromkeys(cls, iterable, value=None):
+            d = cls()
+            for key in iterable:
+                d[key] = value
+            return d
+
+        def __eq__(self, other):
+            if isinstance(other, OrderedDict):
+                if len(self) != len(other):
+                    return False
+                for p, q in  zip(self.items(), other.items()):
+                    if p != q:
+                        return False
+                return True
+            return dict.__eq__(self, other)
+
+        def __ne__(self, other):
+            return not self == other
+

pip/basecommand.py

@@ -7,12 +7,15 @@
 import time
 
 from pip import commands
-from pip.log import logger
-from pip.baseparser import parser, ConfigOptionParser, UpdatingDefaultsHelpFormatter
+from pip.backwardcompat import StringIO, walk_packages, u
+from pip.baseparser import (parser, ConfigOptionParser,
+                            UpdatingDefaultsHelpFormatter)
 from pip.download import urlopen
-from pip.exceptions import (BadCommand, InstallationError, UninstallationError,
-                            CommandError)
+from pip.exceptions import (BadCommand, InstallationError,
+                            UninstallationError, CommandError)
 from pip.backwardcompat import StringIO, walk_packages
+from pip.log import logger
+from pip.locations import serverkey_file
 from pip.status_codes import SUCCESS, ERROR, UNKNOWN_ERROR, VIRTUALENV_NOT_FOUND
 
 
@@ -49,22 +52,42 @@ def merge_options(self, initial_options, options):
         for attr in ['log', 'proxy', 'require_venv',
                      'log_explicit_levels', 'log_file',
                      'timeout', 'default_vcs', 'skip_requirements_regex',
-                     'no_input']:
-            setattr(options, attr, getattr(initial_options, attr) or getattr(options, attr))
+                     'no_input', 'refresh_serverkey']:
+            setattr(options, attr,
+                    getattr(initial_options, attr) or getattr(options, attr))
         options.quiet += initial_options.quiet
         options.verbose += initial_options.verbose
 
+    def refresh_serverkey(self, url='https://pypi.python.org/serverkey'):
+        serverkey_cache = open(serverkey_file, 'wb')
+        try:
+            try:
+                content = urlopen(url).read()
+                serverkey_cache.write(content)
+            except Exception:
+                e = sys.exc_info()[1]
+                raise
+                raise InstallationError('Could not refresh local cache (%s) '
+                                        'of PyPI server key (%s): %s' %
+                                        (serverkey_file, url, e))
+            else:
+                logger.notify('Refreshed local cache (%s) of '
+                              'PyPI server key (%s):\n\n%s' %
+                              (serverkey_file, url, u(content)))
+        finally:
+            serverkey_cache.close()
+
     def setup_logging(self):
         pass
 
     def main(self, args, initial_options):
         options, args = self.parser.parse_args(args)
         self.merge_options(initial_options, options)
 
-        level = 1 # Notify
+        level = 1  # Notify
         level += options.verbose
         level -= options.quiet
-        level = logger.level_for_integer(4-level)
+        level = logger.level_for_integer(4 - level)
         complete_log = []
         logger.consumers.extend(
             [(level, sys.stdout),
@@ -77,9 +100,13 @@ def main(self, args, initial_options):
         if options.require_venv:
             # If a venv is required check if it can really be found
             if not os.environ.get('VIRTUAL_ENV'):
-                logger.fatal('Could not find an activated virtualenv (required).')
+                logger.fatal('Could not find an activated '
+                             'virtualenv (required).')
                 sys.exit(VIRTUALENV_NOT_FOUND)
 
+        if not os.path.exists(serverkey_file) or options.refresh_serverkey:
+            self.refresh_serverkey()
+
         if options.log:
             log_fp = open_logfile(options.log, 'a')
             logger.consumers.append((logger.DEBUG, log_fp))
@@ -159,7 +186,7 @@ def open_logfile(filename, mode='a'):
 
     log_fp = open(filename, mode)
     if exists:
-        log_fp.write('%s\n' % ('-'*60))
+        log_fp.write('%s\n' % ('-' * 60))
         log_fp.write('%s run on %s\n' % (sys.argv[0], time.strftime('%c')))
     return log_fp
 
@@ -182,4 +209,3 @@ def load_all_commands():
 def command_names():
     names = set((pkg[1] for pkg in walk_packages(path=commands.__path__)))
     return list(names)
-

pip/baseparser.py

@@ -208,5 +208,11 @@ def get_default_values(self):
     type='str',
     default='',
     help=optparse.SUPPRESS_HELP)
+parser.add_option(
+    '--refresh-serverkey',
+    dest='refresh_serverkey',
+    action='store_true',
+    default=False,
+    help="Refresh the cached version of PyPI's server key")
 
 parser.disable_interspersed_args()

pip/commands/install.py

@@ -1,3 +1,4 @@
+import optparse
 import os
 import sys
 from pip.req import InstallRequirement, RequirementSet
@@ -34,8 +35,8 @@ def __init__(self):
             action='append',
             default=[],
             metavar='FILENAME',
-            help='Install all the packages listed in the given requirements file.  '
-            'This option can be used multiple times.')
+            help='Install all the packages listed in the given requirements '
+                 'file. This option can be used multiple times.')
         self.parser.add_option(
             '-f', '--find-links',
             dest='find_links',
@@ -47,7 +48,7 @@ def __init__(self):
             '-i', '--index-url', '--pypi-url',
             dest='index_url',
             metavar='URL',
-            default='http://pypi.python.org/simple/',
+            default='https://pypi.python.org/simple/',
             help='Base URL of Python Package Index (default %default)')
         self.parser.add_option(
             '--extra-index-url',
@@ -62,19 +63,25 @@ def __init__(self):
             action='store_true',
             default=False,
             help='Ignore package index (only looking at --find-links URLs instead)')
+        self.parser.add_option(
+            '--no-mirrors',
+            dest='use_mirrors',
+            action='store_false',
+            default=False,
+            help='Ignore the PyPI mirrors')
         self.parser.add_option(
             '-M', '--use-mirrors',
             dest='use_mirrors',
             action='store_true',
-            default=False,
-            help='Use the PyPI mirrors as a fallback in case the main index is down.')
+            default=True,
+            help=optparse.SUPPRESS_HELP)
         self.parser.add_option(
             '--mirrors',
             dest='mirrors',
             metavar='URL',
             action='append',
             default=[],
-            help='Specific mirror URLs to query when --use-mirrors is used')
+            help='Specific mirror URLs to use instead of querying the DNS for list of mirrors')
 
         self.parser.add_option(
             '-b', '--build', '--build-dir', '--build-directory',
@@ -199,9 +206,11 @@ def run(self, options, args):
                 InstallRequirement.from_line(name, None))
         for name in options.editables:
             requirement_set.add_requirement(
-                InstallRequirement.from_editable(name, default_vcs=options.default_vcs))
+                InstallRequirement.from_editable(
+                    name, default_vcs=options.default_vcs))
         for filename in options.requirements:
-            for req in parse_requirements(filename, finder=finder, options=options):
+            for req in parse_requirements(
+                    filename, finder=finder, options=options):
                 requirement_set.add_requirement(req)
         if not requirement_set.has_requirements:
             opts = {'name': self.name}

pip/commands/search.py

@@ -22,7 +22,7 @@ def __init__(self):
             '--index',
             dest='index',
             metavar='URL',
-            default='http://pypi.python.org/pypi',
+            default='https://pypi.python.org/pypi',
             help='Base URL of Python Package Index (default %default)')
 
     def run(self, options, args):