Merge pull request #3085 from patricklaw/add-pip-download-command

Add `pip download` command and deprecate `pip install --download`.

Author: xavfernandez

CHANGES.txt

@@ -23,6 +23,9 @@
 * Allow repository URLs with secure transports to count as trusted. (E.g.,
   "git+ssh" is okay.) :issue:`2811`.
 
+* Implement a top-level ``pip download`` command and deprecate
+  ``pip install --download``.
+
 
 **7.1.2 (2015-08-22)**
 

docs/reference/index.rst

@@ -7,6 +7,7 @@ Reference Guide
 
    pip
    pip_install
+   pip_download
    pip_uninstall
    pip_freeze
    pip_list

docs/reference/pip_download.rst

@@ -0,0 +1,53 @@
+
+.. _`pip download`:
+
+pip download
+------------
+
+.. contents::
+
+Usage
+*****
+
+.. pip-command-usage:: download
+
+
+Description
+***********
+
+.. pip-command-description:: download
+
+
+Overview
+++++++++
+``pip download`` replaces the ``--download`` option to ``pip install``,
+which is now deprecated and will be removed in pip 10.
+
+``pip download`` does the same resolution and downloading as ``pip install``,
+but instead of installing the dependencies, it collects the downloaded
+distributions into the directory provided (defaulting to ``./pip_downloads``).
+This directory can later be passed as the value to
+``pip install --find-links`` to facilitate offline or locked down package
+installation.
+
+
+Options
+*******
+
+.. pip-command-options:: download
+
+.. pip-index-options::
+
+
+Examples
+********
+
+1. Download a package and all of its dependencies
+
+  ::
+
+    $ pip download -d ./pip_downloads SomePackage
+    $ pip download SomePackage  # equivalent to above
+    $ pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage
+
+

pip/basecommand.py

@@ -7,10 +7,12 @@
 import optparse
 
 from pip import cmdoptions
+from pip.index import PackageFinder
 from pip.locations import running_under_virtualenv
 from pip.download import PipSession
 from pip.exceptions import (BadCommand, InstallationError, UninstallationError,
                             CommandError, PreviousBuildDirError)
+
 from pip.compat import logging_dictConfig
 from pip.baseparser import ConfigOptionParser, UpdatingDefaultsHelpFormatter
 from pip.req import InstallRequirement, parse_requirements
@@ -290,3 +292,22 @@ def populate_requirement_set(requirement_set, args, options, finder,
                 msg = ('You must give at least one requirement '
                        'to %(name)s (see "pip help %(name)s")' % opts)
             logger.warning(msg)
+
+    def _build_package_finder(self, options, session):
+        """
+        Create a package finder appropriate to this requirement command.
+        """
+        index_urls = [options.index_url] + options.extra_index_urls
+        if options.no_index:
+            logger.info('Ignoring indexes: %s', ','.join(index_urls))
+            index_urls = []
+
+        return PackageFinder(
+            find_links=options.find_links,
+            format_control=options.format_control,
+            index_urls=index_urls,
+            trusted_hosts=options.trusted_hosts,
+            allow_all_prereleases=options.pre,
+            process_dependency_links=options.process_dependency_links,
+            session=session,
+        )

pip/cmdoptions.py

@@ -553,18 +553,24 @@ def only_binary():
     ]
 }
 
-index_group = {
+non_deprecated_index_group = {
     'name': 'Package Index Options',
     'options': [
         index_url,
         extra_index_url,
         no_index,
         find_links,
+        process_dependency_links,
+    ]
+}
+
+index_group = {
+    'name': 'Package Index Options (including deprecated options)',
+    'options': non_deprecated_index_group['options'] + [
         allow_external,
         allow_all_external,
         no_allow_external,
         allow_unsafe,
         no_allow_unsafe,
-        process_dependency_links,
     ]
 }

pip/commands/__init__.py

@@ -4,6 +4,7 @@
 from __future__ import absolute_import
 
 from pip.commands.completion import CompletionCommand
+from pip.commands.download import DownloadCommand
 from pip.commands.freeze import FreezeCommand
 from pip.commands.help import HelpCommand
 from pip.commands.list import ListCommand
@@ -22,13 +23,15 @@
     ShowCommand.name: ShowCommand,
     InstallCommand.name: InstallCommand,
     UninstallCommand.name: UninstallCommand,
+    DownloadCommand.name: DownloadCommand,
     ListCommand.name: ListCommand,
     WheelCommand.name: WheelCommand,
 }
 
 
 commands_order = [
     InstallCommand,
+    DownloadCommand,
     UninstallCommand,
     FreezeCommand,
     ListCommand,

pip/commands/download.py

@@ -0,0 +1,135 @@
+from __future__ import absolute_import
+
+import logging
+import os
+
+from pip.req import RequirementSet
+from pip.basecommand import RequirementCommand
+from pip import cmdoptions
+from pip.utils import ensure_dir, normalize_path
+from pip.utils.build import BuildDirectory
+from pip.utils.filesystem import check_path_owner
+
+
+DEFAULT_DOWNLOAD_DIR = os.path.join(normalize_path(os.curdir), 'pip_downloads')
+
+
+logger = logging.getLogger(__name__)
+
+
+class DownloadCommand(RequirementCommand):
+    """
+    Download packages from:
+
+    - PyPI (and other indexes) using requirement specifiers.
+    - VCS project urls.
+    - Local project directories.
+    - Local or remote source archives.
+
+    pip also supports downloading from "requirements files", which provide
+    an easy way to specify a whole environment to be downloaded.
+    """
+    name = 'download'
+
+    usage = """
+      %prog [options] <requirement specifier> [package-index-options] ...
+      %prog [options] -r <requirements file> [package-index-options] ...
+      %prog [options] [-e] <vcs project url> ...
+      %prog [options] [-e] <local project path> ...
+      %prog [options] <archive url/path> ..."""
+
+    summary = 'Download packages.'
+
+    def __init__(self, *args, **kw):
+        super(DownloadCommand, self).__init__(*args, **kw)
+
+        cmd_opts = self.cmd_opts
+
+        cmd_opts.add_option(cmdoptions.constraints())
+        cmd_opts.add_option(cmdoptions.editable())
+        cmd_opts.add_option(cmdoptions.requirements())
+        cmd_opts.add_option(cmdoptions.build_dir())
+        cmd_opts.add_option(cmdoptions.no_deps())
+        cmd_opts.add_option(cmdoptions.global_options())
+        cmd_opts.add_option(cmdoptions.no_binary())
+        cmd_opts.add_option(cmdoptions.only_binary())
+        cmd_opts.add_option(cmdoptions.src())
+        cmd_opts.add_option(cmdoptions.no_clean())
+        cmd_opts.add_option(cmdoptions.pre())
+
+        cmd_opts.add_option(
+            '-d', '--dest', '--destination-dir', '--destination-directory',
+            dest='download_dir',
+            metavar='dir',
+            default=DEFAULT_DOWNLOAD_DIR,
+            help=("Download packages into <dir>."),
+        )
+
+        index_opts = cmdoptions.make_option_group(
+            cmdoptions.non_deprecated_index_group,
+            self.parser,
+        )
+
+        self.parser.insert_option_group(0, index_opts)
+        self.parser.insert_option_group(0, cmd_opts)
+
+    def run(self, options, args):
+        options.ignore_installed = True
+        options.src_dir = os.path.abspath(options.src_dir)
+        ensure_dir(options.download_dir)
+
+        with self._build_session(options) as session:
+
+            finder = self._build_package_finder(options, session)
+            build_delete = (not (options.no_clean or options.build_dir))
+            if options.cache_dir and not check_path_owner(options.cache_dir):
+                logger.warning(
+                    "The directory '%s' or its parent directory is not owned "
+                    "by the current user and caching wheels has been "
+                    "disabled. check the permissions and owner of that "
+                    "directory. If executing pip with sudo, you may want "
+                    "sudo's -H flag.",
+                    options.cache_dir,
+                )
+                options.cache_dir = None
+
+            with BuildDirectory(options.build_dir,
+                                delete=build_delete) as build_dir:
+
+                requirement_set = RequirementSet(
+                    build_dir=build_dir,
+                    src_dir=options.src_dir,
+                    download_dir=options.download_dir,
+                    ignore_installed=True,
+                    ignore_dependencies=options.ignore_dependencies,
+                    session=session,
+                    isolated=options.isolated_mode,
+                )
+                self.populate_requirement_set(
+                    requirement_set,
+                    args,
+                    options,
+                    finder,
+                    session,
+                    self.name,
+                    None
+                )
+
+                if not requirement_set.has_requirements:
+                    return
+
+                requirement_set.prepare_files(finder)
+
+                downloaded = ' '.join([
+                    req.name for req in requirement_set.successfully_downloaded
+                ])
+                if downloaded:
+                    logger.info(
+                        'Successfully downloaded %s', downloaded
+                    )
+
+                # Clean up
+                if not options.no_clean:
+                    requirement_set.cleanup_files()
+
+        return requirement_set

pip/commands/install.py

@@ -14,7 +14,6 @@
 from pip.req import RequirementSet
 from pip.basecommand import RequirementCommand
 from pip.locations import virtualenv_no_global, distutils_scheme
-from pip.index import PackageFinder
 from pip.exceptions import (
     InstallationError, CommandError, PreviousBuildDirError,
 )
@@ -168,22 +167,6 @@ def __init__(self, *args, **kw):
         self.parser.insert_option_group(0, index_opts)
         self.parser.insert_option_group(0, cmd_opts)
 
-    def _build_package_finder(self, options, index_urls, session):
-        """
-        Create a package finder appropriate to this install command.
-        This method is meant to be overridden by subclasses, not
-        called directly.
-        """
-        return PackageFinder(
-            find_links=options.find_links,
-            format_control=options.format_control,
-            index_urls=index_urls,
-            trusted_hosts=options.trusted_hosts,
-            allow_all_prereleases=options.pre,
-            process_dependency_links=options.process_dependency_links,
-            session=session,
-        )
-
     def run(self, options, args):
         cmdoptions.resolve_wheel_no_use_binary(options)
         cmdoptions.check_install_build_global(options)
@@ -213,6 +196,12 @@ def run(self, options, args):
             )
 
         if options.download_dir:
+            warnings.warn(
+                "pip install --download has been deprecated and will be "
+                "removed in the future. Pip now has a download command that "
+                "should be used instead.",
+                RemovedInPip10Warning,
+            )
             options.ignore_installed = True
 
         if options.build_dir:
@@ -243,14 +232,10 @@ def run(self, options, args):
             install_options.append('--home=' + temp_target_dir)
 
         global_options = options.global_options or []
-        index_urls = [options.index_url] + options.extra_index_urls
-        if options.no_index:
-            logger.info('Ignoring indexes: %s', ','.join(index_urls))
-            index_urls = []
 
         with self._build_session(options) as session:
 
-            finder = self._build_package_finder(options, index_urls, session)
+            finder = self._build_package_finder(options, session)
             build_delete = (not (options.no_clean or options.build_dir))
             wheel_cache = WheelCache(options.cache_dir, options.format_control)
             if options.cache_dir and not check_path_owner(options.cache_dir):

pip/commands/wheel.py

@@ -6,7 +6,6 @@
 import warnings
 
 from pip.basecommand import RequirementCommand
-from pip.index import PackageFinder
 from pip.exceptions import CommandError, PreviousBuildDirError
 from pip.req import RequirementSet
 from pip.utils import import_or_raise, normalize_path
@@ -161,16 +160,7 @@ def run(self, options, args):
 
         with self._build_session(options) as session:
 
-            finder = PackageFinder(
-                find_links=options.find_links,
-                format_control=options.format_control,
-                index_urls=index_urls,
-                allow_all_prereleases=options.pre,
-                trusted_hosts=options.trusted_hosts,
-                process_dependency_links=options.process_dependency_links,
-                session=session,
-            )
-
+            finder = self._build_package_finder(options, session)
             build_delete = (not (options.no_clean or options.build_dir))
             wheel_cache = WheelCache(options.cache_dir, options.format_control)
             with BuildDirectory(options.build_dir,