PEP-458 Implementation (Secure downloads with signed metadata)

docs/html/development/architecture/package-finding.rst

@@ -135,7 +135,7 @@ class's main method is the ``collect_links()`` method. The :ref:`PackageFinder
 <package-finder-class>` class invokes this method as the first step of its
 ``find_all_candidates()`` method.
 
-``LinkCollector`` also has a ``fetch_page()`` method to fetch the HTML from a
+``LinkCollector`` also has a ``fetch_project_page()`` method to fetch the HTML from a
 project page URL. This method is "unintelligent" in that it doesn't parse the
 HTML.
 

pyproject.toml

@@ -44,6 +44,15 @@ drop = [
   "setuptools",
   "pkg_resources/_vendor/",
   "pkg_resources/extern/",
+  # tuf parts that are not needed by updater client
+  "tuf/api/*",
+  "tuf/scripts/*",
+  "tuf/developer_tool.py",
+  "tuf/repository_tool.py",
+  "tuf/repository_lib.py",
+  "tuf/unittest_toolbox.py",
+  # No need for gpg support in securesystemslib
+  "securesystemslib/gpg/*",
 ]
 
 [tool.vendoring.typing-stubs]

setup.py

@@ -68,6 +68,9 @@ def get_version(rel_path):
         exclude=["contrib", "docs", "tests*", "tasks"],
     ),
     package_data={
+        "pip._internal.network": [
+            "secure_update_bootstrap/*/*"
+        ],
         "pip._vendor": ["vendor.txt"],
         "pip._vendor.certifi": ["*.pem"],
         "pip._vendor.requests": ["*.pem"],

src/pip/_internal/cli/req_command.py

@@ -15,7 +15,9 @@
 from pip._internal.exceptions import CommandError, PreviousBuildDirError
 from pip._internal.index.collector import LinkCollector
 from pip._internal.index.package_finder import PackageFinder
+from pip._internal.locations import USER_DATA_DIR
 from pip._internal.models.selection_prefs import SelectionPreferences
+from pip._internal.network.secure_update import SecureUpdateSession
 from pip._internal.network.session import PipSession
 from pip._internal.operations.prepare import RequirementPreparer
 from pip._internal.req.constructors import (
@@ -47,12 +49,15 @@
 class SessionCommandMixin(CommandContextMixIn):
 
     """
-    A class mixin for command classes needing _build_session().
+    A class mixin for command classes needing a PipSession and/or
+    a SecureUpdateSession: these are commands that need to connect to
+    a repository.
     """
     def __init__(self):
         # type: () -> None
         super(SessionCommandMixin, self).__init__()
         self._session = None  # Optional[PipSession]
+        self._secure_update_session = None  # type: Optional[SecureUpdateSession]
 
     @classmethod
     def _get_index_urls(cls, options):
@@ -119,6 +124,21 @@ def _build_session(self, options, retries=None, timeout=None):
 
         return session
 
+    def get_secure_update_session(self, options):
+        # type: (Values) -> SecureUpdateSession
+        """Get a SecureUpdateSession singleton."""
+
+        if self._secure_update_session is None:
+            index_urls = self._get_index_urls(options)
+            cache_dir = options.cache_dir if options.cache_dir else None
+            self._secure_update_session = SecureUpdateSession(
+                index_urls, USER_DATA_DIR, cache_dir
+            )
+            logger.debug("Initialized secure update session (TUF): %s",
+                         str(self._secure_update_session))
+
+        return self._secure_update_session
+
 
 class IndexGroupCommand(Command, SessionCommandMixin):
 
@@ -147,8 +167,9 @@ def handle_pip_version_check(self, options):
             retries=0,
             timeout=min(5, options.timeout)
         )
+        secure_update_session = self.get_secure_update_session(options)
         with session:
-            pip_self_version_check(session, options)
+            pip_self_version_check(session, secure_update_session, options)
 
 
 KEEPABLE_TEMPDIR_TYPES = [
@@ -200,6 +221,7 @@ def make_requirement_preparer(
         options,                  # type: Values
         req_tracker,              # type: RequirementTracker
         session,                  # type: PipSession
+        secure_update_session,    # type: SecureUpdateSession
         finder,                   # type: PackageFinder
         use_user_site,            # type: bool
         download_dir=None,        # type: str
@@ -232,6 +254,7 @@ def make_requirement_preparer(
             req_tracker=req_tracker,
             session=session,
             progress_bar=options.progress_bar,
+            secure_update_session=secure_update_session,
             finder=finder,
             require_hashes=options.require_hashes,
             use_user_site=use_user_site,
@@ -383,6 +406,7 @@ def _build_package_finder(
         self,
         options,               # type: Values
         session,               # type: PipSession
+        secure_update_session,  # type: SecureUpdateSession
         target_python=None,    # type: Optional[TargetPython]
         ignore_requires_python=None,  # type: Optional[bool]
     ):
@@ -393,7 +417,10 @@ def _build_package_finder(
         :param ignore_requires_python: Whether to ignore incompatible
             "Requires-Python" values in links. Defaults to False.
         """
-        link_collector = LinkCollector.create(session, options=options)
+        link_collector = LinkCollector.create(
+            session=session,
+            secure_update_session=secure_update_session,
+            options=options)
         selection_prefs = SelectionPreferences(
             allow_yanked=True,
             format_control=options.format_control,

src/pip/_internal/commands/download.py

@@ -90,11 +90,13 @@ def run(self, options, args):
         ensure_dir(options.download_dir)
 
         session = self.get_default_session(options)
+        secure_update_session = self.get_secure_update_session(options)
 
         target_python = make_target_python(options)
         finder = self._build_package_finder(
             options=options,
             session=session,
+            secure_update_session=secure_update_session,
             target_python=target_python,
         )
         build_delete = (not (options.no_clean or options.build_dir))
@@ -115,6 +117,7 @@ def run(self, options, args):
             options=options,
             req_tracker=req_tracker,
             session=session,
+            secure_update_session=secure_update_session,
             finder=finder,
             download_dir=options.download_dir,
             use_user_site=False,

src/pip/_internal/commands/install.py

@@ -269,11 +269,13 @@ def run(self, options, args):
         global_options = options.global_options or []
 
         session = self.get_default_session(options)
+        secure_update_session = self.get_secure_update_session(options)
 
         target_python = make_target_python(options)
         finder = self._build_package_finder(
             options=options,
             session=session,
+            secure_update_session=secure_update_session,
             target_python=target_python,
             ignore_requires_python=options.ignore_requires_python,
         )
@@ -301,6 +303,7 @@ def run(self, options, args):
                 options=options,
                 req_tracker=req_tracker,
                 session=session,
+                secure_update_session=secure_update_session,
                 finder=finder,
                 use_user_site=options.use_user_site,
             )

src/pip/_internal/commands/list.py

@@ -126,7 +126,12 @@ def _build_package_finder(self, options, session):
         """
         Create a package finder appropriate to this list command.
         """
-        link_collector = LinkCollector.create(session, options=options)
+        secure_update_session = self.get_secure_update_session(options)
+        link_collector = LinkCollector.create(
+            session=session,
+            secure_update_session=secure_update_session,
+            options=options
+        )
 
         # Pass allow_yanked=False to ignore yanked versions.
         selection_prefs = SelectionPreferences(
@@ -225,7 +230,15 @@ def latest_info(dist):
                 dist.latest_filetype = typ
                 return dist
 
-            for dist in map_multithread(latest_info, packages):
+            # NOTE: TUF is not currently threadsafe: do not run multithreaded
+            # if we have any SecureUpdaters
+            secure_update_session = self.get_secure_update_session(options)
+            if secure_update_session.downloaders:
+                map_func = map
+            else:
+                map_func = map_multithread  # type: ignore
+
+            for dist in map_func(latest_info, packages):
                 if dist is not None:
                     yield dist
 

src/pip/_internal/commands/wheel.py

@@ -113,8 +113,9 @@ def run(self, options, args):
         cmdoptions.check_install_build_global(options)
 
         session = self.get_default_session(options)
+        secure_update_session = self.get_secure_update_session(options)
 
-        finder = self._build_package_finder(options, session)
+        finder = self._build_package_finder(options, session, secure_update_session)
         build_delete = (not (options.no_clean or options.build_dir))
         wheel_cache = WheelCache(options.cache_dir, options.format_control)
 
@@ -137,6 +138,7 @@ def run(self, options, args):
             options=options,
             req_tracker=req_tracker,
             session=session,
+            secure_update_session=secure_update_session,
             finder=finder,
             download_dir=options.wheel_dir,
             use_user_site=False,

src/pip/_internal/index/collector.py

@@ -44,6 +44,7 @@
 
     from pip._vendor.requests import Response
 
+    from pip._internal.network.secure_update import SecureUpdateSession
     from pip._internal.network.session import PipSession
 
     HTMLElement = xml.etree.ElementTree.Element
@@ -582,16 +583,18 @@ class LinkCollector(object):
 
     def __init__(
         self,
-        session,       # type: PipSession
-        search_scope,  # type: SearchScope
+        session,                # type: PipSession
+        secure_update_session,  # type: SecureUpdateSession
+        search_scope,           # type: SearchScope
     ):
         # type: (...) -> None
         self.search_scope = search_scope
         self.session = session
+        self.secure_update_session = secure_update_session
 
     @classmethod
-    def create(cls, session, options, suppress_no_index=False):
-        # type: (PipSession, Values, bool) -> LinkCollector
+    def create(cls, session, secure_update_session, options, suppress_no_index=False):
+        # type: (PipSession, SecureUpdateSession, Values, bool) -> LinkCollector
         """
         :param session: The Session to use to make requests.
         :param suppress_no_index: Whether to ignore the --no-index option
@@ -612,7 +615,9 @@ def create(cls, session, options, suppress_no_index=False):
             find_links=find_links, index_urls=index_urls,
         )
         link_collector = LinkCollector(
-            session=session, search_scope=search_scope,
+            session=session,
+            secure_update_session=secure_update_session,
+            search_scope=search_scope,
         )
         return link_collector
 
@@ -621,12 +626,35 @@ def find_links(self):
         # type: () -> List[str]
         return self.search_scope.find_links
 
-    def fetch_page(self, location):
+    def fetch_project_page(self, location):
         # type: (Link) -> Optional[HTMLPage]
         """
         Fetch an HTML page containing package links.
         """
-        return _get_html_page(location, session=self.session)
+        # check if secure update (TUF) should be used: parse url to find the
+        # index url, then see if we have a secure updater for the index url
+        index_url, _, project = location.url.rstrip('/').rpartition('/')
+        if not project:
+            raise ValueError(
+                'Failed to parse {} as project index URL'.format(location.url)
+            )
+
+        downloader = self.secure_update_session.get_downloader(index_url)
+        if downloader:
+            logger.debug('SecureDownloader found: %s', str(downloader))
+            index_file = downloader.download_index(project)
+            if index_file is None:
+                return None
+            else:
+                with open(index_file, "rb") as f:
+                    return HTMLPage(
+                        content=f.read(),
+                        encoding=None,
+                        url=location.url,  # TODO should this be the real URL?
+                        cache_link_parsing=False)
+        else:
+            logger.debug('SecureDownloader not found for %s', index_url)
+            return _get_html_page(location, session=self.session)
 
     def collect_links(self, project_name):
         # type: (str) -> CollectedLinks

src/pip/_internal/index/package_finder.py

@@ -786,7 +786,7 @@ def process_project_url(self, project_url, link_evaluator):
         logger.debug(
             'Fetching project page and analyzing links: %s', project_url,
         )
-        html_page = self._link_collector.fetch_page(project_url)
+        html_page = self._link_collector.fetch_project_page(project_url)
         if html_page is None:
             return []
 

src/pip/_internal/locations.py

@@ -28,6 +28,7 @@
 
 # Application Directories
 USER_CACHE_DIR = appdirs.user_cache_dir("pip")
+USER_DATA_DIR = appdirs.user_data_dir("pip")
 
 
 def get_major_minor_version():