Add support for "yanked" files (PEP 592)

news/6633.feature

@@ -0,0 +1,2 @@
+Respect whether a file has been marked as "yanked" from a simple repository
+(see `PEP 592 <https://www.python.org/dev/peps/pep-0592/>`__ for details).

src/pip/_internal/commands/list.py

@@ -183,7 +183,10 @@ def iter_packages_latest_infos(self, packages, options):
                                       if not candidate.version.is_prerelease]
 
                 evaluator = finder.candidate_evaluator
-                best_candidate = evaluator.get_best_candidate(all_candidates)
+                # Pass allow_yanked=False to ignore yanked versions.
+                best_candidate = evaluator.get_best_candidate(
+                    all_candidates, allow_yanked=False,
+                )
                 if best_candidate is None:
                     continue
 

src/pip/_internal/index.py

@@ -43,15 +43,19 @@
         Any, Callable, Iterable, Iterator, List, MutableMapping, Optional,
         Sequence, Set, Tuple, Union,
     )
+    import xml.etree.ElementTree
     from pip._vendor.packaging.version import _BaseVersion
     from pip._vendor.requests import Response
     from pip._internal.models.search_scope import SearchScope
     from pip._internal.req import InstallRequirement
     from pip._internal.download import PipSession
 
-    SecureOrigin = Tuple[str, str, Optional[str]]
     BuildTag = Tuple[Any, ...]  # either empty tuple or Tuple[int, str]
-    CandidateSortingKey = Tuple[int, _BaseVersion, BuildTag, Optional[int]]
+    CandidateSortingKey = (
+        Tuple[int, int, _BaseVersion, BuildTag, Optional[int]]
+    )
+    HTMLElement = xml.etree.ElementTree.Element
+    SecureOrigin = Tuple[str, str, Optional[str]]
 
 
 __all__ = ['FormatControl', 'FoundCandidates', 'PackageFinder']
@@ -454,14 +458,24 @@ def make_found_candidates(
     def _sort_key(self, candidate):
         # type: (InstallationCandidate) -> CandidateSortingKey
         """
-        Function used to generate link sort key for link tuples.
-        The greater the return value, the more preferred it is.
-        If not finding wheels, then sorted by version only.
+        Function to pass as the `key` argument to a call to sorted() to sort
+        InstallationCandidates by preference.
+
+        Returns a tuple such that tuples sorting as greater using Python's
+        default comparison operator are more preferred.
+
+        The preference is as follows:
+
+        First and foremost, yanked candidates (in the sense of PEP 592) are
+        always less preferred than candidates that haven't been yanked. Then:
+
+        If not finding wheels, they are sorted by version only.
         If finding wheels, then the sort order is by version, then:
           1. existing installs
           2. wheels ordered via Wheel.support_index_min(self._valid_tags)
           3. source archives
         If prefer_binary was set, then all wheels are sorted above sources.
+
         Note: it was considered to embed this logic into the Link
               comparison operators, but then different sdist links
               with the same version, would have to be considered equal
@@ -470,9 +484,10 @@ def _sort_key(self, candidate):
         support_num = len(valid_tags)
         build_tag = tuple()  # type: BuildTag
         binary_preference = 0
-        if candidate.location.is_wheel:
+        link = candidate.location
+        if link.is_wheel:
             # can raise InvalidWheelFilename
-            wheel = Wheel(candidate.location.filename)
+            wheel = Wheel(link.filename)
             if not self._is_wheel_supported(wheel):
                 raise UnsupportedWheel(
                     "%s is not a supported wheel for this platform. It "
@@ -487,18 +502,52 @@ def _sort_key(self, candidate):
                 build_tag = (int(build_tag_groups[0]), build_tag_groups[1])
         else:  # sdist
             pri = -(support_num)
-        return (binary_preference, candidate.version, build_tag, pri)
+        yank_value = -1 * int(link.is_yanked)  # -1 for yanked.
+        return (
+            yank_value, binary_preference, candidate.version, build_tag, pri,
+        )
 
-    def get_best_candidate(self, candidates):
-        # type: (List[InstallationCandidate]) -> InstallationCandidate
+    # Don't include an allow_yanked default value to make sure each call
+    # site considers whether yanked releases are allowed. This also causes
+    # that decision to be made explicit in the calling code, which helps
+    # people when reading the code.
+    def get_best_candidate(
+        self,
+        candidates,    # type: List[InstallationCandidate]
+        allow_yanked,  # type: bool
+    ):
+        # type: (...) -> Optional[InstallationCandidate]
         """
         Return the best candidate per the instance's sort order, or None if
-        no candidates are given.
+        no candidate is acceptable.
+
+        :param allow_yanked: Whether to permit returning a yanked candidate
+            in the sense of PEP 592. If true, a yanked candidate will be
+            returned only if all candidates have been yanked.
         """
         if not candidates:
             return None
 
-        return max(candidates, key=self._sort_key)
+        best_candidate = max(candidates, key=self._sort_key)
+
+        # Log a warning per PEP 592 if necessary before returning.
+        link = best_candidate.location
+        if not link.is_yanked:
+            return best_candidate
+
+        # Otherwise, all the candidates were yanked.
+        if not allow_yanked:
+            return None
+
+        reason = link.yanked_reason or '<none given>'
+        msg = (
+            'The candidate selected for download or install is a '
+            'yanked version: {candidate}\n'
+            'Reason for being yanked: {reason}'
+        ).format(candidate=best_candidate, reason=reason)
+        logger.warning(msg)
+
+        return best_candidate
 
 
 class FoundCandidates(object):
@@ -540,13 +589,23 @@ def iter_applicable(self):
         # Again, converting version to str to deal with debundling.
         return (c for c in self.iter_all() if str(c.version) in self._versions)
 
-    def get_best(self):
-        # type: () -> Optional[InstallationCandidate]
+    # Don't include an allow_yanked default value to make sure each call
+    # site considers whether yanked releases are allowed. This also causes
+    # that decision to be made explicit in the calling code, which helps
+    # people when reading the code.
+    def get_best(self, allow_yanked):
+        # type: (bool) -> Optional[InstallationCandidate]
         """Return the best candidate available, or None if no applicable
         candidates are found.
+
+        :param allow_yanked: Whether to permit returning a yanked candidate
+            in the sense of PEP 592. If true, a yanked candidate will be
+            returned only if all candidates have been yanked.
         """
         candidates = list(self.iter_applicable())
-        return self._evaluator.get_best_candidate(candidates)
+        return self._evaluator.get_best_candidate(
+            candidates, allow_yanked=allow_yanked,
+        )
 
 
 class PackageFinder(object):
@@ -910,7 +969,7 @@ def find_requirement(self, req, upgrade):
         Raises DistributionNotFound or BestVersionAlreadyInstalled otherwise
         """
         candidates = self.find_candidates(req.name, req.specifier)
-        best_candidate = candidates.get_best()
+        best_candidate = candidates.get_best(allow_yanked=True)
 
         installed_version = None    # type: Optional[_BaseVersion]
         if req.satisfied_by is not None:
@@ -1151,6 +1210,37 @@ def _clean_link(url):
     return urllib_parse.urlunparse(result._replace(path=path))
 
 
+def _create_link_from_element(
+    anchor,    # type: HTMLElement
+    page_url,  # type: str
+    base_url,  # type: str
+):
+    # type: (...) -> Optional[Link]
+    """
+    Convert an anchor element in a simple repository page to a Link.
+    """
+    href = anchor.get("href")
+    if not href:
+        return None
+
+    url = _clean_link(urllib_parse.urljoin(base_url, href))
+    pyrequire = anchor.get('data-requires-python')
+    pyrequire = unescape(pyrequire) if pyrequire else None
+
+    yanked_reason = anchor.get('data-yanked')
+    if yanked_reason:
+        yanked_reason = unescape(yanked_reason)
+
+    link = Link(
+        url,
+        comes_from=page_url,
+        requires_python=pyrequire,
+        yanked_reason=yanked_reason,
+    )
+
+    return link
+
+
 class HTMLPage(object):
     """Represents one page, along with its URL"""
 
@@ -1173,12 +1263,14 @@ def iter_links(self):
         )
         base_url = _determine_base_url(document, self.url)
         for anchor in document.findall(".//a"):
-            if anchor.get("href"):
-                href = anchor.get("href")
-                url = _clean_link(urllib_parse.urljoin(base_url, href))
-                pyrequire = anchor.get('data-requires-python')
-                pyrequire = unescape(pyrequire) if pyrequire else None
-                yield Link(url, self.url, requires_python=pyrequire)
+            link = _create_link_from_element(
+                anchor,
+                page_url=self.url,
+                base_url=base_url,
+            )
+            if link is None:
+                continue
+            yield link
 
 
 Search = namedtuple('Search', 'supplied canonical formats')

src/pip/_internal/models/candidate.py

@@ -29,3 +29,8 @@ def __repr__(self):
         return "<InstallationCandidate({!r}, {!r}, {!r})>".format(
             self.project, self.version, self.location,
         )
+
+    def __str__(self):
+        return '{!r} candidate (version {} at {})'.format(
+            self.project, self.version, self.location,
+        )

src/pip/_internal/models/link.py

@@ -19,17 +19,28 @@ class Link(KeyBasedCompareMixin):
     """Represents a parsed link from a Package Index's simple URL
     """
 
-    def __init__(self, url, comes_from=None, requires_python=None):
-        # type: (str, Optional[Union[str, HTMLPage]], Optional[str]) -> None
+    def __init__(
+        self,
+        url,                   # type: str
+        comes_from=None,       # type: Optional[Union[str, HTMLPage]]
+        requires_python=None,  # type: Optional[str]
+        yanked_reason=None,    # type: Optional[str]
+    ):
+        # type: (...) -> None
         """
-        url:
-            url of the resource pointed to (href of the link)
-        comes_from:
-            instance of HTMLPage where the link was found, or string.
-        requires_python:
-            String containing the `Requires-Python` metadata field, specified
-            in PEP 345. This may be specified by a data-requires-python
-            attribute in the HTML link tag, as described in PEP 503.
+        :param url: url of the resource pointed to (href of the link)
+        :param comes_from: instance of HTMLPage where the link was found,
+            or string.
+        :param requires_python: String containing the `Requires-Python`
+            metadata field, specified in PEP 345. This may be specified by
+            a data-requires-python attribute in the HTML link tag, as
+            described in PEP 503.
+        :param yanked_reason: the reason the file has been yanked, if the
+            file has been yanked, or None if the file hasn't been yanked.
+            This is the value of the "data-yanked" attribute, if present, in
+            a simple repository HTML link. If the file has been yanked but
+            no reason was provided, this should be the empty string. See
+            PEP 592 for more information and the specification.
         """
 
         # url can be a UNC windows share
@@ -43,6 +54,7 @@ def __init__(self, url, comes_from=None, requires_python=None):
 
         self.comes_from = comes_from
         self.requires_python = requires_python if requires_python else None
+        self.yanked_reason = yanked_reason
 
         super(Link, self).__init__(key=url, defining_class=Link)
 
@@ -176,3 +188,8 @@ def is_artifact(self):
             return False
 
         return True
+
+    @property
+    def is_yanked(self):
+        # type: () -> bool
+        return self.yanked_reason is not None

src/pip/_internal/utils/outdated.py

@@ -131,7 +131,11 @@ def pip_version_check(session, options):
                 trusted_hosts=options.trusted_hosts,
                 session=session,
             )
-            candidate = finder.find_candidates("pip").get_best()
+            # Pass allow_yanked=False so we don't suggest upgrading to a
+            # yanked version.
+            candidate = finder.find_candidates("pip").get_best(
+                allow_yanked=False,
+            )
             if candidate is None:
                 return
             pypi_version = str(candidate.version)