run doctests (#2605)

Author: flying-sheep

docs/conf.py

@@ -179,7 +179,7 @@ def setup(app: Sphinx):
 # -- Suppress link warnings ----------------------------------------------------
 
 qualname_overrides = {
-    "sklearn.neighbors._dist_metrics.DistanceMetric": "sklearn.neighbors.DistanceMetric",
+    "sklearn.neighbors._dist_metrics.DistanceMetric": "sklearn.metrics.DistanceMetric",
     # If the docs are built with an old version of numpy, this will make it work:
     "numpy.random.RandomState": "numpy.random.mtrand.RandomState",
     "scanpy.plotting._matrixplot.MatrixPlot": "scanpy.pl.MatrixPlot",
@@ -192,8 +192,10 @@ def setup(app: Sphinx):
     # Will probably be documented
     ("py:class", "scanpy._settings.Verbosity"),
     ("py:class", "scanpy.neighbors.OnFlySymMatrix"),
-    # Currently undocumented: https://github.com/mwaskom/seaborn/issues/1810
+    # Currently undocumented
+    # https://github.com/mwaskom/seaborn/issues/1810
     ("py:class", "seaborn.ClusterGrid"),
+    ("py:class", "samalg.SAM"),
     # Won’t be documented
     ("py:class", "scanpy.plotting._utils._AxesSubplot"),
     ("py:class", "scanpy._utils.Empty"),

docs/extensions/typed_returns.py

@@ -16,6 +16,10 @@ def process_return(lines):
 
 def scanpy_parse_returns_section(self, section):
     lines_raw = list(process_return(self._dedent(self._consume_to_next_section())))
+    if lines_raw[0] == ":":
+        # Remove the “:” inserted by sphinx-autodoc-typehints
+        # https://github.com/tox-dev/sphinx-autodoc-typehints/blob/a5c091f725da8374347802d54c16c3d38833d41c/src/sphinx_autodoc_typehints/patches.py#L66
+        lines_raw.pop(0)
     lines = self._format_block(":returns: ", lines_raw)
     if lines and lines[-1]:
         lines.append("")

docs/release-notes/1.10.0.md

@@ -13,14 +13,15 @@
 
 ```{rubric} Docs
 ```
+* Fixed a lot of broken usage examples {pr}`2605` {smaller}`P Angerer`
 
 ```{rubric} Bug fixes
 ```
 
 * Updated {func}`~scanpy.read_visium` such that it can read spaceranger 2.0 files {smaller}`L Lehner`
 * Fix {func}`~scanpy.pp.normalize_total` {pr}`2466` {smaller}`P Angerer`
 * Fix testing package build {pr}`2468` {smaller}`P Angerer`
-
+* Fix setting `sc.settings.verbosity` in some cases {pr}`2605` {smaller}`P Angerer`
 
 ```{rubric} Ecosystem
 ```

pyproject.toml

@@ -82,7 +82,7 @@ scanpy = "scanpy.cli:console_main"
 
 [project.optional-dependencies]
 test-min = [
-    "pytest>=4.4",
+    "pytest>=7.4.2",
     "pytest-nunit",
     "pytest-mock",
     "profimp",
@@ -106,7 +106,7 @@ test-full = [
     "scanpy[dask-ml]",
 ]
 doc = [
-    "sphinx>=4.4,<5", # remove upper bound when theme supports dark mode
+    "sphinx>=5",
     "sphinx-book-theme",
     "scanpydoc>=0.9.5",
     "sphinx-autodoc-typehints",
@@ -118,7 +118,9 @@ doc = [
     "nbsphinx",
     "ipython>=7.20",  # for nbsphinx code highlighting
     "matplotlib!=3.6.1",
+    # TODO: remove necessity for being able to import doc-linked classes
     "scanpy[paga]",
+    "sam-algorithm",
 ]
 dev = [
     # getting the dev version
@@ -152,6 +154,7 @@ version-file = "scanpy/_version.py"
 addopts = [
     "--import-mode=importlib",
     "--strict-markers",
+    "--doctest-modules",
     "-pscanpy.testing._pytest",
 ]
 testpaths = ["scanpy"]

scanpy/_compat.py

@@ -1,3 +1,7 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
 from packaging import version
 
 try:
@@ -26,6 +30,25 @@ def fullname(typ: type) -> str:
     return f"{module}.{name}"
 
 
+try:
+    from contextlib import chdir
+except ImportError:  # Python < 3.11
+    import os
+    from contextlib import AbstractContextManager
+
+    @dataclass
+    class chdir(AbstractContextManager):
+        path: Path
+        _old_cwd: list[Path] = field(default_factory=list)
+
+        def __enter__(self) -> None:
+            self._old_cwd.append(Path.cwd())
+            os.chdir(self.path)
+
+        def __exit__(self, *_excinfo) -> None:
+            os.chdir(self._old_cwd.pop())
+
+
 def pkg_metadata(package):
     from importlib.metadata import metadata as m
 

scanpy/_settings.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import inspect
 import sys
 from contextlib import contextmanager
@@ -18,25 +20,42 @@
     "hint": "HINT",
     "debug": "DEBUG",
 }
-# Python 3.7 ensures iteration order
+# Python 3.7+ ensures iteration order
 for v, level in enumerate(list(_VERBOSITY_TO_LOGLEVEL.values())):
     _VERBOSITY_TO_LOGLEVEL[v] = level
 
 
+# Collected from the print_* functions in matplotlib.backends
+_Format = Union[
+    Literal["png", "jpg", "tif", "tiff"],
+    Literal["pdf", "ps", "eps", "svg", "svgz", "pgf"],
+    Literal["raw", "rgba"],
+]
+
+
 class Verbosity(IntEnum):
     error = 0
-    warn = 1
+    warning = 1
     info = 2
     hint = 3
     debug = 4
 
+    def __eq__(self, other: Verbosity | int | str) -> bool:
+        if isinstance(other, Verbosity):
+            return self is other
+        if isinstance(other, int):
+            return self.value == other
+        if isinstance(other, str):
+            return self.name == other
+        return NotImplemented
+
     @property
     def level(self) -> int:
         # getLevelName(str) returns the int level…
-        return getLevelName(_VERBOSITY_TO_LOGLEVEL[self])
+        return getLevelName(_VERBOSITY_TO_LOGLEVEL[self.name])
 
     @contextmanager
-    def override(self, verbosity: "Verbosity") -> ContextManager["Verbosity"]:
+    def override(self, verbosity: Verbosity | str | int) -> ContextManager[Verbosity]:
         """\
         Temporarily override verbosity
         """
@@ -45,6 +64,10 @@ def override(self, verbosity: "Verbosity") -> ContextManager["Verbosity"]:
         settings.verbosity = self
 
 
+# backwards compat
+Verbosity.warn = Verbosity.warning
+
+
 def _type_check(var: Any, varname: str, types: Union[type, Tuple[type, ...]]):
     if isinstance(var, types):
         return
@@ -69,7 +92,7 @@ class ScanpyConfig:
     def __init__(
         self,
         *,
-        verbosity: str = "warning",
+        verbosity: Verbosity | int | str = Verbosity.warning,
         plot_suffix: str = "",
         file_format_data: str = "h5ad",
         file_format_figs: str = "pdf",
@@ -160,7 +183,7 @@ def verbosity(self, verbosity: Union[Verbosity, int, str]):
                 self._verbosity = Verbosity(verbosity_str_options.index(verbosity))
         else:
             _type_check(verbosity, "verbosity", (str, int))
-        _set_log_level(self, _VERBOSITY_TO_LOGLEVEL[self._verbosity])
+        _set_log_level(self, _VERBOSITY_TO_LOGLEVEL[self._verbosity.name])
 
     @property
     def plot_suffix(self) -> str:
@@ -386,15 +409,6 @@ def categories_to_ignore(self, categories_to_ignore: Iterable[str]):
     # Functions
     # --------------------------------------------------------------------------------
 
-    # Collected from the print_* functions in matplotlib.backends
-    # fmt: off
-    _Format = Literal[
-        'png', 'jpg', 'tif', 'tiff',
-        'pdf', 'ps', 'eps', 'svg', 'svgz', 'pgf',
-        'raw', 'rgba',
-    ]
-    # fmt: on
-
     def set_figure_params(
         self,
         scanpy: bool = True,

scanpy/cli.py

@@ -116,7 +116,7 @@ def parse_known_args(
 
 
 def _cmd_settings() -> None:
-    from . import settings
+    from ._settings import settings
 
     print(settings)
 

scanpy/datasets/_datasets.py

@@ -95,10 +95,8 @@ def krumsiek11() -> ad.AnnData:
     Annotated data matrix.
     """
     filename = HERE / "krumsiek11.txt"
-    verbosity_save = settings.verbosity
-    settings.verbosity = "error"  # suppress output...
-    adata = read(filename, first_column_names=True)
-    settings.verbosity = verbosity_save
+    with settings.verbosity.override("error"):  # suppress output...
+        adata = read(filename, first_column_names=True)
     adata.uns["iroot"] = 0
     fate_labels = {0: "Stem", 159: "Mo", 319: "Ery", 459: "Mk", 619: "Neu"}
     adata.uns["highlights"] = fate_labels

scanpy/external/pl.py

@@ -6,6 +6,7 @@
 from anndata import AnnData
 from matplotlib.axes import Axes
 
+from ..testing._doctests import doctest_needs
 from .._utils import _doc_params
 from ..plotting import embedding
 from ..plotting._docs import (
@@ -19,6 +20,7 @@
 from .tl._wishbone import _anndata_to_wishbone
 
 
+@doctest_needs("phate")
 @_wraps_plot_scatter
 @_doc_params(
     adata_color_etc=doc_adata_color_etc,

scanpy/external/pp/_bbknn.py

@@ -1,23 +1,23 @@
-from typing import Union, Optional, Callable
+from __future__ import annotations
 
-from anndata import AnnData
-import sklearn
+from typing import TYPE_CHECKING, Union, Optional, Callable
 
-from ..._utils import lazy_import
+from anndata import AnnData
 
+if TYPE_CHECKING:
+    from sklearn.metrics import DistanceMetric
 
-# Import this lazily so we don’t slowly import sklearn.stats just for annotation
-lazy_import("sklearn.neighbors")
-del lazy_import
+from ...testing._doctests import doctest_needs
 
 
+@doctest_needs("bbknn")
 def bbknn(
     adata: AnnData,
     batch_key: str = "batch",
     use_rep: str = "X_pca",
     approx: bool = True,
     use_annoy: bool = True,
-    metric: Union[str, Callable, "sklearn.neighbors.DistanceMetric"] = "euclidean",
+    metric: Union[str, Callable, DistanceMetric] = "euclidean",
     copy: bool = False,
     *,
     neighbors_within_batch: int = 3,
@@ -55,7 +55,7 @@ def bbknn(
     use_rep
         The dimensionality reduction in `.obsm` to use for neighbour detection. Defaults to PCA.
     approx
-        If `True`, use approximate neighbour finding - annoy or pyNNDescent. This results
+        If `True`, use approximate neighbour finding - annoy or PyNNDescent. This results
         in a quicker run time for large datasets while also potentially increasing the degree of
         batch correction.
     use_annoy
@@ -78,12 +78,13 @@ def bbknn(
         'kantorovich', 'wasserstein', 'tsss', 'true_angular', 'hamming', 'jaccard', 'dice', 'matching', 'kulsinski',
         'rogerstanimoto', 'russellrao', 'sokalsneath', 'sokalmichener', 'yule'])
 
-        KDTree supports members of the `sklearn.neighbors.KDTree.valid_metrics` list, or parameterised
-        `sklearn.neighbors.DistanceMetric` `objects
-        <https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.DistanceMetric.html>`_:
+        KDTree supports members of :class:`sklearn.neighbors.KDTree`’s ``valid_metrics`` list, or parameterised
+        :class:`~sklearn.metrics.DistanceMetric` objects:
 
         >>> sklearn.neighbors.KDTree.valid_metrics
         ['p', 'chebyshev', 'cityblock', 'minkowski', 'infinity', 'l2', 'euclidean', 'manhattan', 'l1']
+
+        .. note:: check the relevant documentation for up-to-date lists.
     copy
         If `True`, return a copy instead of writing to the supplied adata.
     neighbors_within_batch

scanpy/external/pp/_harmony_integrate.py

@@ -2,12 +2,13 @@
 Use harmony to integrate cells from different experiments.
 """
 
-from typing import Optional
-
 from anndata import AnnData
 import numpy as np
 
+from ...testing._doctests import doctest_needs
+
 
+@doctest_needs("harmonypy")
 def harmony_integrate(
     adata: AnnData,
     key: str,

scanpy/external/pp/_hashsolo.py

@@ -1,11 +1,3 @@
-#!/usr/bin/env python
-from scipy.stats import norm
-from scanpy._utils import check_nonnegative_integers
-from itertools import product
-import anndata
-import numpy as np
-import pandas as pd
-
 """
 HashSolo script provides a probabilistic cell hashing demultiplexing method
 which generates a noise distribution and signal distribution for
@@ -23,6 +15,16 @@
 hypotheses in a bayesian fashion, and select the most probable hypothesis.
 """
 
+from itertools import product
+
+import anndata
+import numpy as np
+import pandas as pd
+from scipy.stats import norm
+
+from ..._utils import check_nonnegative_integers
+from ...testing._doctests import doctest_skip
+
 
 def _calculate_log_likelihoods(data, number_of_noise_barcodes):
     """Calculate log likelihoods for each hypothesis, negative, singlet, doublet
@@ -255,6 +257,7 @@ def _calculate_bayes_rule(data, priors, number_of_noise_barcodes):
     }
 
 
+@doctest_skip("Illustrative but not runnable doctest code")
 def hashsolo(
     adata: anndata.AnnData,
     cell_hashing_columns: list,

scanpy/external/pp/_magic.py

@@ -9,11 +9,13 @@
 from ... import logging as logg
 from ..._settings import settings
 from ..._utils import AnyRandom
+from ...testing._doctests import doctest_needs
 
 
 MIN_VERSION = "2.0"
 
 
+@doctest_needs("magic")
 def magic(
     adata: AnnData,
     name_list: Union[Literal["all_genes", "pca_only"], Sequence[str], None] = None,