Support Dask in highly_variable_genes (#2809)

Co-authored-by: Isaac Virshup <[email protected]>

Author: flying-sheep

docs/release-notes/1.10.0.md

@@ -17,6 +17,7 @@
   now support a `mask` parameter {pr}`2272` {smaller}`C Bright, T Marcella, & P Angerer`
 * {func}`scanpy.tl.rank_genes_groups` no longer warns that it's default was changed from t-test_overestim_var to t-test {pr}`2798` {smaller}`L Heumos`
 * {func}`scanpy.pp.highly_variable_genes` has new flavor `seurat_v3_paper` that is in its implementation consistent with the paper description in Stuart et al 2018. {pr}`2792` {smaller}`E Roellin`
+* {func}`scanpy.pp.highly_variable_genes` supports dask for the default `seurat` and `cell_ranger` flavors {pr}`2809` {smaller}`P Angerer`
 
 ```{rubric} Docs
 ```

pyproject.toml

@@ -120,6 +120,7 @@ doc = [
     "ipython>=7.20",  # for nbsphinx code highlighting
     "matplotlib!=3.6.1",
     # TODO: remove necessity for being able to import doc-linked classes
+    "dask",
     "scanpy[paga]",
     "sam-algorithm",
 ]

scanpy/_compat.py

@@ -30,7 +30,14 @@ class ZappyArray:
         pass
 
 
-__all__ = ["cache", "DaskArray", "fullname", "pkg_metadata", "pkg_version"]
+__all__ = [
+    "cache",
+    "DaskArray",
+    "ZappyArray",
+    "fullname",
+    "pkg_metadata",
+    "pkg_version",
+]
 
 
 def fullname(typ: type) -> str:

scanpy/preprocessing/_deprecated/highly_variable_genes.py

@@ -111,7 +111,8 @@ def filter_genes_dispersion(  # noqa: PLR0917
     if n_top_genes is not None and not all(
         x is None for x in [min_disp, max_disp, min_mean, max_mean]
     ):
-        logg.info("If you pass `n_top_genes`, all cutoffs are ignored.")
+        msg = "If you pass `n_top_genes`, all cutoffs are ignored."
+        warnings.warn(msg, UserWarning)
     if min_disp is None:
         min_disp = 0.5
     if min_mean is None:

scanpy/preprocessing/_distributed.py

@@ -37,7 +37,7 @@ def materialize_as_ndarray(
 def materialize_as_ndarray(
     a: ArrayLike | tuple[ArrayLike | ZappyArray | DaskArray, ...],
 ) -> tuple[np.ndarray] | np.ndarray:
-    """Convert distributed arrays to ndarrays."""
+    """Compute distributed arrays and convert them to numpy ndarrays."""
     if not isinstance(a, tuple):
         return np.asarray(a)