remove dask-expr from CI runs, fix related tests (pydata#9971)

* remove dask-expr from CI runs, fix related tests

* Explicitely add pyarrow to windows builds, as importing dask.dataframe (dask>=2025.1.0) raises ImportError when missing.

* add whats-new.rst entry

Author: kmuehlbauer

ci/requirements/all-but-numba.yml

@@ -12,7 +12,6 @@ dependencies:
   - cartopy
   - cftime
   - dask-core
-  - dask-expr # dask raises a deprecation warning without this, breaking doctests
   - distributed
   - flox
   - fsspec

ci/requirements/doc.yml

@@ -10,7 +10,6 @@ dependencies:
   - cfgrib
   - kerchunk
   - dask-core>=2022.1
-  - dask-expr
   - hypothesis>=6.75.8
   - h5netcdf>=0.13
   - ipykernel

ci/requirements/environment-3.13.yml

@@ -10,7 +10,6 @@ dependencies:
   - cartopy
   - cftime
   - dask-core
-  - dask-expr
   - distributed
   - flox
   - fsspec

ci/requirements/environment-windows-3.13.yml

@@ -8,7 +8,6 @@ dependencies:
   - cartopy
   - cftime
   - dask-core
-  - dask-expr
   - distributed
   - flox
   - fsspec
@@ -29,6 +28,7 @@ dependencies:
   # - pint>=0.22
   - pip
   - pre-commit
+  - pyarrow # importing dask.dataframe raises an ImportError without this
   - pydap
   - pytest
   - pytest-cov

ci/requirements/environment-windows.yml

@@ -8,7 +8,6 @@ dependencies:
   - cartopy
   - cftime
   - dask-core
-  - dask-expr
   - distributed
   - flox
   - fsspec
@@ -29,6 +28,7 @@ dependencies:
   # - pint>=0.22
   - pip
   - pre-commit
+  - pyarrow # importing dask.dataframe raises an ImportError without this
   - pydap
   - pytest
   - pytest-cov

ci/requirements/environment.yml

@@ -10,7 +10,6 @@ dependencies:
   - cartopy
   - cftime
   - dask-core
-  - dask-expr # dask raises a deprecation warning without this, breaking doctests
   - distributed
   - flox
   - fsspec

doc/whats-new.rst

@@ -67,7 +67,8 @@ Bug fixes
 ~~~~~~~~~
 - Fix issues related to Pandas v3 ("us" vs. "ns" for python datetime, copy on write) and handling of 0d-numpy arrays in datetime/timedelta decoding (:pull:`9953`).
   By `Kai Mühlbauer <https://github.com/kmuehlbauer>`_.
-
+- Remove dask-expr from CI runs, add "pyarrow" dask dependency to windows CI runs, fix related tests (:issue:`9962`, :pull:`9971`).
+  By `Kai Mühlbauer <https://github.com/kmuehlbauer>`_.
 
 Documentation
 ~~~~~~~~~~~~~

xarray/tests/__init__.py

@@ -111,13 +111,18 @@ def _importorskip(
     "dask", minversion="2024.08.1"
 )
 has_dask_ge_2024_11_0, requires_dask_ge_2024_11_0 = _importorskip("dask", "2024.11.0")
-with warnings.catch_warnings():
-    warnings.filterwarnings(
-        "ignore",
-        message="The current Dask DataFrame implementation is deprecated.",
-        category=DeprecationWarning,
-    )
-    has_dask_expr, requires_dask_expr = _importorskip("dask_expr")
+has_dask_ge_2025_1_0, requires_dask_ge_2025_1_0 = _importorskip("dask", "2025.1.0")
+if has_dask_ge_2025_1_0:
+    has_dask_expr = True
+    requires_dask_expr = pytest.mark.skipif(not has_dask_expr, reason="should not skip")
+else:
+    with warnings.catch_warnings():
+        warnings.filterwarnings(
+            "ignore",
+            message="The current Dask DataFrame implementation is deprecated.",
+            category=DeprecationWarning,
+        )
+        has_dask_expr, requires_dask_expr = _importorskip("dask_expr")
 has_bottleneck, requires_bottleneck = _importorskip("bottleneck")
 has_rasterio, requires_rasterio = _importorskip("rasterio")
 has_zarr, requires_zarr = _importorskip("zarr")

xarray/tests/test_dataarray.py

@@ -48,6 +48,7 @@
     assert_identical,
     assert_no_warnings,
     has_dask,
+    has_dask_ge_2025_1_0,
     raise_if_dask_computes,
     requires_bottleneck,
     requires_cupy,
@@ -3437,7 +3438,7 @@ def test_to_dataframe_0length(self) -> None:
 
     @requires_dask_expr
     @requires_dask
-    @pytest.mark.xfail(reason="dask-expr is broken")
+    @pytest.mark.xfail(not has_dask_ge_2025_1_0, reason="dask-expr is broken")
     def test_to_dask_dataframe(self) -> None:
         arr_np = np.arange(3 * 4).reshape(3, 4)
         arr = DataArray(arr_np, [("B", [1, 2, 3]), ("A", list("cdef"))], name="foo")