Use to_numpy in time decoding (#10081)

* Use to_numpy in time decoding

* WIP

* Revert "WIP"

This reverts commit 54be9b1.

* fix

* Update xarray/namedarray/pycompat.py

Author: dcherian

xarray/coding/times.py

@@ -28,7 +28,7 @@
 from xarray.core.utils import attempt_import, emit_user_level_warning
 from xarray.core.variable import Variable
 from xarray.namedarray.parallelcompat import T_ChunkedArray, get_chunked_array_type
-from xarray.namedarray.pycompat import is_chunked_array
+from xarray.namedarray.pycompat import is_chunked_array, to_numpy
 from xarray.namedarray.utils import is_duck_dask_array
 
 try:
@@ -310,7 +310,7 @@ def _decode_cf_datetime_dtype(
     # Dataset.__repr__ when users try to view their lazily decoded array.
     values = indexing.ImplicitToExplicitIndexingAdapter(indexing.as_indexable(data))
     example_value = np.concatenate(
-        [first_n_items(values, 1) or [0], last_item(values) or [0]]
+        [to_numpy(first_n_items(values, 1)), to_numpy(last_item(values))]
     )
 
     try:
@@ -516,7 +516,7 @@ def decode_cf_datetime(
     --------
     cftime.num2date
     """
-    num_dates = np.asarray(num_dates)
+    num_dates = to_numpy(num_dates)
     flat_num_dates = ravel(num_dates)
     if calendar is None:
         calendar = "standard"
@@ -643,7 +643,7 @@ def decode_cf_timedelta(
     """Given an array of numeric timedeltas in netCDF format, convert it into a
     numpy timedelta64 ["s", "ms", "us", "ns"] array.
     """
-    num_timedeltas = np.asarray(num_timedeltas)
+    num_timedeltas = to_numpy(num_timedeltas)
     unit = _netcdf_to_numpy_timeunit(units)
 
     with warnings.catch_warnings():

xarray/core/formatting.py

@@ -18,12 +18,12 @@
 from pandas.errors import OutOfBoundsDatetime
 
 from xarray.core.datatree_render import RenderDataTree
-from xarray.core.duck_array_ops import array_all, array_any, array_equiv, astype
+from xarray.core.duck_array_ops import array_all, array_any, array_equiv, astype, ravel
 from xarray.core.indexing import MemoryCachedArray
 from xarray.core.options import OPTIONS, _get_boolean_with_default
 from xarray.core.treenode import group_subtrees
 from xarray.core.utils import is_duck_array
-from xarray.namedarray.pycompat import array_type, to_duck_array, to_numpy
+from xarray.namedarray.pycompat import array_type, to_duck_array
 
 if TYPE_CHECKING:
     from xarray.core.coordinates import AbstractCoordinates
@@ -94,7 +94,7 @@ def first_n_items(array, n_desired):
     # pass Variable._data
     if isinstance(array, Variable):
         array = array._data
-    return np.ravel(to_duck_array(array))[:n_desired]
+    return ravel(to_duck_array(array))[:n_desired]
 
 
 def last_n_items(array, n_desired):
@@ -118,18 +118,13 @@ def last_n_items(array, n_desired):
     # pass Variable._data
     if isinstance(array, Variable):
         array = array._data
-    return np.ravel(to_duck_array(array))[-n_desired:]
+    return ravel(to_duck_array(array))[-n_desired:]
 
 
 def last_item(array):
-    """Returns the last item of an array in a list or an empty list."""
-    if array.size == 0:
-        # work around for https://github.com/numpy/numpy/issues/5195
-        return []
-
+    """Returns the last item of an array."""
     indexer = (slice(-1, None),) * array.ndim
-    # to_numpy since dask doesn't support tolist
-    return np.ravel(to_numpy(array[indexer])).tolist()
+    return ravel(to_duck_array(array[indexer]))
 
 
 def calc_max_rows_first(max_rows: int) -> int:

xarray/namedarray/pycompat.py

@@ -102,6 +102,12 @@ def to_numpy(
     from xarray.core.indexing import ExplicitlyIndexed
     from xarray.namedarray.parallelcompat import get_chunked_array_type
 
+    try:
+        # for tests only at the moment
+        return data.to_numpy()  # type: ignore[no-any-return,union-attr]
+    except AttributeError:
+        pass
+
     if isinstance(data, ExplicitlyIndexed):
         data = data.get_duck_array()  # type: ignore[no-untyped-call]
 
@@ -122,15 +128,18 @@ def to_numpy(
 
 
 def to_duck_array(data: Any, **kwargs: dict[str, Any]) -> duckarray[_ShapeType, _DType]:
-    from xarray.core.indexing import ExplicitlyIndexed
+    from xarray.core.indexing import (
+        ExplicitlyIndexed,
+        ImplicitToExplicitIndexingAdapter,
+    )
     from xarray.namedarray.parallelcompat import get_chunked_array_type
 
     if is_chunked_array(data):
         chunkmanager = get_chunked_array_type(data)
         loaded_data, *_ = chunkmanager.compute(data, **kwargs)  # type: ignore[var-annotated]
         return loaded_data
 
-    if isinstance(data, ExplicitlyIndexed):
+    if isinstance(data, ExplicitlyIndexed | ImplicitToExplicitIndexingAdapter):
         return data.get_duck_array()  # type: ignore[no-untyped-call, no-any-return]
     elif is_duck_array(data):
         return data

xarray/tests/arrays.py

@@ -51,13 +51,20 @@ def __init__(self, array: np.ndarray):
     def __getitem__(self, key):
         return type(self)(self.array[key])
 
+    def to_numpy(self) -> np.ndarray:
+        """Allow explicit conversions to numpy in `to_numpy`, but disallow np.asarray etc."""
+        return self.array
+
     def __array__(
         self, dtype: np.typing.DTypeLike = None, /, *, copy: bool | None = None
     ) -> np.ndarray:
         raise UnexpectedDataAccess("Tried accessing data")
 
     def __array_namespace__(self):
         """Present to satisfy is_duck_array test."""
+        from xarray.tests import namespace
+
+        return namespace
 
 
 CONCATENATABLEARRAY_HANDLED_ARRAY_FUNCTIONS: dict[str, Callable] = {}

xarray/tests/namespace.py

@@ -0,0 +1,5 @@
+from xarray.core import duck_array_ops
+
+
+def reshape(array, shape, **kwargs):
+    return type(array)(duck_array_ops.reshape(array.array, shape=shape, **kwargs))

xarray/tests/test_coding_times.py

@@ -41,6 +41,7 @@
 from xarray.core.utils import is_duck_dask_array
 from xarray.testing import assert_equal, assert_identical
 from xarray.tests import (
+    DuckArrayWrapper,
     FirstElementAccessibleArray,
     arm_xfail,
     assert_array_equal,
@@ -1935,6 +1936,31 @@ def test_lazy_decode_timedelta_error() -> None:
         decoded.load()
 
 
+@pytest.mark.parametrize(
+    "calendar",
+    [
+        "standard",
+        pytest.param(
+            "360_day", marks=pytest.mark.skipif(not has_cftime, reason="no cftime")
+        ),
+    ],
+)
+def test_duck_array_decode_times(calendar) -> None:
+    from xarray.core.indexing import LazilyIndexedArray
+
+    days = LazilyIndexedArray(DuckArrayWrapper(np.array([1.0, 2.0, 3.0])))
+    var = Variable(
+        ["time"], days, {"units": "days since 2001-01-01", "calendar": calendar}
+    )
+    decoded = conventions.decode_cf_variable(
+        "foo", var, decode_times=CFDatetimeCoder(use_cftime=None)
+    )
+    if calendar not in _STANDARD_CALENDARS:
+        assert decoded.dtype == np.dtype("O")
+    else:
+        assert decoded.dtype == np.dtype("=M8[ns]")
+
+
 @pytest.mark.parametrize("decode_timedelta", [True, False])
 @pytest.mark.parametrize("mask_and_scale", [True, False])
 def test_decode_timedelta_mask_and_scale(