Numpy string coding (#5264)

* Add failing test

* Try fix

* Lint

* Require netCDF4 for test

* Move fix to infer dtype

* Update thanks to @shoyer

* Whats new

* Move test and add comment

* Update whats-new.rst

* Update whats-new.rst

Co-authored-by: Illviljan <[email protected]>

Author: znicholls

doc/whats-new.rst

@@ -37,6 +37,9 @@ Deprecations
 
 Bug fixes
 ~~~~~~~~~
+- Subclasses of ``byte`` and ``str`` (e.g. ``np.str_`` and ``np.bytes_``) will now serialise to disk rather than raising a ``ValueError: unsupported dtype for netCDF4 variable: object`` as they did previously (:pull:`5264`).
+  By `Zeb Nicholls <https://github.com/znicholls>`_.
+
 - Fix applying function with non-xarray arguments using :py:func:`xr.map_blocks`.
   By `Cindy Chiao <https://github.com/tcchiao>`_.
 

xarray/coding/strings.py

@@ -17,6 +17,8 @@
 
 
 def create_vlen_dtype(element_type):
+    if element_type not in (str, bytes):
+        raise TypeError("unsupported type for vlen_dtype: {!r}".format(element_type))
     # based on h5py.special_dtype
     return np.dtype("O", metadata={"element_type": element_type})
 

xarray/conventions.py

@@ -157,8 +157,12 @@ def _infer_dtype(array, name=None):
         return np.dtype(float)
 
     element = array[(0,) * array.ndim]
-    if isinstance(element, (bytes, str)):
-        return strings.create_vlen_dtype(type(element))
+    # We use the base types to avoid subclasses of bytes and str (which might
+    # not play nice with e.g. hdf5 datatypes), such as those from numpy
+    if isinstance(element, bytes):
+        return strings.create_vlen_dtype(bytes)
+    elif isinstance(element, str):
+        return strings.create_vlen_dtype(str)
 
     dtype = np.array(element).dtype
     if dtype.kind != "O":

xarray/tests/test_backends.py

@@ -5393,3 +5393,24 @@ def test_h5netcdf_entrypoint(tmp_path):
     assert entrypoint.guess_can_open("something-local.nc4")
     assert entrypoint.guess_can_open("something-local.cdf")
     assert not entrypoint.guess_can_open("not-found-and-no-extension")
+
+
+@requires_netCDF4
+@pytest.mark.parametrize("str_type", (str, np.str_))
+def test_write_file_from_np_str(str_type, tmpdir) -> None:
+    # https://github.com/pydata/xarray/pull/5264
+    scenarios = [str_type(v) for v in ["scenario_a", "scenario_b", "scenario_c"]]
+    years = range(2015, 2100 + 1)
+    tdf = pd.DataFrame(
+        data=np.random.random((len(scenarios), len(years))),
+        columns=years,
+        index=scenarios,
+    )
+    tdf.index.name = "scenario"
+    tdf.columns.name = "year"
+    tdf = tdf.stack()
+    tdf.name = "tas"
+
+    txr = tdf.to_xarray()
+
+    txr.to_netcdf(tmpdir.join("test.nc"))

xarray/tests/test_coding_strings.py

@@ -29,6 +29,12 @@ def test_vlen_dtype() -> None:
     assert strings.check_vlen_dtype(np.dtype(object)) is None
 
 
+@pytest.mark.parametrize("numpy_str_type", (np.str_, np.bytes_))
+def test_numpy_subclass_handling(numpy_str_type) -> None:
+    with pytest.raises(TypeError, match="unsupported type for vlen_dtype"):
+        strings.create_vlen_dtype(numpy_str_type)
+
+
 def test_EncodedStringCoder_decode() -> None:
     coder = strings.EncodedStringCoder()