From a1597eef0c842e04e479faccfcd752139da8cb78 Mon Sep 17 00:00:00 2001 From: Alex Hilson Date: Wed, 9 May 2018 15:21:39 +0100 Subject: [PATCH 1/5] TST: assert lazy array maintained by to_iris (#2046) --- xarray/tests/test_dataarray.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 32ab3a634cb..9805966c04e 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -3025,12 +3025,11 @@ def test_to_and_from_iris_dask(self): roundtripped = DataArray.from_iris(actual) assert_identical(original, roundtripped) - # If the Iris version supports it then we should get a dask array back + # If the Iris version supports it then we should have a dask array + # at each stage of the conversion if hasattr(actual, 'core_data'): - pass - # TODO This currently fails due to the decoding loading - # the data (#1372) - # self.assertEqual(type(original.data), type(roundtripped.data)) + self.assertEqual(type(original.data), type(actual.core_data())) + self.assertEqual(type(original.data), type(roundtripped.data)) actual.remove_coord('time') auto_time_dimension = DataArray.from_iris(actual) From 99676bdc7d13d85273ddb40978fe6d640f3da79f Mon Sep 17 00:00:00 2001 From: Alex Hilson Date: Wed, 9 May 2018 16:07:47 +0100 Subject: [PATCH 2/5] Add masked_invalid array op, resolves to_iris rechunking issue (#2046) --- xarray/convert.py | 10 ++-------- xarray/core/duck_array_ops.py | 3 +++ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/xarray/convert.py b/xarray/convert.py index a6defd083bf..a3c99119306 100644 --- a/xarray/convert.py +++ b/xarray/convert.py @@ -6,6 +6,7 @@ from .coding.times import CFDatetimeCoder, CFTimedeltaCoder from .conventions import decode_cf +from .core import duck_array_ops from .core.dataarray import DataArray from .core.dtypes import get_fill_value from .core.pycompat import OrderedDict, range @@ -94,7 +95,6 @@ def to_iris(dataarray): # Iris not a hard dependency import iris from iris.fileformats.netcdf import parse_cell_methods - from xarray.core.pycompat import dask_array_type dim_coords = [] aux_coords = [] @@ -121,13 +121,7 @@ def to_iris(dataarray): args['cell_methods'] = \ parse_cell_methods(dataarray.attrs['cell_methods']) - # Create the right type of masked array (should be easier after #1769) - if isinstance(dataarray.data, dask_array_type): - from dask.array import ma as dask_ma - masked_data = dask_ma.masked_invalid(dataarray) - else: - masked_data = np.ma.masked_invalid(dataarray) - + masked_data = duck_array_ops.masked_invalid(dataarray.data) cube = iris.cube.Cube(masked_data, **args) return cube diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index 3a2c123f87e..c7932afd14f 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -101,6 +101,9 @@ def isnull(data): einsum = _dask_or_eager_func('einsum', array_args=slice(1, None), requires_dask='0.17.3') +masked_invalid = _dask_or_eager_func('masked_invalid', eager_module=np.ma, + dask_module=dask_array.ma) + def asarray(data): return data if isinstance(data, dask_array_type) else np.asarray(data) From e37cd302bc2c24d77735e87040af82bc4c5b336a Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Mon, 14 May 2018 12:12:19 -0700 Subject: [PATCH 3/5] Fix dask_module in duck_array_ops.masked_invalid --- xarray/core/duck_array_ops.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index c7932afd14f..b16a9ee78bd 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -101,8 +101,9 @@ def isnull(data): einsum = _dask_or_eager_func('einsum', array_args=slice(1, None), requires_dask='0.17.3') -masked_invalid = _dask_or_eager_func('masked_invalid', eager_module=np.ma, - dask_module=dask_array.ma) +masked_invalid = _dask_or_eager_func( + 'masked_invalid', eager_module=np.ma, + dask_module=getattr(dask_array, 'ma', None) def asarray(data): From 99bdaca6e086920ed05559c0205374af8844b5c2 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Mon, 14 May 2018 12:12:38 -0700 Subject: [PATCH 4/5] Really fix it --- xarray/core/duck_array_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index b16a9ee78bd..ef52b4890ef 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -103,7 +103,7 @@ def isnull(data): masked_invalid = _dask_or_eager_func( 'masked_invalid', eager_module=np.ma, - dask_module=getattr(dask_array, 'ma', None) + dask_module=getattr(dask_array, 'ma', None)) def asarray(data): From eccc231294ad187a320354cb4a37253a4a675c75 Mon Sep 17 00:00:00 2001 From: Alex Hilson Date: Mon, 14 May 2018 23:17:58 +0100 Subject: [PATCH 5/5] Resolving to_iris dask array issues --- doc/whats-new.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 3c2e143bec3..8ae1e1c570a 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -59,6 +59,8 @@ Bug fixes By `Deepak Cherian `_. - ``plot.line()`` learned new kwargs: ``xincrease``, ``yincrease`` that change the direction of the respective axes. By `Deepak Cherian `_. +- Fixed ``to_iris`` to maintain lazy dask array after conversion (:issue:`2046`). + By `Alex Hilson `_ and `Stephan Hoyer `_. .. _whats-new.0.10.3: