diff --git a/doc/whats-new.rst b/doc/whats-new.rst index d9f43fa1868..b1c31d903d8 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -49,10 +49,15 @@ Enhancements Bug fixes ~~~~~~~~~ -- Fixed a bug where `to_netcdf(..., unlimited_dims='bar'` yielded NetCDF files - with spurious 0-length dimensions (i.e. `b`, `a`, and `r`) (:issue:`2134`). +- Fixed a bug where ``to_netcdf(..., unlimited_dims='bar')`` yielded NetCDF + files with spurious 0-length dimensions (i.e. ``b``, ``a``, and ``r``) + (:issue:`2134`). By `Joe Hamman `_. +- Removed spurious warnings with ``Dataset.update(Dataset)`` (:issue:`2161`) + and ``array.equals(array)`` when ``array`` contains ``NaT`` (:issue:`2162`). + By `Stephan Hoyer `_. + - Aggregations with :py:meth:`Dataset.reduce` (including ``mean``, ``sum``, etc) no longer drop unrelated coordinates (:issue:`1470`). Also fixed a bug where non-scalar data-variables that did not include the aggregation diff --git a/xarray/conventions.py b/xarray/conventions.py index ed90c34387b..6171c353a0d 100644 --- a/xarray/conventions.py +++ b/xarray/conventions.py @@ -89,7 +89,7 @@ def maybe_encode_nonstring_dtype(var, name=None): warnings.warn('saving variable %s with floating ' 'point data as an integer dtype without ' 'any _FillValue to use for NaNs' % name, - SerializationWarning, stacklevel=3) + SerializationWarning, stacklevel=10) data = duck_array_ops.around(data)[...] data = data.astype(dtype=dtype) var = Variable(dims, data, attrs, encoding) diff --git a/xarray/core/common.py b/xarray/core/common.py index f623091ebdb..d69c60eed56 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -696,7 +696,7 @@ def _resample_immediately(self, freq, dim, how, skipna, "how=\"{how}\", instead consider using " ".resample({dim}=\"{freq}\").{how}('{dim}') ".format( dim=dim, freq=freq, how=how), - DeprecationWarning, stacklevel=3) + FutureWarning, stacklevel=3) if isinstance(dim, basestring): dim = self[dim] diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index 69b0d0825be..065ac165a0d 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -145,10 +145,13 @@ def array_equiv(arr1, arr2): if arr1.shape != arr2.shape: return False - flag_array = (arr1 == arr2) - flag_array |= (isnull(arr1) & isnull(arr2)) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', "In the future, 'NAT == x'") - return bool(flag_array.all()) + flag_array = (arr1 == arr2) + flag_array |= (isnull(arr1) & isnull(arr2)) + + return bool(flag_array.all()) def array_notnull_equiv(arr1, arr2): @@ -159,11 +162,14 @@ def array_notnull_equiv(arr1, arr2): if arr1.shape != arr2.shape: return False - flag_array = (arr1 == arr2) - flag_array |= isnull(arr1) - flag_array |= isnull(arr2) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', "In the future, 'NAT == x'") + + flag_array = (arr1 == arr2) + flag_array |= isnull(arr1) + flag_array |= isnull(arr2) - return bool(flag_array.all()) + return bool(flag_array.all()) def count(data, axis=None): diff --git a/xarray/core/merge.py b/xarray/core/merge.py index d3c9871abef..40a58d6e84e 100644 --- a/xarray/core/merge.py +++ b/xarray/core/merge.py @@ -547,21 +547,24 @@ def dataset_merge_method(dataset, other, overwrite_vars, compat, join): def dataset_update_method(dataset, other): - """Guts of the Dataset.update method + """Guts of the Dataset.update method. - This drops a duplicated coordinates from `other` (GH:2068) + This drops a duplicated coordinates from `other` if `other` is not an + `xarray.Dataset`, e.g., if it's a dict with DataArray values (GH2068, + GH2180). """ from .dataset import Dataset from .dataarray import DataArray - other = other.copy() - for k, obj in other.items(): - if isinstance(obj, (Dataset, DataArray)): - # drop duplicated coordinates - coord_names = [c for c in obj.coords - if c not in obj.dims and c in dataset.coords] - if coord_names: - other[k] = obj.drop(coord_names) + if not isinstance(other, Dataset): + other = OrderedDict(other) + for key, value in other.items(): + if isinstance(value, DataArray): + # drop conflicting coordinates + coord_names = [c for c in value.coords + if c not in value.dims and c in dataset.coords] + if coord_names: + other[key] = value.drop(coord_names) return merge_core([dataset, other], priority_arg=1, indexes=dataset.indexes) diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index 7584ed79a06..3acd26235ce 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -87,7 +87,10 @@ def _importorskip(modname, minversion=None): has_pathlib, requires_pathlib = _importorskip('pathlib2') if has_dask: import dask - dask.set_options(get=dask.get) + if LooseVersion(dask.__version__) < '0.18': + dask.set_options(get=dask.get) + else: + dask.config.set(scheduler='sync') try: import_seaborn() has_seaborn = True @@ -191,7 +194,10 @@ def source_ndarray(array): """Given an ndarray, return the base object which holds its memory, or the object itself. """ - base = getattr(array, 'base', np.asarray(array).base) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'DatetimeIndex.base') + warnings.filterwarnings('ignore', 'TimedeltaIndex.base') + base = getattr(array, 'base', np.asarray(array).base) if base is None: base = array return base diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 513f5f0834e..714d720bc59 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -363,21 +363,26 @@ def test_roundtrip_cftime_datetime_data_enable_cftimeindex(self): expected_decoded_t0 = np.array([date_type(1, 1, 1)]) expected_calendar = times[0].calendar - with xr.set_options(enable_cftimeindex=True): - with self.roundtrip(expected, save_kwargs=kwds) as actual: - abs_diff = abs(actual.t.values - expected_decoded_t) - assert (abs_diff <= np.timedelta64(1, 's')).all() - assert (actual.t.encoding['units'] == - 'days since 0001-01-01 00:00:00.000000') - assert (actual.t.encoding['calendar'] == - expected_calendar) - - abs_diff = abs(actual.t0.values - expected_decoded_t0) - assert (abs_diff <= np.timedelta64(1, 's')).all() - assert (actual.t0.encoding['units'] == - 'days since 0001-01-01') - assert (actual.t.encoding['calendar'] == - expected_calendar) + with warnings.catch_warnings(): + if expected_calendar in {'proleptic_gregorian', 'gregorian'}: + warnings.filterwarnings( + 'ignore', 'Unable to decode time axis') + + with xr.set_options(enable_cftimeindex=True): + with self.roundtrip(expected, save_kwargs=kwds) as actual: + abs_diff = abs(actual.t.values - expected_decoded_t) + assert (abs_diff <= np.timedelta64(1, 's')).all() + assert (actual.t.encoding['units'] == + 'days since 0001-01-01 00:00:00.000000') + assert (actual.t.encoding['calendar'] == + expected_calendar) + + abs_diff = abs(actual.t0.values - expected_decoded_t0) + assert (abs_diff <= np.timedelta64(1, 's')).all() + assert (actual.t0.encoding['units'] == + 'days since 0001-01-01') + assert (actual.t.encoding['calendar'] == + expected_calendar) def test_roundtrip_timedelta_data(self): time_deltas = pd.to_timedelta(['1h', '2h', 'NaT']) @@ -767,8 +772,11 @@ def test_default_fill_value(self): # Test default encoding for int: ds = Dataset({'x': ('y', np.arange(10.0))}) kwargs = dict(encoding={'x': {'dtype': 'int16'}}) - with self.roundtrip(ds, save_kwargs=kwargs) as actual: - self.assertTrue('_FillValue' not in actual.x.encoding) + with warnings.catch_warnings(): + warnings.filterwarnings( + 'ignore', '.*floating point data as an integer') + with self.roundtrip(ds, save_kwargs=kwargs) as actual: + self.assertTrue('_FillValue' not in actual.x.encoding) self.assertEqual(ds.x.encoding, {}) # Test default encoding for implicit int: diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index 6329e91ac78..4d6ca731bb2 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -130,7 +130,9 @@ def test_decode_cf_datetime_overflow(): expected = (datetime(1677, 12, 31), datetime(2262, 4, 12)) for i, day in enumerate(days): - result = coding.times.decode_cf_datetime(day, units) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'Unable to decode time axis') + result = coding.times.decode_cf_datetime(day, units) assert result == expected[i] diff --git a/xarray/tests/test_conventions.py b/xarray/tests/test_conventions.py index 62ff8d7ee1a..acc1c978579 100644 --- a/xarray/tests/test_conventions.py +++ b/xarray/tests/test_conventions.py @@ -219,7 +219,9 @@ def test_decode_cf_datetime_transition_to_invalid(self): ds = Dataset(coords={'time': [0, 266 * 365]}) units = 'days since 2000-01-01 00:00:00' ds.time.attrs = dict(units=units) - ds_decoded = conventions.decode_cf(ds) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'unable to decode time') + ds_decoded = conventions.decode_cf(ds) expected = [datetime(2000, 1, 1, 0, 0), datetime(2265, 10, 28, 0, 0)] diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index 1e4f313897b..ee5b3514348 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -459,8 +459,7 @@ def counting_get(*args, **kwargs): count[0] += 1 return dask.get(*args, **kwargs) - with dask.set_options(get=counting_get): - ds.load() + ds.load(get=counting_get) assert count[0] == 1 def test_stack(self): diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 35e270f0db7..a2b9262557f 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -4,6 +4,7 @@ from copy import deepcopy from distutils.version import LooseVersion from textwrap import dedent +import warnings import numpy as np import pandas as pd @@ -321,11 +322,14 @@ def test_constructor_from_self_described(self): actual = DataArray(series) assert_equal(expected[0].reset_coords('x', drop=True), actual) - panel = pd.Panel({0: frame}) - actual = DataArray(panel) - expected = DataArray([data], expected.coords, ['dim_0', 'x', 'y']) - expected['dim_0'] = [0] - assert_identical(expected, actual) + if hasattr(pd, 'Panel'): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', r'\W*Panel is deprecated') + panel = pd.Panel({0: frame}) + actual = DataArray(panel) + expected = DataArray([data], expected.coords, ['dim_0', 'x', 'y']) + expected['dim_0'] = [0] + assert_identical(expected, actual) expected = DataArray(data, coords={'x': ['a', 'b'], 'y': [-1, -2], @@ -2310,7 +2314,7 @@ def test_resample_old_vs_new_api(self): array = DataArray(np.ones(10), [('time', times)]) # Simple mean - with pytest.warns(DeprecationWarning): + with pytest.warns(FutureWarning): old_mean = array.resample('1D', 'time', how='mean') new_mean = array.resample(time='1D').mean() assert_identical(old_mean, new_mean) @@ -2319,7 +2323,7 @@ def test_resample_old_vs_new_api(self): attr_array = array.copy() attr_array.attrs['meta'] = 'data' - with pytest.warns(DeprecationWarning): + with pytest.warns(FutureWarning): old_mean = attr_array.resample('1D', dim='time', how='mean', keep_attrs=True) new_mean = attr_array.resample(time='1D').mean(keep_attrs=True) @@ -2330,7 +2334,7 @@ def test_resample_old_vs_new_api(self): nan_array = array.copy() nan_array[1] = np.nan - with pytest.warns(DeprecationWarning): + with pytest.warns(FutureWarning): old_mean = nan_array.resample('1D', 'time', how='mean', skipna=False) new_mean = nan_array.resample(time='1D').mean(skipna=False) @@ -2344,12 +2348,12 @@ def test_resample_old_vs_new_api(self): # Discard attributes on the call using the new api to match # convention from old api new_api = getattr(resampler, method)(keep_attrs=False) - with pytest.warns(DeprecationWarning): + with pytest.warns(FutureWarning): old_api = array.resample('1D', dim='time', how=method) assert_identical(new_api, old_api) for method in [np.mean, np.sum, np.max, np.min]: new_api = resampler.reduce(method) - with pytest.warns(DeprecationWarning): + with pytest.warns(FutureWarning): old_api = array.resample('1D', dim='time', how=method) assert_identical(new_api, old_api) @@ -2703,9 +2707,13 @@ def test_to_pandas(self): # roundtrips for shape in [(3,), (3, 4), (3, 4, 5)]: + if len(shape) > 2 and not hasattr(pd, 'Panel'): + continue dims = list('abc')[:len(shape)] da = DataArray(np.random.randn(*shape), dims=dims) - roundtripped = DataArray(da.to_pandas()).drop(dims) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', r'\W*Panel is deprecated') + roundtripped = DataArray(da.to_pandas()).drop(dims) assert_identical(da, roundtripped) with raises_regex(ValueError, 'cannot convert'): diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 76e41c43c6d..b0cf4ef1572 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -5,6 +5,7 @@ from distutils.version import LooseVersion from io import StringIO from textwrap import dedent +import warnings import numpy as np import pandas as pd @@ -338,14 +339,20 @@ def test_constructor_pandas_single(self): das = [ DataArray(np.random.rand(4), dims=['a']), # series DataArray(np.random.rand(4, 3), dims=['a', 'b']), # df - DataArray(np.random.rand(4, 3, 2), dims=['a', 'b', 'c']), # panel ] - for a in das: - pandas_obj = a.to_pandas() - ds_based_on_pandas = Dataset(pandas_obj) - for dim in ds_based_on_pandas.data_vars: - assert_array_equal(ds_based_on_pandas[dim], pandas_obj[dim]) + if hasattr(pd, 'Panel'): + das.append( + DataArray(np.random.rand(4, 3, 2), dims=['a', 'b', 'c'])) + + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', r'\W*Panel is deprecated') + for a in das: + pandas_obj = a.to_pandas() + ds_based_on_pandas = Dataset(pandas_obj) + for dim in ds_based_on_pandas.data_vars: + assert_array_equal( + ds_based_on_pandas[dim], pandas_obj[dim]) def test_constructor_compat(self): data = OrderedDict([('x', DataArray(0, coords={'y': 1})), @@ -2136,6 +2143,22 @@ def test_update(self): actual.update(other) assert_identical(expected, actual) + def test_update_overwrite_coords(self): + data = Dataset({'a': ('x', [1, 2])}, {'b': 3}) + data.update(Dataset(coords={'b': 4})) + expected = Dataset({'a': ('x', [1, 2])}, {'b': 4}) + assert_identical(data, expected) + + data = Dataset({'a': ('x', [1, 2])}, {'b': 3}) + data.update(Dataset({'c': 5}, coords={'b': 4})) + expected = Dataset({'a': ('x', [1, 2]), 'c': 5}, {'b': 4}) + assert_identical(data, expected) + + data = Dataset({'a': ('x', [1, 2])}, {'b': 3}) + data.update({'c': DataArray(5, coords={'b': 4})}) + expected = Dataset({'a': ('x', [1, 2]), 'c': 5}, {'b': 3}) + assert_identical(data, expected) + def test_update_auto_align(self): ds = Dataset({'x': ('t', [3, 4])}, {'t': [0, 1]}) @@ -2340,14 +2363,14 @@ def test_setitem_with_coords(self): actual = ds.copy() actual['var3'] = other assert_identical(expected, actual) - assert 'numbers' in other # should not change other + assert 'numbers' in other.coords # should not change other # with alignment other = ds['var3'].isel(dim3=slice(1, -1)) other['numbers'] = ('dim3', np.arange(8)) actual = ds.copy() actual['var3'] = other - assert 'numbers' in other # should not change other + assert 'numbers' in other.coords # should not change other expected = ds.copy() expected['var3'] = ds['var3'].isel(dim3=slice(1, -1)) assert_identical(expected, actual) @@ -2359,7 +2382,7 @@ def test_setitem_with_coords(self): actual = ds.copy() actual['var3'] = other assert 'position' in actual - assert 'position' in other + assert 'position' in other.coords # assigning a coordinate-only dataarray actual = ds.copy() @@ -2771,7 +2794,7 @@ def test_resample_old_vs_new_api(self): # Discard attributes on the call using the new api to match # convention from old api new_api = getattr(resampler, method)(keep_attrs=False) - with pytest.warns(DeprecationWarning): + with pytest.warns(FutureWarning): old_api = ds.resample('1D', dim='time', how=method) assert_identical(new_api, old_api) diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index 722d1af14f7..c486a394ae6 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -6,6 +6,7 @@ from datetime import datetime, timedelta from distutils.version import LooseVersion from textwrap import dedent +import warnings import numpy as np import pandas as pd @@ -138,8 +139,10 @@ def _assertIndexedLikeNDArray(self, variable, expected_value0, assert variable.equals(variable.copy()) assert variable.identical(variable.copy()) # check value is equal for both ndarray and Variable - assert variable.values[0] == expected_value0 - assert variable[0].values == expected_value0 + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', "In the future, 'NAT == x'") + assert variable.values[0] == expected_value0 + assert variable[0].values == expected_value0 # check type or dtype is consistent for both ndarray and Variable if expected_dtype is None: # check output type instead of array dtype