diff --git a/doc/api-hidden.rst b/doc/api-hidden.rst index 0d8189d21cf..4b2fed8be37 100644 --- a/doc/api-hidden.rst +++ b/doc/api-hidden.rst @@ -152,3 +152,4 @@ plot.FacetGrid.map CFTimeIndex.shift + CFTimeIndex.to_datetimeindex diff --git a/doc/time-series.rst b/doc/time-series.rst index c1a686b409f..7befd954f35 100644 --- a/doc/time-series.rst +++ b/doc/time-series.rst @@ -71,10 +71,11 @@ One unfortunate limitation of using ``datetime64[ns]`` is that it limits the native representation of dates to those that fall between the years 1678 and 2262. When a netCDF file contains dates outside of these bounds, dates will be returned as arrays of :py:class:`cftime.datetime` objects and a :py:class:`~xarray.CFTimeIndex` -can be used for indexing. The :py:class:`~xarray.CFTimeIndex` enables only a subset of -the indexing functionality of a :py:class:`pandas.DatetimeIndex` and is only enabled -when using the standalone version of ``cftime`` (not the version packaged with -earlier versions ``netCDF4``). See :ref:`CFTimeIndex` for more information. +will be used for indexing. :py:class:`~xarray.CFTimeIndex` enables a subset of +the indexing functionality of a :py:class:`pandas.DatetimeIndex` and is only +fully compatible with the standalone version of ``cftime`` (not the version +packaged with earlier versions ``netCDF4``). See :ref:`CFTimeIndex` for more +information. Datetime indexing ----------------- @@ -221,18 +222,28 @@ Non-standard calendars and dates outside the Timestamp-valid range Through the standalone ``cftime`` library and a custom subclass of :py:class:`pandas.Index`, xarray supports a subset of the indexing functionality enabled through the standard :py:class:`pandas.DatetimeIndex` for -dates from non-standard calendars or dates using a standard calendar, but -outside the `Timestamp-valid range`_ (approximately between years 1678 and -2262). This behavior has not yet been turned on by default; to take advantage -of this functionality, you must have the ``enable_cftimeindex`` option set to -``True`` within your context (see :py:func:`~xarray.set_options` for more -information). It is expected that this will become the default behavior in -xarray version 0.11. - -For instance, you can create a DataArray indexed by a time -coordinate with a no-leap calendar within a context manager setting the -``enable_cftimeindex`` option, and the time index will be cast to a -:py:class:`~xarray.CFTimeIndex`: +dates from non-standard calendars commonly used in climate science or dates +using a standard calendar, but outside the `Timestamp-valid range`_ +(approximately between years 1678 and 2262). + +.. note:: + + As of xarray version 0.11, by default, :py:class:`cftime.datetime` objects + will be used to represent times (either in indexes, as a + :py:class:`~xarray.CFTimeIndex`, or in data arrays with dtype object) if + any of the following are true: + + - The dates are from a non-standard calendar + - Any dates are outside the Timestamp-valid range. + + Otherwise pandas-compatible dates from a standard calendar will be + represented with the ``np.datetime64[ns]`` data type, enabling the use of a + :py:class:`pandas.DatetimeIndex` or arrays with dtype ``np.datetime64[ns]`` + and their full set of associated features. + +For example, you can create a DataArray indexed by a time +coordinate with dates from a no-leap calendar and a +:py:class:`~xarray.CFTimeIndex` will automatically be used: .. ipython:: python @@ -241,27 +252,11 @@ coordinate with a no-leap calendar within a context manager setting the dates = [DatetimeNoLeap(year, month, 1) for year, month in product(range(1, 3), range(1, 13))] - with xr.set_options(enable_cftimeindex=True): - da = xr.DataArray(np.arange(24), coords=[dates], dims=['time'], - name='foo') + da = xr.DataArray(np.arange(24), coords=[dates], dims=['time'], name='foo') -.. note:: - - With the ``enable_cftimeindex`` option activated, a :py:class:`~xarray.CFTimeIndex` - will be used for time indexing if any of the following are true: - - - The dates are from a non-standard calendar - - Any dates are outside the Timestamp-valid range - - Otherwise a :py:class:`pandas.DatetimeIndex` will be used. In addition, if any - variable (not just an index variable) is encoded using a non-standard - calendar, its times will be decoded into :py:class:`cftime.datetime` objects, - regardless of whether or not they can be represented using - ``np.datetime64[ns]`` objects. - xarray also includes a :py:func:`~xarray.cftime_range` function, which enables -creating a :py:class:`~xarray.CFTimeIndex` with regularly-spaced dates. For instance, we can -create the same dates and DataArray we created above using: +creating a :py:class:`~xarray.CFTimeIndex` with regularly-spaced dates. For +instance, we can create the same dates and DataArray we created above using: .. ipython:: python @@ -317,13 +312,42 @@ For data indexed by a :py:class:`~xarray.CFTimeIndex` xarray currently supports: .. ipython:: python - da.to_netcdf('example.nc') - xr.open_dataset('example.nc') + da.to_netcdf('example-no-leap.nc') + xr.open_dataset('example-no-leap.nc') .. note:: - Currently resampling along the time dimension for data indexed by a - :py:class:`~xarray.CFTimeIndex` is not supported. + While much of the time series functionality that is possible for standard + dates has been implemented for dates from non-standard calendars, there are + still some remaining important features that have yet to be implemented, + for example: + + - Resampling along the time dimension for data indexed by a + :py:class:`~xarray.CFTimeIndex` (:issue:`2191`, :issue:`2458`) + - Built-in plotting of data with :py:class:`cftime.datetime` coordinate axes + (:issue:`2164`). + + For some use-cases it may still be useful to convert from + a :py:class:`~xarray.CFTimeIndex` to a :py:class:`pandas.DatetimeIndex`, + despite the difference in calendar types (e.g. to allow the use of some + forms of resample with non-standard calendars). The recommended way of + doing this is to use the built-in + :py:meth:`~xarray.CFTimeIndex.to_datetimeindex` method: + + .. ipython:: python + + modern_times = xr.cftime_range('2000', periods=24, freq='MS', calendar='noleap') + da = xr.DataArray(range(24), [('time', modern_times)]) + da + datetimeindex = da.indexes['time'].to_datetimeindex() + da['time'] = datetimeindex + da.resample(time='Y').mean('time') + + However in this case one should use caution to only perform operations which + do not depend on differences between dates (e.g. differentiation, + interpolation, or upsampling with resample), as these could introduce subtle + and silent errors due to the difference in calendar types between the dates + encoded in your data and the dates stored in memory. .. _Timestamp-valid range: https://pandas.pydata.org/pandas-docs/stable/timeseries.html#timestamp-limitations .. _ISO 8601-format: https://en.wikipedia.org/wiki/ISO_8601 diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 2ffbc60622d..9db3d35af84 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -33,6 +33,22 @@ v0.11.0 (unreleased) Breaking changes ~~~~~~~~~~~~~~~~ +- For non-standard calendars commonly used in climate science, xarray will now + always use :py:class:`cftime.datetime` objects, rather than by default try to + coerce them to ``np.datetime64[ns]`` objects. A + :py:class:`~xarray.CFTimeIndex` will be used for indexing along time + coordinates in these cases. A new method, + :py:meth:`~xarray.CFTimeIndex.to_datetimeindex`, has been added + to aid in converting from a :py:class:`~xarray.CFTimeIndex` to a + :py:class:`pandas.DatetimeIndex` for the remaining use-cases where + using a :py:class:`~xarray.CFTimeIndex` is still a limitation (e.g. for + resample or plotting). Setting the ``enable_cftimeindex`` option is now a + no-op and emits a ``FutureWarning``. +- ``Dataset.T`` has been removed as a shortcut for :py:meth:`Dataset.transpose`. + Call :py:meth:`Dataset.transpose` directly instead. +- Iterating over a ``Dataset`` now includes only data variables, not coordinates. + Similarily, calling ``len`` and ``bool`` on a ``Dataset`` now + includes only data variables - Finished deprecation cycles: - ``Dataset.T`` has been removed as a shortcut for :py:meth:`Dataset.transpose`. Call :py:meth:`Dataset.transpose` directly instead. diff --git a/xarray/coding/cftimeindex.py b/xarray/coding/cftimeindex.py index 5de055c1b9a..2ce996b2bd2 100644 --- a/xarray/coding/cftimeindex.py +++ b/xarray/coding/cftimeindex.py @@ -42,6 +42,7 @@ from __future__ import absolute_import import re +import warnings from datetime import timedelta import numpy as np @@ -50,6 +51,8 @@ from xarray.core import pycompat from xarray.core.utils import is_scalar +from .times import cftime_to_nptime, infer_calendar_name, _STANDARD_CALENDARS + def named(name, pattern): return '(?P<' + name + '>' + pattern + ')' @@ -381,6 +384,56 @@ def _add_delta(self, deltas): # pandas. No longer used as of pandas 0.23. return self + deltas + def to_datetimeindex(self, unsafe=False): + """If possible, convert this index to a pandas.DatetimeIndex. + + Parameters + ---------- + unsafe : bool + Flag to turn off warning when converting from a CFTimeIndex with + a non-standard calendar to a DatetimeIndex (default ``False``). + + Returns + ------- + pandas.DatetimeIndex + + Raises + ------ + ValueError + If the CFTimeIndex contains dates that are not possible in the + standard calendar or outside the pandas.Timestamp-valid range. + + Warns + ----- + RuntimeWarning + If converting from a non-standard calendar to a DatetimeIndex. + + Warnings + -------- + Note that for non-standard calendars, this will change the calendar + type of the index. In that case the result of this method should be + used with caution. + + Examples + -------- + >>> import xarray as xr + >>> times = xr.cftime_range('2000', periods=2, calendar='gregorian') + >>> times + CFTimeIndex([2000-01-01 00:00:00, 2000-01-02 00:00:00], dtype='object') + >>> times.to_datetimeindex() + DatetimeIndex(['2000-01-01', '2000-01-02'], dtype='datetime64[ns]', freq=None) + """ # noqa: E501 + nptimes = cftime_to_nptime(self) + calendar = infer_calendar_name(self) + if calendar not in _STANDARD_CALENDARS and not unsafe: + warnings.warn( + 'Converting a CFTimeIndex with dates from a non-standard ' + 'calendar, {!r}, to a pandas.DatetimeIndex, which uses dates ' + 'from the standard calendar. This may lead to subtle errors ' + 'in operations that depend on the length of time between ' + 'dates.'.format(calendar), RuntimeWarning) + return pd.DatetimeIndex(nptimes) + def _parse_iso8601_without_reso(date_type, datetime_str): date, _ = _parse_iso8601_with_reso(date_type, datetime_str) diff --git a/xarray/coding/times.py b/xarray/coding/times.py index 16380976def..dfc4b2fb023 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -12,7 +12,6 @@ from ..core import indexing from ..core.common import contains_cftime_datetimes from ..core.formatting import first_n_items, format_timestamp, last_item -from ..core.options import OPTIONS from ..core.pycompat import PY3 from ..core.variable import Variable from .variables import ( @@ -61,8 +60,9 @@ def _require_standalone_cftime(): try: import cftime # noqa: F401 except ImportError: - raise ImportError('Using a CFTimeIndex requires the standalone ' - 'version of the cftime library.') + raise ImportError('Decoding times with non-standard calendars ' + 'or outside the pandas.Timestamp-valid range ' + 'requires the standalone cftime package.') def _netcdf_to_numpy_timeunit(units): @@ -84,41 +84,32 @@ def _unpack_netcdf_time_units(units): return delta_units, ref_date -def _decode_datetime_with_cftime(num_dates, units, calendar, - enable_cftimeindex): +def _decode_datetime_with_cftime(num_dates, units, calendar): cftime = _import_cftime() - if enable_cftimeindex: - _require_standalone_cftime() + + if cftime.__name__ == 'cftime': dates = np.asarray(cftime.num2date(num_dates, units, calendar, only_use_cftime_datetimes=True)) else: + # Must be using num2date from an old version of netCDF4 which + # does not have the only_use_cftime_datetimes option. dates = np.asarray(cftime.num2date(num_dates, units, calendar)) if (dates[np.nanargmin(num_dates)].year < 1678 or dates[np.nanargmax(num_dates)].year >= 2262): - if not enable_cftimeindex or calendar in _STANDARD_CALENDARS: + if calendar in _STANDARD_CALENDARS: warnings.warn( 'Unable to decode time axis into full ' 'numpy.datetime64 objects, continuing using dummy ' 'cftime.datetime objects instead, reason: dates out ' 'of range', SerializationWarning, stacklevel=3) else: - if enable_cftimeindex: - if calendar in _STANDARD_CALENDARS: - dates = cftime_to_nptime(dates) - else: - try: - dates = cftime_to_nptime(dates) - except ValueError as e: - warnings.warn( - 'Unable to decode time axis into full ' - 'numpy.datetime64 objects, continuing using ' - 'dummy cftime.datetime objects instead, reason:' - '{0}'.format(e), SerializationWarning, stacklevel=3) + if calendar in _STANDARD_CALENDARS: + dates = cftime_to_nptime(dates) return dates -def _decode_cf_datetime_dtype(data, units, calendar, enable_cftimeindex): +def _decode_cf_datetime_dtype(data, units, calendar): # Verify that at least the first and last date can be decoded # successfully. Otherwise, tracebacks end up swallowed by # Dataset.__repr__ when users try to view their lazily decoded array. @@ -128,8 +119,7 @@ def _decode_cf_datetime_dtype(data, units, calendar, enable_cftimeindex): last_item(values) or [0]]) try: - result = decode_cf_datetime(example_value, units, calendar, - enable_cftimeindex) + result = decode_cf_datetime(example_value, units, calendar) except Exception: calendar_msg = ('the default calendar' if calendar is None else 'calendar %r' % calendar) @@ -145,8 +135,7 @@ def _decode_cf_datetime_dtype(data, units, calendar, enable_cftimeindex): return dtype -def decode_cf_datetime(num_dates, units, calendar=None, - enable_cftimeindex=False): +def decode_cf_datetime(num_dates, units, calendar=None): """Given an array of numeric dates in netCDF format, convert it into a numpy array of date time objects. @@ -200,8 +189,7 @@ def decode_cf_datetime(num_dates, units, calendar=None, except (OutOfBoundsDatetime, OverflowError): dates = _decode_datetime_with_cftime( - flat_num_dates.astype(np.float), units, calendar, - enable_cftimeindex) + flat_num_dates.astype(np.float), units, calendar) return dates.reshape(num_dates.shape) @@ -291,7 +279,16 @@ def cftime_to_nptime(times): times = np.asarray(times) new = np.empty(times.shape, dtype='M8[ns]') for i, t in np.ndenumerate(times): - dt = datetime(t.year, t.month, t.day, t.hour, t.minute, t.second) + try: + # Use pandas.Timestamp in place of datetime.datetime, because + # NumPy casts it safely it np.datetime64[ns] for dates outside + # 1678 to 2262 (this is not currently the case for + # datetime.datetime). + dt = pd.Timestamp(t.year, t.month, t.day, t.hour, t.minute, + t.second, t.microsecond) + except ValueError as e: + raise ValueError('Cannot convert date {} to a date in the ' + 'standard calendar. Reason: {}.'.format(t, e)) new[i] = np.datetime64(dt) return new @@ -404,15 +401,12 @@ def encode(self, variable, name=None): def decode(self, variable, name=None): dims, data, attrs, encoding = unpack_for_decoding(variable) - enable_cftimeindex = OPTIONS['enable_cftimeindex'] if 'units' in attrs and 'since' in attrs['units']: units = pop_to(attrs, encoding, 'units') calendar = pop_to(attrs, encoding, 'calendar') - dtype = _decode_cf_datetime_dtype( - data, units, calendar, enable_cftimeindex) + dtype = _decode_cf_datetime_dtype(data, units, calendar) transform = partial( - decode_cf_datetime, units=units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + decode_cf_datetime, units=units, calendar=calendar) data = lazy_elemwise_func(data, transform, dtype) return Variable(dims, data, attrs, encoding) diff --git a/xarray/core/common.py b/xarray/core/common.py index e303c485523..508a19b7115 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -659,6 +659,7 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, from .dataarray import DataArray from .resample import RESAMPLE_DIM + from ..coding.cftimeindex import CFTimeIndex if keep_attrs is None: keep_attrs = _get_keep_attrs(default=False) @@ -687,6 +688,20 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, else: raise TypeError("Dimension name should be a string; " "was passed %r" % dim) + + if isinstance(self.indexes[dim_name], CFTimeIndex): + raise NotImplementedError( + 'Resample is currently not supported along a dimension ' + 'indexed by a CFTimeIndex. For certain kinds of downsampling ' + 'it may be possible to work around this by converting your ' + 'time index to a DatetimeIndex using ' + 'CFTimeIndex.to_datetimeindex. Use caution when doing this ' + 'however, because switching to a DatetimeIndex from a ' + 'CFTimeIndex with a non-standard calendar entails a change ' + 'in the calendar type, which could lead to subtle and silent ' + 'errors.' + ) + group = DataArray(dim, [(dim.dims, dim)], name=RESAMPLE_DIM) grouper = pd.Grouper(freq=freq, closed=closed, label=label, base=base) resampler = self._resample_cls(self, group=group, dim=dim_name, @@ -700,6 +715,8 @@ def _resample_immediately(self, freq, dim, how, skipna, """Implement the original version of .resample() which immediately executes the desired resampling operation. """ from .dataarray import DataArray + from ..coding.cftimeindex import CFTimeIndex + RESAMPLE_DIM = '__resample_dim__' warnings.warn("\n.resample() has been modified to defer " @@ -709,8 +726,22 @@ def _resample_immediately(self, freq, dim, how, skipna, dim=dim, freq=freq, how=how), FutureWarning, stacklevel=3) + if isinstance(self.indexes[dim], CFTimeIndex): + raise NotImplementedError( + 'Resample is currently not supported along a dimension ' + 'indexed by a CFTimeIndex. For certain kinds of downsampling ' + 'it may be possible to work around this by converting your ' + 'time index to a DatetimeIndex using ' + 'CFTimeIndex.to_datetimeindex. Use caution when doing this ' + 'however, because switching to a DatetimeIndex from a ' + 'CFTimeIndex with a non-standard calendar entails a change ' + 'in the calendar type, which could lead to subtle and silent ' + 'errors.' + ) + if isinstance(dim, basestring): dim = self[dim] + group = DataArray(dim, [(dim.dims, dim)], name=RESAMPLE_DIM) grouper = pd.Grouper(freq=freq, how=how, closed=closed, label=label, base=base) diff --git a/xarray/core/options.py b/xarray/core/options.py index eb3013d5233..ab461ca86bc 100644 --- a/xarray/core/options.py +++ b/xarray/core/options.py @@ -1,5 +1,7 @@ from __future__ import absolute_import, division, print_function +import warnings + DISPLAY_WIDTH = 'display_width' ARITHMETIC_JOIN = 'arithmetic_join' ENABLE_CFTIMEINDEX = 'enable_cftimeindex' @@ -12,7 +14,7 @@ OPTIONS = { DISPLAY_WIDTH: 80, ARITHMETIC_JOIN: 'inner', - ENABLE_CFTIMEINDEX: False, + ENABLE_CFTIMEINDEX: True, FILE_CACHE_MAXSIZE: 128, CMAP_SEQUENTIAL: 'viridis', CMAP_DIVERGENT: 'RdBu_r', @@ -40,8 +42,16 @@ def _set_file_cache_maxsize(value): FILE_CACHE.maxsize = value +def _warn_on_setting_enable_cftimeindex(enable_cftimeindex): + warnings.warn( + 'The enable_cftimeindex option is now a no-op ' + 'and will be removed in a future version of xarray.', + FutureWarning) + + _SETTERS = { FILE_CACHE_MAXSIZE: _set_file_cache_maxsize, + ENABLE_CFTIMEINDEX: _warn_on_setting_enable_cftimeindex } @@ -65,9 +75,6 @@ class set_options(object): Default: ``80``. - ``arithmetic_join``: DataArray/Dataset alignment in binary operations. Default: ``'inner'``. - - ``enable_cftimeindex``: flag to enable using a ``CFTimeIndex`` - for time indexes with non-standard calendars or dates outside the - Timestamp-valid range. Default: ``False``. - ``file_cache_maxsize``: maximum number of open files to hold in xarray's global least-recently-usage cached. This should be smaller than your system's per-process file descriptor limit, e.g., ``ulimit -n`` on Linux. @@ -102,7 +109,7 @@ class set_options(object): """ def __init__(self, **kwargs): - self.old = OPTIONS.copy() + self.old = {} for k, v in kwargs.items(): if k not in OPTIONS: raise ValueError( @@ -111,6 +118,7 @@ def __init__(self, **kwargs): if k in _VALIDATORS and not _VALIDATORS[k](v): raise ValueError( 'option %r given an invalid value: %r' % (k, v)) + self.old[k] = OPTIONS[k] self._apply_update(kwargs) def _apply_update(self, options_dict): @@ -123,5 +131,4 @@ def __enter__(self): return def __exit__(self, type, value, traceback): - OPTIONS.clear() self._apply_update(self.old) diff --git a/xarray/core/utils.py b/xarray/core/utils.py index 5c9d8bfbf77..015916d668e 100644 --- a/xarray/core/utils.py +++ b/xarray/core/utils.py @@ -13,7 +13,6 @@ import numpy as np import pandas as pd -from .options import OPTIONS from .pycompat import ( OrderedDict, basestring, bytes_type, dask_array_type, iteritems) @@ -41,16 +40,13 @@ def wrapper(*args, **kwargs): def _maybe_cast_to_cftimeindex(index): from ..coding.cftimeindex import CFTimeIndex - if not OPTIONS['enable_cftimeindex']: - return index - else: - if index.dtype == 'O': - try: - return CFTimeIndex(index) - except (ImportError, TypeError): - return index - else: + if index.dtype == 'O': + try: + return CFTimeIndex(index) + except (ImportError, TypeError): return index + else: + return index def safe_cast_to_index(array): diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index 7129157ec7f..8d21e084946 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -145,8 +145,13 @@ def plot(darray, row=None, col=None, col_wrap=None, ax=None, hue=None, darray = darray.squeeze() if contains_cftime_datetimes(darray): - raise NotImplementedError('Plotting arrays of cftime.datetime objects ' - 'is currently not possible.') + raise NotImplementedError( + 'Built-in plotting of arrays of cftime.datetime objects or arrays ' + 'indexed by cftime.datetime objects is currently not implemented ' + 'within xarray. A possible workaround is to use the ' + 'nc-time-axis package ' + '(https://github.com/SciTools/nc-time-axis) to convert the dates ' + 'to a plottable type and plot your data directly with matplotlib.') plot_dims = set(darray.dims) plot_dims.discard(row) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index c6a2df733fa..80d3a9d526e 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -356,7 +356,7 @@ def test_roundtrip_numpy_datetime_data(self): assert actual.t0.encoding['units'] == 'days since 1950-01-01' @requires_cftime - def test_roundtrip_cftime_datetime_data_enable_cftimeindex(self): + def test_roundtrip_cftime_datetime_data(self): from .test_coding_times import _all_cftime_date_types date_types = _all_cftime_date_types() @@ -373,21 +373,20 @@ def test_roundtrip_cftime_datetime_data_enable_cftimeindex(self): warnings.filterwarnings( 'ignore', 'Unable to decode time axis') - with xr.set_options(enable_cftimeindex=True): - with self.roundtrip(expected, save_kwargs=kwds) as actual: - abs_diff = abs(actual.t.values - expected_decoded_t) - assert (abs_diff <= np.timedelta64(1, 's')).all() - assert (actual.t.encoding['units'] == - 'days since 0001-01-01 00:00:00.000000') - assert (actual.t.encoding['calendar'] == - expected_calendar) - - abs_diff = abs(actual.t0.values - expected_decoded_t0) - assert (abs_diff <= np.timedelta64(1, 's')).all() - assert (actual.t0.encoding['units'] == - 'days since 0001-01-01') - assert (actual.t.encoding['calendar'] == - expected_calendar) + with self.roundtrip(expected, save_kwargs=kwds) as actual: + abs_diff = abs(actual.t.values - expected_decoded_t) + assert (abs_diff <= np.timedelta64(1, 's')).all() + assert (actual.t.encoding['units'] == + 'days since 0001-01-01 00:00:00.000000') + assert (actual.t.encoding['calendar'] == + expected_calendar) + + abs_diff = abs(actual.t0.values - expected_decoded_t0) + assert (abs_diff <= np.timedelta64(1, 's')).all() + assert (actual.t0.encoding['units'] == + 'days since 0001-01-01') + assert (actual.t.encoding['calendar'] == + expected_calendar) def test_roundtrip_timedelta_data(self): time_deltas = pd.to_timedelta(['1h', '2h', 'NaT']) @@ -2087,7 +2086,7 @@ def test_roundtrip_numpy_datetime_data(self): with self.roundtrip(expected) as actual: assert_identical(expected, actual) - def test_roundtrip_cftime_datetime_data_enable_cftimeindex(self): + def test_roundtrip_cftime_datetime_data(self): # Override method in DatasetIOBase - remove not applicable # save_kwds from .test_coding_times import _all_cftime_date_types @@ -2099,33 +2098,12 @@ def test_roundtrip_cftime_datetime_data_enable_cftimeindex(self): expected_decoded_t = np.array(times) expected_decoded_t0 = np.array([date_type(1, 1, 1)]) - with xr.set_options(enable_cftimeindex=True): - with self.roundtrip(expected) as actual: - abs_diff = abs(actual.t.values - expected_decoded_t) - assert (abs_diff <= np.timedelta64(1, 's')).all() - - abs_diff = abs(actual.t0.values - expected_decoded_t0) - assert (abs_diff <= np.timedelta64(1, 's')).all() - - def test_roundtrip_cftime_datetime_data_disable_cftimeindex(self): - # Override method in DatasetIOBase - remove not applicable - # save_kwds - from .test_coding_times import _all_cftime_date_types - - date_types = _all_cftime_date_types() - for date_type in date_types.values(): - times = [date_type(1, 1, 1), date_type(1, 1, 2)] - expected = Dataset({'t': ('t', times), 't0': times[0]}) - expected_decoded_t = np.array(times) - expected_decoded_t0 = np.array([date_type(1, 1, 1)]) - - with xr.set_options(enable_cftimeindex=False): - with self.roundtrip(expected) as actual: - abs_diff = abs(actual.t.values - expected_decoded_t) - assert (abs_diff <= np.timedelta64(1, 's')).all() + with self.roundtrip(expected) as actual: + abs_diff = abs(actual.t.values - expected_decoded_t) + assert (abs_diff <= np.timedelta64(1, 's')).all() - abs_diff = abs(actual.t0.values - expected_decoded_t0) - assert (abs_diff <= np.timedelta64(1, 's')).all() + abs_diff = abs(actual.t0.values - expected_decoded_t0) + assert (abs_diff <= np.timedelta64(1, 's')).all() def test_write_store(self): # Override method in DatasetIOBase - not applicable to dask diff --git a/xarray/tests/test_cftimeindex.py b/xarray/tests/test_cftimeindex.py index e18c55d2fae..5e710827ff8 100644 --- a/xarray/tests/test_cftimeindex.py +++ b/xarray/tests/test_cftimeindex.py @@ -13,7 +13,8 @@ from xarray.tests import assert_array_equal, assert_identical from . import has_cftime, has_cftime_or_netCDF4, requires_cftime -from .test_coding_times import _all_cftime_date_types +from .test_coding_times import (_all_cftime_date_types, _ALL_CALENDARS, + _NON_STANDARD_CALENDARS) def date_dict(year=None, month=None, day=None, @@ -360,7 +361,7 @@ def test_groupby(da): @pytest.mark.skipif(not has_cftime, reason='cftime not installed') def test_resample_error(da): - with pytest.raises(TypeError): + with pytest.raises(NotImplementedError, match='to_datetimeindex'): da.resample(time='Y') @@ -594,18 +595,16 @@ def test_indexing_in_dataframe_iloc(df, index): @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize('enable_cftimeindex', [False, True]) -def test_concat_cftimeindex(date_type, enable_cftimeindex): - with xr.set_options(enable_cftimeindex=enable_cftimeindex): - da1 = xr.DataArray( - [1., 2.], coords=[[date_type(1, 1, 1), date_type(1, 2, 1)]], - dims=['time']) - da2 = xr.DataArray( - [3., 4.], coords=[[date_type(1, 3, 1), date_type(1, 4, 1)]], - dims=['time']) - da = xr.concat([da1, da2], dim='time') - - if enable_cftimeindex and has_cftime: +def test_concat_cftimeindex(date_type): + da1 = xr.DataArray( + [1., 2.], coords=[[date_type(1, 1, 1), date_type(1, 2, 1)]], + dims=['time']) + da2 = xr.DataArray( + [3., 4.], coords=[[date_type(1, 3, 1), date_type(1, 4, 1)]], + dims=['time']) + da = xr.concat([da1, da2], dim='time') + + if has_cftime: assert isinstance(da.indexes['time'], CFTimeIndex) else: assert isinstance(da.indexes['time'], pd.Index) @@ -746,3 +745,37 @@ def test_parse_array_of_cftime_strings(): expected = np.array(DatetimeNoLeap(2000, 1, 1)) result = _parse_array_of_cftime_strings(strings, DatetimeNoLeap) np.testing.assert_array_equal(result, expected) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('calendar', _ALL_CALENDARS) +@pytest.mark.parametrize('unsafe', [False, True]) +def test_to_datetimeindex(calendar, unsafe): + index = xr.cftime_range('2000', periods=5, calendar=calendar) + expected = pd.date_range('2000', periods=5) + + if calendar in _NON_STANDARD_CALENDARS and not unsafe: + with pytest.warns(RuntimeWarning, match='non-standard'): + result = index.to_datetimeindex() + else: + result = index.to_datetimeindex() + + assert result.equals(expected) + np.testing.assert_array_equal(result, expected) + assert isinstance(result, pd.DatetimeIndex) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('calendar', _ALL_CALENDARS) +def test_to_datetimeindex_out_of_range(calendar): + index = xr.cftime_range('0001', periods=5, calendar=calendar) + with pytest.raises(ValueError, match='0001'): + index.to_datetimeindex() + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('calendar', ['all_leap', '360_day']) +def test_to_datetimeindex_feb_29(calendar): + index = xr.cftime_range('2001-02-28', periods=2, calendar=calendar) + with pytest.raises(ValueError, match='29'): + index.to_datetimeindex() diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index f76b8c3ceab..0ca57f98a6d 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -7,10 +7,9 @@ import pandas as pd import pytest -from xarray import DataArray, Variable, coding, decode_cf, set_options -from xarray.coding.times import (_import_cftime, decode_cf_datetime, - encode_cf_datetime) -from xarray.coding.variables import SerializationWarning +from xarray import DataArray, Variable, coding, decode_cf +from xarray.coding.times import (_import_cftime, cftime_to_nptime, + decode_cf_datetime, encode_cf_datetime) from xarray.core.common import contains_cftime_datetimes from . import ( @@ -48,21 +47,14 @@ ([0.5, 1.5], 'hours since 1900-01-01T00:00:00'), (0, 'milliseconds since 2000-01-01T00:00:00'), (0, 'microseconds since 2000-01-01T00:00:00'), - (np.int32(788961600), 'seconds since 1981-01-01') # GH2002 + (np.int32(788961600), 'seconds since 1981-01-01'), # GH2002 + (12300 + np.arange(5), 'hour since 1680-01-01 00:00:00.500000') ] _CF_DATETIME_TESTS = [num_dates_units + (calendar,) for num_dates_units, calendar in product(_CF_DATETIME_NUM_DATES_UNITS, _STANDARD_CALENDARS)] -@np.vectorize -def _ensure_naive_tz(dt): - if hasattr(dt, 'tzinfo'): - return dt.replace(tzinfo=None) - else: - return dt - - def _all_cftime_date_types(): try: import cftime @@ -83,24 +75,27 @@ def _all_cftime_date_types(): _CF_DATETIME_TESTS) def test_cf_datetime(num_dates, units, calendar): cftime = _import_cftime() - expected = _ensure_naive_tz( - cftime.num2date(num_dates, units, calendar)) + if cftime.__name__ == 'cftime': + expected = cftime.num2date(num_dates, units, calendar, + only_use_cftime_datetimes=True) + else: + expected = cftime.num2date(num_dates, units, calendar) + min_y = np.ravel(np.atleast_1d(expected))[np.nanargmin(num_dates)].year + max_y = np.ravel(np.atleast_1d(expected))[np.nanargmax(num_dates)].year + if min_y >= 1678 and max_y < 2262: + expected = cftime_to_nptime(expected) + with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime(num_dates, units, calendar) - if (isinstance(actual, np.ndarray) and - np.issubdtype(actual.dtype, np.datetime64)): - # self.assertEqual(actual.dtype.kind, 'M') - # For some reason, numpy 1.8 does not compare ns precision - # datetime64 arrays as equal to arrays of datetime objects, - # but it works for us precision. Thus, convert to us - # precision for the actual array equal comparison... - actual_cmp = actual.astype('M8[us]') - else: - actual_cmp = actual - assert_array_equal(expected, actual_cmp) + + abs_diff = np.atleast_1d(abs(actual - expected)).astype(np.timedelta64) + # once we no longer support versions of netCDF4 older than 1.1.5, + # we could do this check with near microsecond accuracy: + # https://github.com/Unidata/netcdf4-python/issues/355 + assert (abs_diff <= np.timedelta64(1, 's')).all() encoded, _, _ = coding.times.encode_cf_datetime(actual, units, calendar) if '1-1-1' not in units: @@ -124,8 +119,12 @@ def test_decode_cf_datetime_overflow(): # checks for # https://github.com/pydata/pandas/issues/14068 # https://github.com/pydata/xarray/issues/975 + try: + from cftime import DatetimeGregorian + except ImportError: + from netcdftime import DatetimeGregorian - from datetime import datetime + datetime = DatetimeGregorian units = 'days since 2000-01-01 00:00:00' # date after 2262 and before 1678 @@ -151,39 +150,32 @@ def test_decode_cf_datetime_non_standard_units(): @requires_cftime_or_netCDF4 def test_decode_cf_datetime_non_iso_strings(): # datetime strings that are _almost_ ISO compliant but not quite, - # but which netCDF4.num2date can still parse correctly + # but which cftime.num2date can still parse correctly expected = pd.date_range(periods=100, start='2000-01-01', freq='h') cases = [(np.arange(100), 'hours since 2000-01-01 0'), (np.arange(100), 'hours since 2000-1-1 0'), (np.arange(100), 'hours since 2000-01-01 0:00')] for num_dates, units in cases: actual = coding.times.decode_cf_datetime(num_dates, units) - assert_array_equal(actual, expected) + abs_diff = abs(actual - expected.values) + # once we no longer support versions of netCDF4 older than 1.1.5, + # we could do this check with near microsecond accuracy: + # https://github.com/Unidata/netcdf4-python/issues/355 + assert (abs_diff <= np.timedelta64(1, 's')).all() @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_STANDARD_CALENDARS, [False, True])) -def test_decode_standard_calendar_inside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - +@pytest.mark.parametrize('calendar', _STANDARD_CALENDARS) +def test_decode_standard_calendar_inside_timestamp_range(calendar): cftime = _import_cftime() + units = 'days since 0001-01-01' - times = pd.date_range('2001-04-01-00', end='2001-04-30-23', - freq='H') - noleap_time = cftime.date2num(times.to_pydatetime(), units, - calendar=calendar) + times = pd.date_range('2001-04-01-00', end='2001-04-30-23', freq='H') + time = cftime.date2num(times.to_pydatetime(), units, calendar=calendar) expected = times.values expected_dtype = np.dtype('M8[ns]') - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', 'Unable to decode time axis') - actual = coding.times.decode_cf_datetime( - noleap_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + actual = coding.times.decode_cf_datetime(time, units, calendar=calendar) assert actual.dtype == expected_dtype abs_diff = abs(actual - expected) # once we no longer support versions of netCDF4 older than 1.1.5, @@ -193,32 +185,28 @@ def test_decode_standard_calendar_inside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_NON_STANDARD_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _NON_STANDARD_CALENDARS) def test_decode_non_standard_calendar_inside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - + calendar): cftime = _import_cftime() units = 'days since 0001-01-01' times = pd.date_range('2001-04-01-00', end='2001-04-30-23', freq='H') - noleap_time = cftime.date2num(times.to_pydatetime(), units, - calendar=calendar) - if enable_cftimeindex: - expected = cftime.num2date(noleap_time, units, calendar=calendar) - expected_dtype = np.dtype('O') + non_standard_time = cftime.date2num( + times.to_pydatetime(), units, calendar=calendar) + + if cftime.__name__ == 'cftime': + expected = cftime.num2date( + non_standard_time, units, calendar=calendar, + only_use_cftime_datetimes=True) else: - expected = times.values - expected_dtype = np.dtype('M8[ns]') + expected = cftime.num2date(non_standard_time, units, + calendar=calendar) - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', 'Unable to decode time axis') - actual = coding.times.decode_cf_datetime( - noleap_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + expected_dtype = np.dtype('O') + + actual = coding.times.decode_cf_datetime( + non_standard_time, units, calendar=calendar) assert actual.dtype == expected_dtype abs_diff = abs(actual - expected) # once we no longer support versions of netCDF4 older than 1.1.5, @@ -228,33 +216,27 @@ def test_decode_non_standard_calendar_inside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_ALL_CALENDARS, [False, True])) -def test_decode_dates_outside_timestamp_range( - calendar, enable_cftimeindex): +@pytest.mark.parametrize('calendar', _ALL_CALENDARS) +def test_decode_dates_outside_timestamp_range(calendar): from datetime import datetime - - if enable_cftimeindex: - pytest.importorskip('cftime') - cftime = _import_cftime() units = 'days since 0001-01-01' times = [datetime(1, 4, 1, h) for h in range(1, 5)] - noleap_time = cftime.date2num(times, units, calendar=calendar) - if enable_cftimeindex: - expected = cftime.num2date(noleap_time, units, calendar=calendar, + time = cftime.date2num(times, units, calendar=calendar) + + if cftime.__name__ == 'cftime': + expected = cftime.num2date(time, units, calendar=calendar, only_use_cftime_datetimes=True) else: - expected = cftime.num2date(noleap_time, units, calendar=calendar) + expected = cftime.num2date(time, units, calendar=calendar) + expected_date_type = type(expected[0]) with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime( - noleap_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + time, units, calendar=calendar) assert all(isinstance(value, expected_date_type) for value in actual) abs_diff = abs(actual - expected) # once we no longer support versions of netCDF4 older than 1.1.5, @@ -264,57 +246,37 @@ def test_decode_dates_outside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_STANDARD_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _STANDARD_CALENDARS) def test_decode_standard_calendar_single_element_inside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - + calendar): units = 'days since 0001-01-01' for num_time in [735368, [735368], [[735368]]]: with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime( - num_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + num_time, units, calendar=calendar) assert actual.dtype == np.dtype('M8[ns]') @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_NON_STANDARD_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _NON_STANDARD_CALENDARS) def test_decode_non_standard_calendar_single_element_inside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - + calendar): units = 'days since 0001-01-01' for num_time in [735368, [735368], [[735368]]]: with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime( - num_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) - if enable_cftimeindex: - assert actual.dtype == np.dtype('O') - else: - assert actual.dtype == np.dtype('M8[ns]') + num_time, units, calendar=calendar) + assert actual.dtype == np.dtype('O') @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_NON_STANDARD_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _NON_STANDARD_CALENDARS) def test_decode_single_element_outside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - + calendar): cftime = _import_cftime() units = 'days since 0001-01-01' for days in [1, 1470376]: @@ -323,40 +285,39 @@ def test_decode_single_element_outside_timestamp_range( warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime( - num_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) - expected = cftime.num2date(days, units, calendar) + num_time, units, calendar=calendar) + + if cftime.__name__ == 'cftime': + expected = cftime.num2date(days, units, calendar, + only_use_cftime_datetimes=True) + else: + expected = cftime.num2date(days, units, calendar) + assert isinstance(actual.item(), type(expected)) @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_STANDARD_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _STANDARD_CALENDARS) def test_decode_standard_calendar_multidim_time_inside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - + calendar): cftime = _import_cftime() units = 'days since 0001-01-01' times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D') times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D') - noleap_time1 = cftime.date2num(times1.to_pydatetime(), - units, calendar=calendar) - noleap_time2 = cftime.date2num(times2.to_pydatetime(), - units, calendar=calendar) - mdim_time = np.empty((len(noleap_time1), 2), ) - mdim_time[:, 0] = noleap_time1 - mdim_time[:, 1] = noleap_time2 + time1 = cftime.date2num(times1.to_pydatetime(), + units, calendar=calendar) + time2 = cftime.date2num(times2.to_pydatetime(), + units, calendar=calendar) + mdim_time = np.empty((len(time1), 2), ) + mdim_time[:, 0] = time1 + mdim_time[:, 1] = time2 expected1 = times1.values expected2 = times2.values actual = coding.times.decode_cf_datetime( - mdim_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + mdim_time, units, calendar=calendar) assert actual.dtype == np.dtype('M8[ns]') abs_diff1 = abs(actual[:, 0] - expected1) @@ -369,39 +330,35 @@ def test_decode_standard_calendar_multidim_time_inside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_NON_STANDARD_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _NON_STANDARD_CALENDARS) def test_decode_nonstandard_calendar_multidim_time_inside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - + calendar): cftime = _import_cftime() units = 'days since 0001-01-01' times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D') times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D') - noleap_time1 = cftime.date2num(times1.to_pydatetime(), - units, calendar=calendar) - noleap_time2 = cftime.date2num(times2.to_pydatetime(), - units, calendar=calendar) - mdim_time = np.empty((len(noleap_time1), 2), ) - mdim_time[:, 0] = noleap_time1 - mdim_time[:, 1] = noleap_time2 - - if enable_cftimeindex: - expected1 = cftime.num2date(noleap_time1, units, calendar) - expected2 = cftime.num2date(noleap_time2, units, calendar) - expected_dtype = np.dtype('O') + time1 = cftime.date2num(times1.to_pydatetime(), + units, calendar=calendar) + time2 = cftime.date2num(times2.to_pydatetime(), + units, calendar=calendar) + mdim_time = np.empty((len(time1), 2), ) + mdim_time[:, 0] = time1 + mdim_time[:, 1] = time2 + + if cftime.__name__ == 'cftime': + expected1 = cftime.num2date(time1, units, calendar, + only_use_cftime_datetimes=True) + expected2 = cftime.num2date(time2, units, calendar, + only_use_cftime_datetimes=True) else: - expected1 = times1.values - expected2 = times2.values - expected_dtype = np.dtype('M8[ns]') + expected1 = cftime.num2date(time1, units, calendar) + expected2 = cftime.num2date(time2, units, calendar) + + expected_dtype = np.dtype('O') actual = coding.times.decode_cf_datetime( - mdim_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + mdim_time, units, calendar=calendar) assert actual.dtype == expected_dtype abs_diff1 = abs(actual[:, 0] - expected1) @@ -414,41 +371,34 @@ def test_decode_nonstandard_calendar_multidim_time_inside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_ALL_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _ALL_CALENDARS) def test_decode_multidim_time_outside_timestamp_range( - calendar, enable_cftimeindex): + calendar): from datetime import datetime - - if enable_cftimeindex: - pytest.importorskip('cftime') - cftime = _import_cftime() units = 'days since 0001-01-01' times1 = [datetime(1, 4, day) for day in range(1, 6)] times2 = [datetime(1, 5, day) for day in range(1, 6)] - noleap_time1 = cftime.date2num(times1, units, calendar=calendar) - noleap_time2 = cftime.date2num(times2, units, calendar=calendar) - mdim_time = np.empty((len(noleap_time1), 2), ) - mdim_time[:, 0] = noleap_time1 - mdim_time[:, 1] = noleap_time2 - - if enable_cftimeindex: - expected1 = cftime.num2date(noleap_time1, units, calendar, + time1 = cftime.date2num(times1, units, calendar=calendar) + time2 = cftime.date2num(times2, units, calendar=calendar) + mdim_time = np.empty((len(time1), 2), ) + mdim_time[:, 0] = time1 + mdim_time[:, 1] = time2 + + if cftime.__name__ == 'cftime': + expected1 = cftime.num2date(time1, units, calendar, only_use_cftime_datetimes=True) - expected2 = cftime.num2date(noleap_time2, units, calendar, + expected2 = cftime.num2date(time2, units, calendar, only_use_cftime_datetimes=True) else: - expected1 = cftime.num2date(noleap_time1, units, calendar) - expected2 = cftime.num2date(noleap_time2, units, calendar) + expected1 = cftime.num2date(time1, units, calendar) + expected2 = cftime.num2date(time2, units, calendar) with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime( - mdim_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + mdim_time, units, calendar=calendar) assert actual.dtype == np.dtype('O') @@ -462,66 +412,51 @@ def test_decode_multidim_time_outside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(['360_day', 'all_leap', '366_day'], [False, True])) -def test_decode_non_standard_calendar_single_element_fallback( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - +@pytest.mark.parametrize('calendar', ['360_day', 'all_leap', '366_day']) +def test_decode_non_standard_calendar_single_element( + calendar): cftime = _import_cftime() - units = 'days since 0001-01-01' + try: dt = cftime.netcdftime.datetime(2001, 2, 29) except AttributeError: - # Must be using standalone netcdftime library + # Must be using the standalone cftime library dt = cftime.datetime(2001, 2, 29) num_time = cftime.date2num(dt, units, calendar) - if enable_cftimeindex: - actual = coding.times.decode_cf_datetime( - num_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) - else: - with pytest.warns(SerializationWarning, - match='Unable to decode time axis'): - actual = coding.times.decode_cf_datetime( - num_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + actual = coding.times.decode_cf_datetime( + num_time, units, calendar=calendar) - expected = np.asarray(cftime.num2date(num_time, units, calendar)) + if cftime.__name__ == 'cftime': + expected = np.asarray(cftime.num2date( + num_time, units, calendar, only_use_cftime_datetimes=True)) + else: + expected = np.asarray(cftime.num2date(num_time, units, calendar)) assert actual.dtype == np.dtype('O') assert expected == actual @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(['360_day'], [False, True])) -def test_decode_non_standard_calendar_fallback( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - +def test_decode_360_day_calendar(): cftime = _import_cftime() + calendar = '360_day' # ensure leap year doesn't matter for year in [2010, 2011, 2012, 2013, 2014]: units = 'days since {0}-01-01'.format(year) num_times = np.arange(100) - expected = cftime.num2date(num_times, units, calendar) + + if cftime.__name__ == 'cftime': + expected = cftime.num2date(num_times, units, calendar, + only_use_cftime_datetimes=True) + else: + expected = cftime.num2date(num_times, units, calendar) with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') actual = coding.times.decode_cf_datetime( - num_times, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) - if enable_cftimeindex: - assert len(w) == 0 - else: - assert len(w) == 1 - assert 'Unable to decode time axis' in str(w[0].message) + num_times, units, calendar=calendar) + assert len(w) == 0 assert actual.dtype == np.dtype('O') assert_array_equal(actual, expected) @@ -667,11 +602,8 @@ def test_format_cftime_datetime(date_args, expected): assert result == expected -@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_ALL_CALENDARS, [False, True])) -def test_decode_cf_enable_cftimeindex(calendar, enable_cftimeindex): +@pytest.mark.parametrize('calendar', _ALL_CALENDARS) +def test_decode_cf(calendar): days = [1., 2., 3.] da = DataArray(days, coords=[days], dims=['time'], name='test') ds = da.to_dataset() @@ -680,17 +612,13 @@ def test_decode_cf_enable_cftimeindex(calendar, enable_cftimeindex): ds[v].attrs['units'] = 'days since 2001-01-01' ds[v].attrs['calendar'] = calendar - if (not has_cftime and enable_cftimeindex and - calendar not in _STANDARD_CALENDARS): + if not has_cftime_or_netCDF4 and calendar not in _STANDARD_CALENDARS: with pytest.raises(ValueError): - with set_options(enable_cftimeindex=enable_cftimeindex): - ds = decode_cf(ds) - else: - with set_options(enable_cftimeindex=enable_cftimeindex): ds = decode_cf(ds) + else: + ds = decode_cf(ds) - if (enable_cftimeindex and - calendar not in _STANDARD_CALENDARS): + if calendar not in _STANDARD_CALENDARS: assert ds.test.dtype == np.dtype('O') else: assert ds.test.dtype == np.dtype('M8[ns]') @@ -764,7 +692,7 @@ def test_contains_cftime_datetimes_non_cftimes_dask(non_cftime_data): @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') @pytest.mark.parametrize('shape', [(24,), (8, 3), (2, 4, 3)]) -def test_encode_datetime_overflow(shape): +def test_encode_cf_datetime_overflow(shape): # Test for fix to GH 2272 dates = pd.date_range('2100', periods=24).values.reshape(shape) units = 'days since 1800-01-01' diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 433a669e340..abdbdbdbfdf 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -2279,12 +2279,9 @@ def test_resample_cftimeindex(self): cftime = _import_cftime() times = cftime.num2date(np.arange(12), units='hours since 0001-01-01', calendar='noleap') - with set_options(enable_cftimeindex=True): - array = DataArray(np.arange(12), [('time', times)]) + array = DataArray(np.arange(12), [('time', times)]) - with raises_regex(TypeError, - 'Only valid with DatetimeIndex, ' - 'TimedeltaIndex or PeriodIndex'): + with raises_regex(NotImplementedError, 'to_datetimeindex'): array.resample(time='6H').mean() def test_resample_first(self): diff --git a/xarray/tests/test_options.py b/xarray/tests/test_options.py index a21ea3e6b64..d594e1dcd18 100644 --- a/xarray/tests/test_options.py +++ b/xarray/tests/test_options.py @@ -33,8 +33,9 @@ def test_arithmetic_join(): def test_enable_cftimeindex(): with pytest.raises(ValueError): xarray.set_options(enable_cftimeindex=None) - with xarray.set_options(enable_cftimeindex=True): - assert OPTIONS['enable_cftimeindex'] + with pytest.warns(FutureWarning, match='no-op'): + with xarray.set_options(enable_cftimeindex=True): + assert OPTIONS['enable_cftimeindex'] def test_file_cache_maxsize(): diff --git a/xarray/tests/test_utils.py b/xarray/tests/test_utils.py index 33021fc5ef4..ed07af0d7bb 100644 --- a/xarray/tests/test_utils.py +++ b/xarray/tests/test_utils.py @@ -46,19 +46,17 @@ def test_safe_cast_to_index(): @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize('enable_cftimeindex', [False, True]) -def test_safe_cast_to_index_cftimeindex(enable_cftimeindex): +def test_safe_cast_to_index_cftimeindex(): date_types = _all_cftime_date_types() for date_type in date_types.values(): dates = [date_type(1, 1, day) for day in range(1, 20)] - if enable_cftimeindex and has_cftime: + if has_cftime: expected = CFTimeIndex(dates) else: expected = pd.Index(dates) - with set_options(enable_cftimeindex=enable_cftimeindex): - actual = utils.safe_cast_to_index(np.array(dates)) + actual = utils.safe_cast_to_index(np.array(dates)) assert_array_equal(expected, actual) assert expected.dtype == actual.dtype assert isinstance(actual, type(expected)) @@ -66,13 +64,11 @@ def test_safe_cast_to_index_cftimeindex(enable_cftimeindex): # Test that datetime.datetime objects are never used in a CFTimeIndex @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize('enable_cftimeindex', [False, True]) -def test_safe_cast_to_index_datetime_datetime(enable_cftimeindex): +def test_safe_cast_to_index_datetime_datetime(): dates = [datetime(1, 1, day) for day in range(1, 20)] expected = pd.Index(dates) - with set_options(enable_cftimeindex=enable_cftimeindex): - actual = utils.safe_cast_to_index(np.array(dates)) + actual = utils.safe_cast_to_index(np.array(dates)) assert_array_equal(expected, actual) assert isinstance(actual, pd.Index)