Merge pull request #126 from jhamman/cast_nc_datetime

Return numpy.datetime64 arrays for non-standard calendars

Author: shoyer

test/test_conventions.py

@@ -86,7 +86,9 @@ def test_cf_datetime(self):
             for calendar in ['standard', 'gregorian', 'proleptic_gregorian']:
                 expected = nc4.num2date(num_dates, units, calendar)
                 print(num_dates, units, calendar)
-                actual = conventions.decode_cf_datetime(num_dates, units, calendar)
+                with warnings.catch_warnings():
+                    warnings.filterwarnings('ignore', 'Unable to decode time axis')
+                    actual = conventions.decode_cf_datetime(num_dates, units, calendar)
                 if (isinstance(actual, np.ndarray)
                         and np.issubdtype(actual.dtype, np.datetime64)):
                     self.assertEqual(actual.dtype, np.dtype('M8[ns]'))
@@ -111,8 +113,6 @@ def test_cf_datetime(self):
 
     @requires_netCDF4
     def test_decoded_cf_datetime_array(self):
-        import netCDF4 as nc4
-
         actual = conventions.DecodedCFDatetimeArray(
             [0, 1, 2], 'days since 1900-01-01', 'standard')
         expected = pd.date_range('1900-01-01', periods=3).values
@@ -125,13 +125,103 @@ def test_decoded_cf_datetime_array(self):
         self.assertEqual(actual.dtype, np.dtype('datetime64[ns]'))
         self.assertArrayEqual(actual, expected)
 
-        num_dates = [722000, 720000.5]
-        units = 'days since 0001-01-01 0:0:0'
+    @requires_netCDF4
+    def test_decode_non_standard_calendar(self):
+        import netCDF4 as nc4
+
+        for calendar in ['noleap', '365_day', '360_day', 'julian', 'all_leap',
+                         '366_day']:
+            units = 'days since 0001-01-01'
+            times = pd.date_range('2001-04-01-00', end='2001-04-30-23',
+                                  freq='H')
+            noleap_time = nc4.date2num(times.to_pydatetime(), units,
+                                       calendar=calendar)
+            expected = times.values
+            with warnings.catch_warnings():
+                warnings.filterwarnings('ignore', 'Unable to decode time axis')
+                actual = conventions.decode_cf_datetime(noleap_time, units,
+                                                        calendar=calendar)
+            self.assertEqual(actual.dtype, np.dtype('M8[ns]'))
+            self.assertArrayEqual(actual, expected)
+
+    @requires_netCDF4
+    def test_decode_non_standard_calendar_single_element(self):
+        units = 'days since 0001-01-01'
+        for calendar in ['noleap', '365_day', '360_day', 'julian', 'all_leap',
+                         '366_day']:
+            for num_time in [735368, [735368], [[735368]]]:
+                with warnings.catch_warnings():
+                    warnings.filterwarnings('ignore', 'Unable to decode time axis')
+                    actual = conventions.decode_cf_datetime(num_time, units,
+                                                            calendar=calendar)
+                self.assertEqual(actual.dtype, np.dtype('M8[ns]'))
+
+    @requires_netCDF4
+    def test_decode_non_standard_calendar_single_element_fallback(self):
+        import netCDF4 as nc4
+
+        units = 'days since 0001-01-01'
+        dt = nc4.netcdftime.datetime(2001, 2, 29)
+        for calendar in ['360_day', 'all_leap', '366_day']:
+            num_time = nc4.date2num(dt, units, calendar)
+            with warnings.catch_warnings(record=True) as w:
+                warnings.simplefilter('always')
+                actual = conventions.decode_cf_datetime(num_time, units,
+                                                        calendar=calendar)
+                self.assertEqual(len(w), 1)
+                self.assertIn('Unable to decode time axis',
+                              str(w[0].message))
+            expected = np.asarray(nc4.num2date(num_time, units, calendar))
+            print(num_time, calendar, actual, expected)
+            self.assertEqual(actual.dtype, np.dtype('O'))
+            self.assertEqual(expected, actual)
+
+    @requires_netCDF4
+    def test_decode_non_standard_calendar_multidim_time(self):
+        import netCDF4 as nc4
+
         calendar = 'noleap'
-        actual = conventions.DecodedCFDatetimeArray(num_dates, units, calendar)
-        expected = nc4.num2date(num_dates, units, calendar)
-        self.assertEqual(actual.dtype, np.dtype('O'))
-        self.assertArrayEqual(actual, expected)
+        units = 'days since 0001-01-01'
+        times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D')
+        times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D')
+        noleap_time1 = nc4.date2num(times1.to_pydatetime(), units,
+                                    calendar=calendar)
+        noleap_time2 = nc4.date2num(times2.to_pydatetime(), units,
+                                    calendar=calendar)
+        mdim_time = np.empty((len(noleap_time1), 2), )
+        mdim_time[:, 0] = noleap_time1
+        mdim_time[:, 1] = noleap_time2
+
+        expected1 = times1.values
+        expected2 = times2.values
+        with warnings.catch_warnings():
+            warnings.filterwarnings('ignore', 'Unable to decode time axis')
+            actual = conventions.decode_cf_datetime(mdim_time, units,
+                                                    calendar=calendar)
+        self.assertEqual(actual.dtype, np.dtype('M8[ns]'))
+        self.assertArrayEqual(actual[:, 0], expected1)
+        self.assertArrayEqual(actual[:, 1], expected2)
+
+    @requires_netCDF4
+    def test_decode_non_standard_calendar_fallback(self):
+        import netCDF4 as nc4
+        for year in [2010, 2011, 2012, 2013, 2014]: # insure leap year doesn't matter
+            for calendar in ['360_day', '366_day', 'all_leap']:
+                calendar = '360_day'
+                units = 'days since {0}-01-01'.format(year)
+                num_times = np.arange(100)
+                expected = nc4.num2date(num_times, units, calendar)
+
+                with warnings.catch_warnings(record=True) as w:
+                    warnings.simplefilter('always')
+                    actual = conventions.decode_cf_datetime(num_times, units,
+                                                            calendar=calendar)
+                    self.assertEqual(len(w), 1)
+                    self.assertIn('Unable to decode time axis',
+                                  str(w[0].message))
+
+                self.assertEqual(actual.dtype, np.dtype('O'))
+                self.assertArrayEqual(actual, expected)
 
     @requires_netCDF4
     def test_cf_datetime_nan(self):

xray/conventions.py

@@ -1,5 +1,6 @@
 import numpy as np
 import pandas as pd
+import warnings
 from collections import defaultdict, OrderedDict
 from datetime import datetime
 
@@ -88,7 +89,25 @@ def nan_safe_num2date(num):
     if ((calendar not in _STANDARD_CALENDARS
             or min_date.year < 1678 or max_date.year >= 2262)
             and min_date is not pd.NaT):
+
         dates = nc4.num2date(num_dates, units, calendar)
+
+        if min_date.year >= 1678 and max_date.year < 2262:
+            try:
+                dates = nctime_to_nptime(dates)
+            except ValueError as e:
+                warnings.warn('Unable to decode time axis into full '
+                              'numpy.datetime64 objects, continuing using '
+                              'dummy netCDF4.datetime objects instead, reason:'
+                              '{0}'.format(e), RuntimeWarning, stacklevel=2)
+                dates = np.asarray(dates)
+        else:
+            warnings.warn('Unable to decode time axis into full '
+                          'numpy.datetime64 objects, continuing using dummy '
+                          'netCDF4.datetime objects instead, reason: dates out'
+                          ' of range', RuntimeWarning, stacklevel=2)
+            dates = np.asarray(dates)
+
     else:
         # we can safely use np.datetime64 with nanosecond precision (pandas
         # likes ns precision so it can directly make DatetimeIndex objects)
@@ -122,6 +141,7 @@ def nan_safe_num2date(num):
                      + np.datetime64(min_date))
         # restore original shape and ensure dates are given in ns
         dates = dates.reshape(num_dates.shape).astype('M8[ns]')
+
     return dates
 
 
@@ -144,6 +164,16 @@ def guess_time_units(dates):
     return '%s since %s' % (time_unit, dates[0])
 
 
+def nctime_to_nptime(times):
+    """Given an array of netCDF4.datetime objects, return an array of
+    numpy.datetime64 objects of the same size"""
+    times = np.asarray(times)
+    new = np.empty(times.shape, dtype='M8[ns]')
+    for i, t in np.ndenumerate(times):
+        new[i] = np.datetime64(datetime(*t.timetuple()[:6]))
+    return new
+
+
 def encode_cf_datetime(dates, units=None, calendar=None):
     """Given an array of datetime objects, returns the tuple `(num, units,
     calendar)` suitable for a CF complient time variable.
@@ -246,13 +276,7 @@ def __init__(self, array, units, calendar=None):
 
     @property
     def dtype(self):
-        if self.calendar is None or self.calendar in _STANDARD_CALENDARS:
-            # TODO: return the proper dtype (object) for a standard calendar
-            # that can't be expressed in ns precision. Perhaps we could guess
-            # this from the units?
-            return np.dtype('datetime64[ns]')
-        else:
-            return np.dtype('O')
+        return np.dtype('datetime64[ns]')
 
     def __getitem__(self, key):
         return decode_cf_datetime(self.array, units=self.units,