From 3926da5f7c120331ced6efdd5ed0dc95280e69db Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 28 Oct 2017 11:10:39 -0700 Subject: [PATCH 01/10] move NaT to self-contained module --- pandas/_libs/period.pyx | 5 +- pandas/_libs/src/inference.pyx | 2 +- pandas/_libs/tslib.pxd | 1 - pandas/_libs/tslib.pyx | 358 ++------------------------ pandas/_libs/tslibs/nattype.pxd | 6 + pandas/_libs/tslibs/nattype.pyx | 388 +++++++++++++++++++++++++++++ pandas/_libs/tslibs/strptime.pyx | 11 +- pandas/_libs/tslibs/timedeltas.pyx | 5 +- setup.py | 18 +- 9 files changed, 434 insertions(+), 360 deletions(-) create mode 100644 pandas/_libs/tslibs/nattype.pxd create mode 100644 pandas/_libs/tslibs/nattype.pyx diff --git a/pandas/_libs/period.pyx b/pandas/_libs/period.pyx index cf6ef91d3e608..6cc2563cb0588 100644 --- a/pandas/_libs/period.pyx +++ b/pandas/_libs/period.pyx @@ -33,10 +33,11 @@ from pandas._libs import tslib from pandas._libs.tslib import Timestamp, iNaT, NaT from tslibs.timezones cimport ( is_utc, is_tzlocal, get_utcoffset, get_dst_info, maybe_get_tz) -from tslib cimport _nat_scalar_rules from tslibs.parsing import parse_time_string, NAT_SENTINEL from tslibs.frequencies cimport get_freq_code +from tslibs.nattype import _nat_strings +from tslibs.nattype cimport _nat_scalar_rules from pandas.tseries import offsets from pandas.tseries import frequencies @@ -1185,7 +1186,7 @@ class Period(_Period): converted = other.asfreq(freq) ordinal = converted.ordinal - elif is_null_datetimelike(value) or value in tslib._nat_strings: + elif is_null_datetimelike(value) or value in _nat_strings: ordinal = iNaT elif is_string_object(value) or util.is_integer_object(value): diff --git a/pandas/_libs/src/inference.pyx b/pandas/_libs/src/inference.pyx index b0a64e1ccc225..ae2dce86ffe69 100644 --- a/pandas/_libs/src/inference.pyx +++ b/pandas/_libs/src/inference.pyx @@ -2,7 +2,7 @@ import sys from decimal import Decimal cimport util cimport cython -from tslib import NaT +from tslibs.nattype import NaT from tslibs.timezones cimport get_timezone from datetime import datetime, timedelta iNaT = util.get_nat() diff --git a/pandas/_libs/tslib.pxd b/pandas/_libs/tslib.pxd index 147320b108cc8..5ceff32cfbac7 100644 --- a/pandas/_libs/tslib.pxd +++ b/pandas/_libs/tslib.pxd @@ -2,7 +2,6 @@ from numpy cimport ndarray, int64_t cdef convert_to_tsobject(object, object, object, bint, bint) cpdef convert_to_timedelta64(object, object) -cdef bint _nat_scalar_rules[6] cdef bint _check_all_nulls(obj) cdef _to_i8(object val) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index 5269cddf8d2fd..e970b1dedce11 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -802,228 +802,33 @@ class Timestamp(_Timestamp): return self + other -_nat_strings = set(['NaT', 'nat', 'NAT', 'nan', 'NaN', 'NAN']) +# ---------------------------------------------------------------------- +# NaT Construction +# Here we patch the docstrings of NaT methods to match those of Timestamp +from tslibs.nattype import (NaT, NaTType, + _make_error_func, _make_nat_func, + _nat_strings) +from tslibs.nattype cimport _nat_scalar_rules, _checknull_with_nat -def _make_nat_func(func_name, cls): - def f(*args, **kwargs): - return NaT - f.__name__ = func_name - f.__doc__ = getattr(cls, func_name).__doc__ - return f +# We patch these NaTType methods with the Timestamp docstrings. +NaTType.astimezone = _make_error_func('astimezone', Timestamp) +NaTType.fromordinal = _make_error_func('fromordinal', Timestamp) +# _nat_methods +NaTType.to_pydatetime = _make_nat_func('to_pydatetime', Timestamp) -def _make_nan_func(func_name, cls): - def f(*args, **kwargs): - return np.nan - f.__name__ = func_name - f.__doc__ = getattr(cls, func_name).__doc__ - return f +NaTType.now = _make_nat_func('now', Timestamp) +NaTType.today = _make_nat_func('today', Timestamp) +NaTType.round = _make_nat_func('round', Timestamp) +NaTType.floor = _make_nat_func('floor', Timestamp) +NaTType.ceil = _make_nat_func('ceil', Timestamp) +NaTType.tz_convert = _make_nat_func('tz_convert', Timestamp) +NaTType.tz_localize = _make_nat_func('tz_localize', Timestamp) +NaTType.replace = _make_nat_func('replace', Timestamp) -def _make_error_func(func_name, cls): - def f(*args, **kwargs): - raise ValueError("NaTType does not support " + func_name) - - f.__name__ = func_name - if cls is not None: - f.__doc__ = getattr(cls, func_name).__doc__ - return f - - -class NaTType(_NaT): - """(N)ot-(A)-(T)ime, the time equivalent of NaN""" - - def __new__(cls): - cdef _NaT base - - base = _NaT.__new__(cls, 1, 1, 1) - base.value = NPY_NAT - base.freq = None - - return base - - def __repr__(self): - return 'NaT' - - def __str__(self): - return 'NaT' - - def isoformat(self, sep='T'): - # This allows Timestamp(ts.isoformat()) to always correctly roundtrip. - return 'NaT' - - def __hash__(self): - return NPY_NAT - - def __int__(self): - return NPY_NAT - - def __long__(self): - return NPY_NAT - - def __reduce_ex__(self, protocol): - # python 3.6 compat - # http://bugs.python.org/issue28730 - # now __reduce_ex__ is defined and higher priority than __reduce__ - return self.__reduce__() - - def __reduce__(self): - return (__nat_unpickle, (None, )) - - def total_seconds(self): - """ - Total duration of timedelta in seconds (to ns precision) - """ - # GH 10939 - return np.nan - - @property - def is_leap_year(self): - return False - - @property - def is_month_start(self): - return False - - @property - def is_quarter_start(self): - return False - - @property - def is_year_start(self): - return False - - @property - def is_month_end(self): - return False - - @property - def is_quarter_end(self): - return False - - @property - def is_year_end(self): - return False - - def __rdiv__(self, other): - return _nat_rdivide_op(self, other) - - def __rtruediv__(self, other): - return _nat_rdivide_op(self, other) - - def __rfloordiv__(self, other): - return _nat_rdivide_op(self, other) - - def __rmul__(self, other): - if is_integer_object(other) or is_float_object(other): - return NaT - return NotImplemented - - # ---------------------------------------------------------------------- - # inject the Timestamp field properties - # these by definition return np.nan - - year = property(fget=lambda self: np.nan) - quarter = property(fget=lambda self: np.nan) - month = property(fget=lambda self: np.nan) - day = property(fget=lambda self: np.nan) - hour = property(fget=lambda self: np.nan) - minute = property(fget=lambda self: np.nan) - second = property(fget=lambda self: np.nan) - millisecond = property(fget=lambda self: np.nan) - microsecond = property(fget=lambda self: np.nan) - nanosecond = property(fget=lambda self: np.nan) - - week = property(fget=lambda self: np.nan) - dayofyear = property(fget=lambda self: np.nan) - weekofyear = property(fget=lambda self: np.nan) - days_in_month = property(fget=lambda self: np.nan) - daysinmonth = property(fget=lambda self: np.nan) - dayofweek = property(fget=lambda self: np.nan) - weekday_name = property(fget=lambda self: np.nan) - - # inject Timedelta properties - days = property(fget=lambda self: np.nan) - seconds = property(fget=lambda self: np.nan) - microseconds = property(fget=lambda self: np.nan) - nanoseconds = property(fget=lambda self: np.nan) - - # inject pd.Period properties - qyear = property(fget=lambda self: np.nan) - - # ---------------------------------------------------------------------- - # GH9513 NaT methods (except to_datetime64) to raise, return np.nan, or - # return NaT create functions that raise, for binding to NaTType - # These are the ones that can get their docstrings from datetime. - - # nan methods - weekday = _make_nan_func('weekday', datetime) - isoweekday = _make_nan_func('isoweekday', datetime) - - # _nat_methods - date = _make_nat_func('date', datetime) - - utctimetuple = _make_error_func('utctimetuple', datetime) - timetz = _make_error_func('timetz', datetime) - timetuple = _make_error_func('timetuple', datetime) - strptime = _make_error_func('strptime', datetime) - strftime = _make_error_func('strftime', datetime) - isocalendar = _make_error_func('isocalendar', datetime) - dst = _make_error_func('dst', datetime) - ctime = _make_error_func('ctime', datetime) - time = _make_error_func('time', datetime) - toordinal = _make_error_func('toordinal', datetime) - tzname = _make_error_func('tzname', datetime) - utcoffset = _make_error_func('utcoffset', datetime) - - # Timestamp has empty docstring for some methods. - utcfromtimestamp = _make_error_func('utcfromtimestamp', None) - fromtimestamp = _make_error_func('fromtimestamp', None) - combine = _make_error_func('combine', None) - utcnow = _make_error_func('utcnow', None) - - timestamp = _make_error_func('timestamp', Timestamp) - - # GH9513 NaT methods (except to_datetime64) to raise, return np.nan, or - # return NaT create functions that raise, for binding to NaTType - astimezone = _make_error_func('astimezone', Timestamp) - fromordinal = _make_error_func('fromordinal', Timestamp) - - # _nat_methods - to_pydatetime = _make_nat_func('to_pydatetime', Timestamp) - - now = _make_nat_func('now', Timestamp) - today = _make_nat_func('today', Timestamp) - round = _make_nat_func('round', Timestamp) - floor = _make_nat_func('floor', Timestamp) - ceil = _make_nat_func('ceil', Timestamp) - - tz_convert = _make_nat_func('tz_convert', Timestamp) - tz_localize = _make_nat_func('tz_localize', Timestamp) - replace = _make_nat_func('replace', Timestamp) - - def to_datetime(self): - """ - DEPRECATED: use :meth:`to_pydatetime` instead. - - Convert a Timestamp object to a native Python datetime object. - """ - warnings.warn("to_datetime is deprecated. Use self.to_pydatetime()", - FutureWarning, stacklevel=2) - return self.to_pydatetime(warn=False) - - -def __nat_unpickle(*args): - # return constant defined in the module - return NaT - -NaT = NaTType() - -cdef inline bint _checknull_with_nat(object val): - """ utility to check if a value is a nat or not """ - return val is None or ( - PyFloat_Check(val) and val != val) or val is NaT +# ---------------------------------------------------------------------- cdef inline bint _check_all_nulls(object val): """ utility to check if a value is any type of null """ @@ -1042,8 +847,6 @@ cdef inline bint _check_all_nulls(object val): res = 0 return res -cdef inline bint _cmp_nat_dt(_NaT lhs, _Timestamp rhs, int op) except -1: - return _nat_scalar_rules[op] cpdef object get_value_box(ndarray arr, object loc): @@ -1161,7 +964,7 @@ cdef class _Timestamp(datetime): if isinstance(other, _Timestamp): ots = other elif other is NaT: - return _cmp_nat_dt(other, self, _reverse_ops[op]) + return op == Py_NE elif PyDateTime_Check(other): if self.nanosecond == 0: val = self.to_pydatetime() @@ -1453,123 +1256,6 @@ cdef inline bint is_timestamp(object o): return Py_TYPE(o) == ts_type # isinstance(o, Timestamp) -cdef bint _nat_scalar_rules[6] - -_nat_scalar_rules[Py_EQ] = False -_nat_scalar_rules[Py_NE] = True -_nat_scalar_rules[Py_LT] = False -_nat_scalar_rules[Py_LE] = False -_nat_scalar_rules[Py_GT] = False -_nat_scalar_rules[Py_GE] = False - - -cdef _nat_divide_op(self, other): - if PyDelta_Check(other) or is_timedelta64_object(other) or other is NaT: - return np.nan - if is_integer_object(other) or is_float_object(other): - return NaT - return NotImplemented - -cdef _nat_rdivide_op(self, other): - if PyDelta_Check(other): - return np.nan - return NotImplemented - - -cdef class _NaT(datetime): - cdef readonly: - int64_t value - object freq - - def __hash__(_NaT self): - # py3k needs this defined here - return hash(self.value) - - def __richcmp__(_NaT self, object other, int op): - cdef int ndim = getattr(other, 'ndim', -1) - - if ndim == -1: - return _nat_scalar_rules[op] - - if ndim == 0: - if is_datetime64_object(other): - return _nat_scalar_rules[op] - else: - raise TypeError('Cannot compare type %r with type %r' % - (type(self).__name__, type(other).__name__)) - return PyObject_RichCompare(other, self, _reverse_ops[op]) - - def __add__(self, other): - if PyDateTime_Check(other): - return NaT - - elif hasattr(other, 'delta'): - # Timedelta, offsets.Tick, offsets.Week - return NaT - elif getattr(other, '_typ', None) in ['dateoffset', 'series', - 'period', 'datetimeindex', - 'timedeltaindex']: - # Duplicate logic in _Timestamp.__add__ to avoid needing - # to subclass; allows us to @final(_Timestamp.__add__) - return NotImplemented - return NaT - - def __sub__(self, other): - # Duplicate some logic from _Timestamp.__sub__ to avoid needing - # to subclass; allows us to @final(_Timestamp.__sub__) - if PyDateTime_Check(other): - return NaT - elif PyDelta_Check(other): - return NaT - - elif getattr(other, '_typ', None) == 'datetimeindex': - # a Timestamp-DatetimeIndex -> yields a negative TimedeltaIndex - return -other.__sub__(self) - - elif getattr(other, '_typ', None) == 'timedeltaindex': - # a Timestamp-TimedeltaIndex -> yields a negative TimedeltaIndex - return (-other).__add__(self) - - elif hasattr(other, 'delta'): - # offsets.Tick, offsets.Week - neg_other = -other - return self + neg_other - - elif getattr(other, '_typ', None) in ['period', - 'periodindex', 'dateoffset']: - return NotImplemented - - return NaT - - def __pos__(self): - return NaT - - def __neg__(self): - return NaT - - def __div__(self, other): - return _nat_divide_op(self, other) - - def __truediv__(self, other): - return _nat_divide_op(self, other) - - def __floordiv__(self, other): - return _nat_divide_op(self, other) - - def __mul__(self, other): - if is_integer_object(other) or is_float_object(other): - return NaT - return NotImplemented - - @property - def asm8(self): - return np.datetime64(NPY_NAT, 'ns') - - def to_datetime64(self): - """ Returns a numpy.datetime64 object with 'ns' precision """ - return np.datetime64('NaT') - - # lightweight C object to hold datetime & int64 pair cdef class _TSObject: cdef: diff --git a/pandas/_libs/tslibs/nattype.pxd b/pandas/_libs/tslibs/nattype.pxd new file mode 100644 index 0000000000000..7ded36bb1bdc0 --- /dev/null +++ b/pandas/_libs/tslibs/nattype.pxd @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# cython: profile=False + +cdef bint _nat_scalar_rules[6] + +cdef bint _checknull_with_nat(object val) diff --git a/pandas/_libs/tslibs/nattype.pyx b/pandas/_libs/tslibs/nattype.pyx new file mode 100644 index 0000000000000..336d2ffd30d59 --- /dev/null +++ b/pandas/_libs/tslibs/nattype.pyx @@ -0,0 +1,388 @@ +# -*- coding: utf-8 -*- +# cython: profile=False +import warnings + +from cpython cimport ( + PyFloat_Check, PyComplex_Check, + PyObject_RichCompare, + Py_GT, Py_GE, Py_EQ, Py_NE, Py_LT, Py_LE) + +from cpython.datetime cimport (datetime, + PyDateTime_Check, PyDelta_Check, + PyDateTime_IMPORT) +PyDateTime_IMPORT + +import numpy as np +cimport numpy as np +from numpy cimport int64_t +np.import_array() + +from util cimport (get_nat, + is_integer_object, is_float_object, + is_datetime64_object, is_timedelta64_object) + +# ---------------------------------------------------------------------- +# Constants +_nat_strings = set(['NaT', 'nat', 'NAT', 'nan', 'NaN', 'NAN']) + +cdef int64_t NPY_NAT = get_nat() + +cdef bint _nat_scalar_rules[6] +_nat_scalar_rules[Py_EQ] = False +_nat_scalar_rules[Py_NE] = True +_nat_scalar_rules[Py_LT] = False +_nat_scalar_rules[Py_LE] = False +_nat_scalar_rules[Py_GT] = False +_nat_scalar_rules[Py_GE] = False + +# ---------------------------------------------------------------------- + + +def _make_nan_func(func_name, cls): + def f(*args, **kwargs): + return np.nan + f.__name__ = func_name + f.__doc__ = getattr(cls, func_name).__doc__ + return f + + +def _make_nat_func(func_name, cls): + def f(*args, **kwargs): + return NaT + + f.__name__ = func_name + if isinstance(cls, str): + # passed the literal docstring directly + f.__doc__ = cls + else: + f.__doc__ = getattr(cls, func_name).__doc__ + return f + + +def _make_error_func(func_name, cls): + def f(*args, **kwargs): + raise ValueError("NaTType does not support " + func_name) + + f.__name__ = func_name + if isinstance(cls, str): + # passed the literal docstring directly + f.__doc__ = cls + elif cls is not None: + f.__doc__ = getattr(cls, func_name).__doc__ + return f + + +cdef _nat_divide_op(self, other): + if PyDelta_Check(other) or is_timedelta64_object(other) or other is NaT: + return np.nan + if is_integer_object(other) or is_float_object(other): + return NaT + return NotImplemented + + +cdef _nat_rdivide_op(self, other): + if PyDelta_Check(other): + return np.nan + return NotImplemented + + +def __nat_unpickle(*args): + # return constant defined in the module + return NaT + +# ---------------------------------------------------------------------- + + +cdef class _NaT(datetime): + cdef readonly: + int64_t value + object freq + + def __hash__(_NaT self): + # py3k needs this defined here + return hash(self.value) + + def __richcmp__(_NaT self, object other, int op): + cdef int ndim = getattr(other, 'ndim', -1) + + if ndim == -1: + return _nat_scalar_rules[op] + + if ndim == 0: + if is_datetime64_object(other): + return _nat_scalar_rules[op] + else: + raise TypeError('Cannot compare type %r with type %r' % + (type(self).__name__, type(other).__name__)) + # Note: instead of passing "other, self, _reverse_ops[op]", we observe + # that `_nat_scalar_rules` is invariant under `_reverse_ops`, + # rendering it unnecessary. + return PyObject_RichCompare(other, self, op) + + def __add__(self, other): + if PyDateTime_Check(other): + return NaT + + elif hasattr(other, 'delta'): + # Timedelta, offsets.Tick, offsets.Week + return NaT + elif getattr(other, '_typ', None) in ['dateoffset', 'series', + 'period', 'datetimeindex', + 'timedeltaindex']: + # Duplicate logic in _Timestamp.__add__ to avoid needing + # to subclass; allows us to @final(_Timestamp.__add__) + return NotImplemented + return NaT + + def __sub__(self, other): + # Duplicate some logic from _Timestamp.__sub__ to avoid needing + # to subclass; allows us to @final(_Timestamp.__sub__) + if PyDateTime_Check(other): + return NaT + elif PyDelta_Check(other): + return NaT + + elif getattr(other, '_typ', None) == 'datetimeindex': + # a Timestamp-DatetimeIndex -> yields a negative TimedeltaIndex + return -other.__sub__(self) + + elif getattr(other, '_typ', None) == 'timedeltaindex': + # a Timestamp-TimedeltaIndex -> yields a negative TimedeltaIndex + return (-other).__add__(self) + + elif hasattr(other, 'delta'): + # offsets.Tick, offsets.Week + neg_other = -other + return self + neg_other + + elif getattr(other, '_typ', None) in ['period', + 'periodindex', 'dateoffset']: + return NotImplemented + + return NaT + + def __pos__(self): + return NaT + + def __neg__(self): + return NaT + + def __div__(self, other): + return _nat_divide_op(self, other) + + def __truediv__(self, other): + return _nat_divide_op(self, other) + + def __floordiv__(self, other): + return _nat_divide_op(self, other) + + def __mul__(self, other): + if is_integer_object(other) or is_float_object(other): + return NaT + return NotImplemented + + @property + def asm8(self): + return np.datetime64(NPY_NAT, 'ns') + + def to_datetime64(self): + """ Returns a numpy.datetime64 object with 'ns' precision """ + return np.datetime64('NaT') + + +class NaTType(_NaT): + """(N)ot-(A)-(T)ime, the time equivalent of NaN""" + + def __new__(cls): + cdef _NaT base + + base = _NaT.__new__(cls, 1, 1, 1) + base.value = NPY_NAT + base.freq = None + + return base + + def __repr__(self): + return 'NaT' + + def __str__(self): + return 'NaT' + + def isoformat(self, sep='T'): + # This allows Timestamp(ts.isoformat()) to always correctly roundtrip. + return 'NaT' + + def __hash__(self): + return NPY_NAT + + def __int__(self): + return NPY_NAT + + def __long__(self): + return NPY_NAT + + def __reduce_ex__(self, protocol): + # python 3.6 compat + # http://bugs.python.org/issue28730 + # now __reduce_ex__ is defined and higher priority than __reduce__ + return self.__reduce__() + + def __reduce__(self): + return (__nat_unpickle, (None, )) + + def total_seconds(self): + """ + Total duration of timedelta in seconds (to ns precision) + """ + # GH 10939 + return np.nan + + @property + def is_leap_year(self): + return False + + @property + def is_month_start(self): + return False + + @property + def is_quarter_start(self): + return False + + @property + def is_year_start(self): + return False + + @property + def is_month_end(self): + return False + + @property + def is_quarter_end(self): + return False + + @property + def is_year_end(self): + return False + + def __rdiv__(self, other): + return _nat_rdivide_op(self, other) + + def __rtruediv__(self, other): + return _nat_rdivide_op(self, other) + + def __rfloordiv__(self, other): + return _nat_rdivide_op(self, other) + + def __rmul__(self, other): + if is_integer_object(other) or is_float_object(other): + return NaT + return NotImplemented + + # ---------------------------------------------------------------------- + # inject the Timestamp field properties + # these by definition return np.nan + + year = property(fget=lambda self: np.nan) + quarter = property(fget=lambda self: np.nan) + month = property(fget=lambda self: np.nan) + day = property(fget=lambda self: np.nan) + hour = property(fget=lambda self: np.nan) + minute = property(fget=lambda self: np.nan) + second = property(fget=lambda self: np.nan) + millisecond = property(fget=lambda self: np.nan) + microsecond = property(fget=lambda self: np.nan) + nanosecond = property(fget=lambda self: np.nan) + + week = property(fget=lambda self: np.nan) + dayofyear = property(fget=lambda self: np.nan) + weekofyear = property(fget=lambda self: np.nan) + days_in_month = property(fget=lambda self: np.nan) + daysinmonth = property(fget=lambda self: np.nan) + dayofweek = property(fget=lambda self: np.nan) + weekday_name = property(fget=lambda self: np.nan) + + # inject Timedelta properties + days = property(fget=lambda self: np.nan) + seconds = property(fget=lambda self: np.nan) + microseconds = property(fget=lambda self: np.nan) + nanoseconds = property(fget=lambda self: np.nan) + + # inject pd.Period properties + qyear = property(fget=lambda self: np.nan) + + # ---------------------------------------------------------------------- + # GH9513 NaT methods (except to_datetime64) to raise, return np.nan, or + # return NaT create functions that raise, for binding to NaTType + # These are the ones that can get their docstrings from datetime. + + # nan methods + weekday = _make_nan_func('weekday', datetime) + isoweekday = _make_nan_func('isoweekday', datetime) + + # _nat_methods + date = _make_nat_func('date', datetime) + + utctimetuple = _make_error_func('utctimetuple', datetime) + timetz = _make_error_func('timetz', datetime) + timetuple = _make_error_func('timetuple', datetime) + strptime = _make_error_func('strptime', datetime) + strftime = _make_error_func('strftime', datetime) + isocalendar = _make_error_func('isocalendar', datetime) + dst = _make_error_func('dst', datetime) + ctime = _make_error_func('ctime', datetime) + time = _make_error_func('time', datetime) + toordinal = _make_error_func('toordinal', datetime) + tzname = _make_error_func('tzname', datetime) + utcoffset = _make_error_func('utcoffset', datetime) + + # Timestamp has empty docstring for some methods. + utcfromtimestamp = _make_error_func('utcfromtimestamp', None) + fromtimestamp = _make_error_func('fromtimestamp', None) + combine = _make_error_func('combine', None) + utcnow = _make_error_func('utcnow', None) + + # ---------------------------------------------------------------------- + # The remaining methods are created with empty docstrings that will + # be patched with the `Timestamp` versions once those are imported. + + timestamp = _make_error_func('timestamp', '') + + # GH9513 NaT methods (except to_datetime64) to raise, return np.nan, or + # return NaT create functions that raise, for binding to NaTType + astimezone = _make_error_func('astimezone', '') + fromordinal = _make_error_func('fromordinal', '') + + # _nat_methods + to_pydatetime = _make_nat_func('to_pydatetime', '') + + now = _make_nat_func('now', '') + today = _make_nat_func('today', '') + round = _make_nat_func('round', '') + floor = _make_nat_func('floor', '') + ceil = _make_nat_func('ceil', '') + + tz_convert = _make_nat_func('tz_convert', '') + tz_localize = _make_nat_func('tz_localize', '') + replace = _make_nat_func('replace', '') + + def to_datetime(self): + """ + DEPRECATED: use :meth:`to_pydatetime` instead. + + Convert a Timestamp object to a native Python datetime object. + """ + warnings.warn("to_datetime is deprecated. Use self.to_pydatetime()", + FutureWarning, stacklevel=2) + return self.to_pydatetime(warn=False) + + +NaT = NaTType() + + +# ---------------------------------------------------------------------- + +cdef inline bint _checknull_with_nat(object val): + """ utility to check if a value is a nat or not """ + return val is None or ( + PyFloat_Check(val) and val != val) or val is NaT diff --git a/pandas/_libs/tslibs/strptime.pyx b/pandas/_libs/tslibs/strptime.pyx index 59a7376280da0..de1731331cc36 100644 --- a/pandas/_libs/tslibs/strptime.pyx +++ b/pandas/_libs/tslibs/strptime.pyx @@ -44,15 +44,8 @@ from util cimport is_string_object, get_nat cdef int64_t NPY_NAT = get_nat() -cdef set _nat_strings = set(['NaT', 'nat', 'NAT', 'nan', 'NaN', 'NAN']) - - -# TODO: Consolidate with other implementations -cdef inline bint _checknull_with_nat(object val): - """ utility to check if a value is a nat or not """ - return (val is None or - (PyFloat_Check(val) and val != val) or - (isinstance(val, datetime) and not val == val)) +from nattype cimport _checknull_with_nat +from nattype import _nat_strings def array_strptime(ndarray[object] values, object fmt, diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index 1785c85da4949..03377d21ed484 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -9,12 +9,11 @@ from numpy cimport int64_t cimport util +from nattype import _nat_strings + # ---------------------------------------------------------------------- # Constants -# TODO: Get this from tslibs.nattype once available -_nat_strings = set(['NaT', 'nat', 'NAT', 'nan', 'NaN', 'NAN']) - cdef int64_t NPY_NAT = util.get_nat() cdef dict timedelta_abbrevs = { 'D': 'd', diff --git a/setup.py b/setup.py index 5b9b13ee97acf..d608520ac55e6 100755 --- a/setup.py +++ b/setup.py @@ -472,8 +472,8 @@ def pxd(name): tseries_depends = ['pandas/_libs/src/datetime/np_datetime.h', 'pandas/_libs/src/datetime/np_datetime_strings.h', 'pandas/_libs/src/datetime.pxd'] -npdt_srces = ['pandas/_libs/src/datetime/np_datetime.c', - 'pandas/_libs/src/datetime/np_datetime_strings.c'] +np_datetime_sources = ['pandas/_libs/src/datetime/np_datetime.c', + 'pandas/_libs/src/datetime/np_datetime_strings.c'] # some linux distros require it libraries = ['m'] if not is_platform_windows() else [] @@ -488,27 +488,29 @@ def pxd(name): _pxi_dep['hashtable'])}, '_libs.tslibs.strptime': {'pyxfile': '_libs/tslibs/strptime', 'depends': tseries_depends, - 'sources': npdt_srces}, + 'sources': np_datetime_sources}, '_libs.tslib': {'pyxfile': '_libs/tslib', 'pxdfiles': ['_libs/src/util', '_libs/lib'], 'depends': tseries_depends, - 'sources': npdt_srces}, + 'sources': np_datetime_sources}, '_libs.tslibs.timedeltas': {'pyxfile': '_libs/tslibs/timedeltas'}, '_libs.tslibs.timezones': {'pyxfile': '_libs/tslibs/timezones'}, '_libs.tslibs.fields': {'pyxfile': '_libs/tslibs/fields', 'depends': tseries_depends, - 'sources': npdt_srces}, + 'sources': np_datetime_sources}, '_libs.period': {'pyxfile': '_libs/period', 'depends': (tseries_depends + ['pandas/_libs/src/period_helper.h']), - 'sources': npdt_srces + [ + 'sources': np_datetime_sources + [ 'pandas/_libs/src/period_helper.c']}, '_libs.tslibs.parsing': {'pyxfile': '_libs/tslibs/parsing', 'pxdfiles': ['_libs/src/util']}, '_libs.tslibs.frequencies': {'pyxfile': '_libs/tslibs/frequencies', 'pxdfiles': ['_libs/src/util']}, + '_libs.tslibs.nattype': {'pyxfile': '_libs/tslibs/nattype', + 'pxdfiles': ['_libs/src/util']}, '_libs.index': {'pyxfile': '_libs/index', - 'sources': npdt_srces, + 'sources': np_datetime_sources, 'pxdfiles': ['_libs/src/util', '_libs/hashtable'], 'depends': _pxi_dep['index']}, '_libs.algos': {'pyxfile': '_libs/algos', @@ -621,7 +623,7 @@ def pxd(name): 'pandas/_libs/src/ujson/python/JSONtoObj.c', 'pandas/_libs/src/ujson/lib/ultrajsonenc.c', 'pandas/_libs/src/ujson/lib/ultrajsondec.c'] + - npdt_srces), + np_datetime_sources), include_dirs=(['pandas/_libs/src/ujson/python', 'pandas/_libs/src/ujson/lib', 'pandas/_libs/src/datetime'] + From c23284555f44dcb2745d9a6db6e8f8f6f1170a3c Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 28 Oct 2017 13:22:50 -0700 Subject: [PATCH 02/10] de-privatize _nat_strings, copy/paste docstrings instead of patching --- pandas/_libs/period.pyx | 4 +- pandas/_libs/tslib.pyx | 39 ++---- pandas/_libs/tslibs/nattype.pyx | 188 ++++++++++++++++++++++++++--- pandas/_libs/tslibs/strptime.pyx | 4 +- pandas/_libs/tslibs/timedeltas.pyx | 4 +- pandas/core/tools/datetimes.py | 2 +- 6 files changed, 188 insertions(+), 53 deletions(-) diff --git a/pandas/_libs/period.pyx b/pandas/_libs/period.pyx index 6cc2563cb0588..ee90f469de771 100644 --- a/pandas/_libs/period.pyx +++ b/pandas/_libs/period.pyx @@ -36,7 +36,7 @@ from tslibs.timezones cimport ( from tslibs.parsing import parse_time_string, NAT_SENTINEL from tslibs.frequencies cimport get_freq_code -from tslibs.nattype import _nat_strings +from tslibs.nattype import nat_strings from tslibs.nattype cimport _nat_scalar_rules from pandas.tseries import offsets @@ -1186,7 +1186,7 @@ class Period(_Period): converted = other.asfreq(freq) ordinal = converted.ordinal - elif is_null_datetimelike(value) or value in _nat_strings: + elif is_null_datetimelike(value) or value in nat_strings: ordinal = iNaT elif is_string_object(value) or util.is_integer_object(value): diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index e970b1dedce11..68990c2d1ad82 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -94,6 +94,9 @@ from tslibs.fields import ( get_date_name_field, get_start_end_field, get_date_field, build_field_sarray) +from tslibs.nattype import NaT, nat_strings +from tslibs.nattype cimport _checknull_with_nat + cdef inline object create_timestamp_from_ts( int64_t value, pandas_datetimestruct dts, @@ -802,32 +805,6 @@ class Timestamp(_Timestamp): return self + other - -# ---------------------------------------------------------------------- -# NaT Construction -# Here we patch the docstrings of NaT methods to match those of Timestamp -from tslibs.nattype import (NaT, NaTType, - _make_error_func, _make_nat_func, - _nat_strings) -from tslibs.nattype cimport _nat_scalar_rules, _checknull_with_nat - -# We patch these NaTType methods with the Timestamp docstrings. -NaTType.astimezone = _make_error_func('astimezone', Timestamp) -NaTType.fromordinal = _make_error_func('fromordinal', Timestamp) - -# _nat_methods -NaTType.to_pydatetime = _make_nat_func('to_pydatetime', Timestamp) - -NaTType.now = _make_nat_func('now', Timestamp) -NaTType.today = _make_nat_func('today', Timestamp) -NaTType.round = _make_nat_func('round', Timestamp) -NaTType.floor = _make_nat_func('floor', Timestamp) -NaTType.ceil = _make_nat_func('ceil', Timestamp) - -NaTType.tz_convert = _make_nat_func('tz_convert', Timestamp) -NaTType.tz_localize = _make_nat_func('tz_localize', Timestamp) -NaTType.replace = _make_nat_func('replace', Timestamp) - # ---------------------------------------------------------------------- cdef inline bint _check_all_nulls(object val): @@ -1432,7 +1409,7 @@ cpdef convert_str_to_tsobject(object ts, object tz, object unit, assert util.is_string_object(ts) - if len(ts) == 0 or ts in _nat_strings: + if len(ts) == 0 or ts in nat_strings: ts = NaT elif ts == 'now': # Issue 9000, we short-circuit rather than going @@ -1829,7 +1806,7 @@ cpdef array_with_unit_to_datetime(ndarray values, unit, errors='coerce'): iresult[i] = NPY_NAT elif util.is_string_object(val): - if len(val) == 0 or val in _nat_strings: + if len(val) == 0 or val in nat_strings: iresult[i] = NPY_NAT else: @@ -1890,7 +1867,7 @@ cpdef array_with_unit_to_datetime(ndarray values, unit, errors='coerce'): oresult[i] = val elif util.is_string_object(val): - if len(val) == 0 or val in _nat_strings: + if len(val) == 0 or val in nat_strings: oresult[i] = NaT else: @@ -2008,7 +1985,7 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise', # string try: - if len(val) == 0 or val in _nat_strings: + if len(val) == 0 or val in nat_strings: iresult[i] = NPY_NAT continue @@ -2111,7 +2088,7 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise', oresult[i] = val elif util.is_string_object(val): - if len(val) == 0 or val in _nat_strings: + if len(val) == 0 or val in nat_strings: oresult[i] = 'NaT' continue diff --git a/pandas/_libs/tslibs/nattype.pyx b/pandas/_libs/tslibs/nattype.pyx index 336d2ffd30d59..54861e2520785 100644 --- a/pandas/_libs/tslibs/nattype.pyx +++ b/pandas/_libs/tslibs/nattype.pyx @@ -23,7 +23,7 @@ from util cimport (get_nat, # ---------------------------------------------------------------------- # Constants -_nat_strings = set(['NaT', 'nat', 'NAT', 'nan', 'NaN', 'NAN']) +nat_strings = set(['NaT', 'nat', 'NAT', 'nan', 'NaN', 'NAN']) cdef int64_t NPY_NAT = get_nat() @@ -343,28 +343,186 @@ class NaTType(_NaT): utcnow = _make_error_func('utcnow', None) # ---------------------------------------------------------------------- - # The remaining methods are created with empty docstrings that will - # be patched with the `Timestamp` versions once those are imported. + # The remaining methods have docstrings copy/pasted from the analogous + # Timestamp methods. - timestamp = _make_error_func('timestamp', '') + timestamp = _make_error_func('timestamp', + """Return POSIX timestamp as float.""") # GH9513 NaT methods (except to_datetime64) to raise, return np.nan, or # return NaT create functions that raise, for binding to NaTType - astimezone = _make_error_func('astimezone', '') - fromordinal = _make_error_func('fromordinal', '') + astimezone = _make_error_func('astimezone', + """ + Convert tz-aware Timestamp to another time zone. + + Parameters + ---------- + tz : string, pytz.timezone, dateutil.tz.tzfile or None + Time zone for time which Timestamp will be converted to. + None will remove timezone holding UTC time. + + Returns + ------- + converted : Timestamp + + Raises + ------ + TypeError + If Timestamp is tz-naive. + """) + fromordinal = _make_error_func('fromordinal', + """ + passed an ordinal, translate and convert to a ts + note: by definition there cannot be any tz info on the ordinal itself + + Parameters + ---------- + ordinal : int + date corresponding to a proleptic Gregorian ordinal + freq : str, DateOffset + Offset which Timestamp will have + tz : string, pytz.timezone, dateutil.tz.tzfile or None + Time zone for time which Timestamp will have. + offset : str, DateOffset + Deprecated, use freq + """) # _nat_methods - to_pydatetime = _make_nat_func('to_pydatetime', '') + to_pydatetime = _make_nat_func('to_pydatetime', + """ + Convert a Timestamp object to a native Python datetime object. + + If warn=True, issue a warning if nanoseconds is nonzero. + """) + + now = _make_nat_func('now', + """ + Return the current time in the local timezone. Equivalent + to datetime.now([tz]) + + Parameters + ---------- + tz : string / timezone object, default None + Timezone to localize to + """) + today = _make_nat_func('today', + """ + Return the current time in the local timezone. This differs + from datetime.today() in that it can be localized to a + passed timezone. + + Parameters + ---------- + tz : string / timezone object, default None + Timezone to localize to + """) + round = _make_nat_func('round', + """ + Round the Timestamp to the specified resolution - now = _make_nat_func('now', '') - today = _make_nat_func('today', '') - round = _make_nat_func('round', '') - floor = _make_nat_func('floor', '') - ceil = _make_nat_func('ceil', '') + Returns + ------- + a new Timestamp rounded to the given resolution of `freq` - tz_convert = _make_nat_func('tz_convert', '') - tz_localize = _make_nat_func('tz_localize', '') - replace = _make_nat_func('replace', '') + Parameters + ---------- + freq : a freq string indicating the rounding resolution + + Raises + ------ + ValueError if the freq cannot be converted + """) + floor = _make_nat_func('floor', + """ + return a new Timestamp floored to this resolution + + Parameters + ---------- + freq : a freq string indicating the flooring resolution + """) + ceil = _make_nat_func('ceil', + """ + return a new Timestamp ceiled to this resolution + + Parameters + ---------- + freq : a freq string indicating the ceiling resolution + """) + + tz_convert = _make_nat_func('tz_convert', + """ + Convert tz-aware Timestamp to another time zone. + + Parameters + ---------- + tz : string, pytz.timezone, dateutil.tz.tzfile or None + Time zone for time which Timestamp will be converted to. + None will remove timezone holding UTC time. + + Returns + ------- + converted : Timestamp + + Raises + ------ + TypeError + If Timestamp is tz-naive. + """) + tz_localize = _make_nat_func('tz_localize', + """ + Convert naive Timestamp to local time zone, or remove + timezone from tz-aware Timestamp. + + Parameters + ---------- + tz : string, pytz.timezone, dateutil.tz.tzfile or None + Time zone for time which Timestamp will be converted to. + None will remove timezone holding local time. + ambiguous : bool, 'NaT', default 'raise' + - bool contains flags to determine if time is dst or not (note + that this flag is only applicable for ambiguous fall dst dates) + - 'NaT' will return NaT for an ambiguous time + - 'raise' will raise an AmbiguousTimeError for an ambiguous time + errors : 'raise', 'coerce', default 'raise' + - 'raise' will raise a NonExistentTimeError if a timestamp is not + valid in the specified timezone (e.g. due to a transition from + or to DST time) + - 'coerce' will return NaT if the timestamp can not be converted + into the specified timezone + + .. versionadded:: 0.19.0 + + Returns + ------- + localized : Timestamp + + Raises + ------ + TypeError + If the Timestamp is tz-aware and tz is not None. + """) + replace = _make_nat_func('replace', + """ + implements datetime.replace, handles nanoseconds + + Parameters + ---------- + year : int, optional + month : int, optional + day : int, optional + hour : int, optional + minute : int, optional + second : int, optional + microsecond : int, optional + nanosecond: int, optional + tzinfo : tz-convertible, optional + fold : int, optional, default is 0 + added in 3.6, NotImplemented + + Returns + ------- + Timestamp with fields replaced + """) def to_datetime(self): """ diff --git a/pandas/_libs/tslibs/strptime.pyx b/pandas/_libs/tslibs/strptime.pyx index de1731331cc36..149509b12da6a 100644 --- a/pandas/_libs/tslibs/strptime.pyx +++ b/pandas/_libs/tslibs/strptime.pyx @@ -45,7 +45,7 @@ from util cimport is_string_object, get_nat cdef int64_t NPY_NAT = get_nat() from nattype cimport _checknull_with_nat -from nattype import _nat_strings +from nattype import nat_strings def array_strptime(ndarray[object] values, object fmt, @@ -143,7 +143,7 @@ def array_strptime(ndarray[object] values, object fmt, for i in range(n): val = values[i] if is_string_object(val): - if val in _nat_strings: + if val in nat_strings: iresult[i] = NPY_NAT continue else: diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index 03377d21ed484..da1163e25f5c6 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -9,7 +9,7 @@ from numpy cimport int64_t cimport util -from nattype import _nat_strings +from nattype import nat_strings # ---------------------------------------------------------------------- # Constants @@ -114,7 +114,7 @@ cdef inline parse_timedelta_string(object ts): # have_value : track if we have at least 1 leading unit # have_hhmmss : tracks if we have a regular format hh:mm:ss - if len(ts) == 0 or ts in _nat_strings: + if len(ts) == 0 or ts in nat_strings: return NPY_NAT # decode ts if necessary diff --git a/pandas/core/tools/datetimes.py b/pandas/core/tools/datetimes.py index e335dfe3a4142..ae8aa275b2bae 100644 --- a/pandas/core/tools/datetimes.py +++ b/pandas/core/tools/datetimes.py @@ -535,7 +535,7 @@ def calc_with_mask(carg, mask): # string with NaN-like try: - mask = ~algorithms.isin(arg, list(tslib._nat_strings)) + mask = ~algorithms.isin(arg, list(tslib.nat_strings)) return calc_with_mask(arg, mask) except: pass From 776d581e55435f5e01dadde62cd28af01a9fc4e3 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 28 Oct 2017 13:26:06 -0700 Subject: [PATCH 03/10] Update pickle_compat --- pandas/compat/pickle_compat.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pandas/compat/pickle_compat.py b/pandas/compat/pickle_compat.py index f6223c48994ae..cab5d8bce6692 100644 --- a/pandas/compat/pickle_compat.py +++ b/pandas/compat/pickle_compat.py @@ -74,10 +74,12 @@ def load_reduce(self): ('pandas._libs.sparse', 'BlockIndex'), ('pandas.tslib', 'Timestamp'): ('pandas._libs.tslib', 'Timestamp'), - ('pandas.tslib', '__nat_unpickle'): - ('pandas._libs.tslib', '__nat_unpickle'), ('pandas._period', 'Period'): ('pandas._libs.period', 'Period'), + # 18014 moved __nat_unpickle from _libs.tslib-->_libs.tslibs.nattype + ('pandas.tslib', '__nat_unpickle'): + ('pandas._libs.tslibs.nattype', '__nat_unpickle'), + # 15998 top-level dirs moving ('pandas.sparse.array', 'SparseArray'): ('pandas.core.sparse.array', 'SparseArray'), From 3a44c2c35aa1222f16e092712decf9eadd0b2e3c Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 28 Oct 2017 14:21:04 -0700 Subject: [PATCH 04/10] fix import of NaTType --- pandas/_libs/src/ujson/python/objToJSON.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pandas/_libs/src/ujson/python/objToJSON.c b/pandas/_libs/src/ujson/python/objToJSON.c index ae7854dfc1427..f799b7f6b4785 100644 --- a/pandas/_libs/src/ujson/python/objToJSON.c +++ b/pandas/_libs/src/ujson/python/objToJSON.c @@ -162,7 +162,7 @@ void initObjToJSON(void) #endif { PyObject *mod_pandas; - PyObject *mod_tslib; + PyObject *mod_nattype; PyObject *mod_decimal = PyImport_ImportModule("decimal"); type_decimal = PyObject_GetAttrString(mod_decimal, "Decimal"); Py_INCREF(type_decimal); @@ -180,10 +180,11 @@ void initObjToJSON(void) Py_DECREF(mod_pandas); } - mod_tslib = PyImport_ImportModule("pandas._libs.tslib"); - if (mod_tslib) { - cls_nat = (PyTypeObject *)PyObject_GetAttrString(mod_tslib, "NaTType"); - Py_DECREF(mod_tslib); + mod_nattype = PyImport_ImportModule("pandas._libs.tslibs.nattype"); + if (mod_nattype) { + cls_nat = (PyTypeObject *)PyObject_GetAttrString(mod_nattype, + "NaTType"); + Py_DECREF(mod_nattype); } /* Initialise numpy API and use 2/3 compatible return */ From 009c907ba9776f93ff8b9e0bdc88e685b336e835 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 28 Oct 2017 22:04:09 -0700 Subject: [PATCH 05/10] update import; fixes downstream test --- pandas/tslib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tslib.py b/pandas/tslib.py index c960a4eaf59ad..c06b34c1b0483 100644 --- a/pandas/tslib.py +++ b/pandas/tslib.py @@ -3,5 +3,5 @@ import warnings warnings.warn("The pandas.tslib module is deprecated and will be " "removed in a future version.", FutureWarning, stacklevel=2) -from pandas._libs.tslib import (Timestamp, Timedelta, - NaT, NaTType, OutOfBoundsDatetime) +from pandas._libs.tslib import Timestamp, Timedelta, OutOfBoundsDatetime +from pandas._libs.tslibs.nattype import NaT, NaTType From 449d5deae01d7d4d87de912152f3db3d1e9dc43f Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 28 Oct 2017 23:09:27 -0700 Subject: [PATCH 06/10] import nat_unpickle to fix backcompat pickle --- pandas/_libs/tslib.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index 68990c2d1ad82..89b9f52bd78fe 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -94,7 +94,8 @@ from tslibs.fields import ( get_date_name_field, get_start_end_field, get_date_field, build_field_sarray) -from tslibs.nattype import NaT, nat_strings +from tslibs.nattype import NaT, nat_strings, __nat_unpickle +# Note: __nat_unpickle needs to be in the namespace for backward compat pickle from tslibs.nattype cimport _checknull_with_nat From 05e5981042468edcab2457bd5d14fb826ad19c81 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 29 Oct 2017 10:37:09 -0700 Subject: [PATCH 07/10] flake8 fixup --- pandas/_libs/tslib.pyx | 1 - pandas/_libs/tslibs/nattype.pyx | 28 ++++++++++++++-------------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index 89b9f52bd78fe..960c1cfea427e 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -826,7 +826,6 @@ cdef inline bint _check_all_nulls(object val): return res - cpdef object get_value_box(ndarray arr, object loc): cdef: Py_ssize_t i, sz diff --git a/pandas/_libs/tslibs/nattype.pyx b/pandas/_libs/tslibs/nattype.pyx index 54861e2520785..dedc115501cd0 100644 --- a/pandas/_libs/tslibs/nattype.pyx +++ b/pandas/_libs/tslibs/nattype.pyx @@ -138,7 +138,7 @@ cdef class _NaT(datetime): # Duplicate some logic from _Timestamp.__sub__ to avoid needing # to subclass; allows us to @final(_Timestamp.__sub__) if PyDateTime_Check(other): - return NaT + return NaT elif PyDelta_Check(other): return NaT @@ -337,7 +337,7 @@ class NaTType(_NaT): utcoffset = _make_error_func('utcoffset', datetime) # Timestamp has empty docstring for some methods. - utcfromtimestamp = _make_error_func('utcfromtimestamp', None) + utcfromtimestamp = _make_error_func('utcfromtimestamp', None) fromtimestamp = _make_error_func('fromtimestamp', None) combine = _make_error_func('combine', None) utcnow = _make_error_func('utcnow', None) @@ -346,12 +346,12 @@ class NaTType(_NaT): # The remaining methods have docstrings copy/pasted from the analogous # Timestamp methods. - timestamp = _make_error_func('timestamp', + timestamp = _make_error_func('timestamp', # noqa:E128 """Return POSIX timestamp as float.""") # GH9513 NaT methods (except to_datetime64) to raise, return np.nan, or # return NaT create functions that raise, for binding to NaTType - astimezone = _make_error_func('astimezone', + astimezone = _make_error_func('astimezone', # noqa:E128 """ Convert tz-aware Timestamp to another time zone. @@ -370,7 +370,7 @@ class NaTType(_NaT): TypeError If Timestamp is tz-naive. """) - fromordinal = _make_error_func('fromordinal', + fromordinal = _make_error_func('fromordinal', # noqa:E128 """ passed an ordinal, translate and convert to a ts note: by definition there cannot be any tz info on the ordinal itself @@ -388,14 +388,14 @@ class NaTType(_NaT): """) # _nat_methods - to_pydatetime = _make_nat_func('to_pydatetime', + to_pydatetime = _make_nat_func('to_pydatetime', # noqa:E128 """ Convert a Timestamp object to a native Python datetime object. If warn=True, issue a warning if nanoseconds is nonzero. """) - now = _make_nat_func('now', + now = _make_nat_func('now', # noqa:E128 """ Return the current time in the local timezone. Equivalent to datetime.now([tz]) @@ -405,7 +405,7 @@ class NaTType(_NaT): tz : string / timezone object, default None Timezone to localize to """) - today = _make_nat_func('today', + today = _make_nat_func('today', # noqa:E128 """ Return the current time in the local timezone. This differs from datetime.today() in that it can be localized to a @@ -416,7 +416,7 @@ class NaTType(_NaT): tz : string / timezone object, default None Timezone to localize to """) - round = _make_nat_func('round', + round = _make_nat_func('round', # noqa:E128 """ Round the Timestamp to the specified resolution @@ -432,7 +432,7 @@ class NaTType(_NaT): ------ ValueError if the freq cannot be converted """) - floor = _make_nat_func('floor', + floor = _make_nat_func('floor', # noqa:E128 """ return a new Timestamp floored to this resolution @@ -440,7 +440,7 @@ class NaTType(_NaT): ---------- freq : a freq string indicating the flooring resolution """) - ceil = _make_nat_func('ceil', + ceil = _make_nat_func('ceil', # noqa:E128 """ return a new Timestamp ceiled to this resolution @@ -449,7 +449,7 @@ class NaTType(_NaT): freq : a freq string indicating the ceiling resolution """) - tz_convert = _make_nat_func('tz_convert', + tz_convert = _make_nat_func('tz_convert', # noqa:E128 """ Convert tz-aware Timestamp to another time zone. @@ -468,7 +468,7 @@ class NaTType(_NaT): TypeError If Timestamp is tz-naive. """) - tz_localize = _make_nat_func('tz_localize', + tz_localize = _make_nat_func('tz_localize', # noqa:E128 """ Convert naive Timestamp to local time zone, or remove timezone from tz-aware Timestamp. @@ -501,7 +501,7 @@ class NaTType(_NaT): TypeError If the Timestamp is tz-aware and tz is not None. """) - replace = _make_nat_func('replace', + replace = _make_nat_func('replace', # noqa:E128 """ implements datetime.replace, handles nanoseconds From 63ff688a6e5943865faa63cb61568f3453329df2 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 30 Oct 2017 17:49:30 -0700 Subject: [PATCH 08/10] remove import to try to recreate test failure --- pandas/_libs/tslib.pyx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index 7e70cb9ef341f..b40bb63e7223d 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -103,8 +103,7 @@ from tslibs.conversion import ( tz_localize_to_utc, tz_convert, tz_convert_single) -from tslibs.nattype import NaT, nat_strings, __nat_unpickle -# Note: __nat_unpickle needs to be in the namespace for backward compat pickle +from tslibs.nattype import NaT, nat_strings from tslibs.nattype cimport _checknull_with_nat From 5f3b5125e2e969fde21f8c30c390977d985a9754 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 30 Oct 2017 19:10:10 -0700 Subject: [PATCH 09/10] nat_pickle fixup --- pandas/compat/pickle_compat.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/compat/pickle_compat.py b/pandas/compat/pickle_compat.py index cab5d8bce6692..c3caaf7288738 100644 --- a/pandas/compat/pickle_compat.py +++ b/pandas/compat/pickle_compat.py @@ -78,6 +78,8 @@ def load_reduce(self): # 18014 moved __nat_unpickle from _libs.tslib-->_libs.tslibs.nattype ('pandas.tslib', '__nat_unpickle'): + ('pandas._libs.tslib', '__nat_unpickle'), + ('pandas._libs.tslib', '__nat_unpickle'): ('pandas._libs.tslibs.nattype', '__nat_unpickle'), # 15998 top-level dirs moving From 11e699cb78c71325a7f5c190bb7031ffe0198ea1 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 30 Oct 2017 19:27:45 -0700 Subject: [PATCH 10/10] another try at nat_pickle --- pandas/compat/pickle_compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/compat/pickle_compat.py b/pandas/compat/pickle_compat.py index c3caaf7288738..8015642919611 100644 --- a/pandas/compat/pickle_compat.py +++ b/pandas/compat/pickle_compat.py @@ -78,7 +78,7 @@ def load_reduce(self): # 18014 moved __nat_unpickle from _libs.tslib-->_libs.tslibs.nattype ('pandas.tslib', '__nat_unpickle'): - ('pandas._libs.tslib', '__nat_unpickle'), + ('pandas._libs.tslibs.nattype', '__nat_unpickle'), ('pandas._libs.tslib', '__nat_unpickle'): ('pandas._libs.tslibs.nattype', '__nat_unpickle'),