From 1990e3f2431e44150c50d00abaff3e9c085603c3 Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Tue, 20 Feb 2018 13:13:55 -0800
Subject: [PATCH 01/13] dispatch Series[datetime64] comparison ops to
DatetimeIndex
---
pandas/core/ops.py | 30 +++++++++++++++---------------
pandas/tests/test_base.py | 20 +++++++++++---------
2 files changed, 26 insertions(+), 24 deletions(-)
diff --git a/pandas/core/ops.py b/pandas/core/ops.py
index da65f1f31ed2a..3fa0cc02d64fd 100644
--- a/pandas/core/ops.py
+++ b/pandas/core/ops.py
@@ -10,8 +10,7 @@
import numpy as np
import pandas as pd
-from pandas._libs import (lib, index as libindex,
- algos as libalgos)
+from pandas._libs import lib, algos as libalgos
from pandas import compat
from pandas.util._decorators import Appender
@@ -944,24 +943,20 @@ def na_op(x, y):
# integer comparisons
# we have a datetime/timedelta and may need to convert
+ assert not needs_i8_conversion(x)
mask = None
- if (needs_i8_conversion(x) or
- (not is_scalar(y) and needs_i8_conversion(y))):
-
- if is_scalar(y):
- mask = isna(x)
- y = libindex.convert_scalar(x, com._values_from_object(y))
- else:
- mask = isna(x) | isna(y)
- y = y.view('i8')
+ if not is_scalar(y) and needs_i8_conversion(y):
+ mask = isna(x) | isna(y)
+ y = y.view('i8')
x = x.view('i8')
- try:
+ method = getattr(x, name, None)
+ if method is not None:
with np.errstate(all='ignore'):
result = getattr(x, name)(y)
if result is NotImplemented:
raise TypeError("invalid type comparison")
- except AttributeError:
+ else:
result = op(x, y)
if mask is not None and mask.any():
@@ -991,6 +986,12 @@ def wrapper(self, other, axis=None):
return self._constructor(res_values, index=self.index,
name=res_name)
+ if is_datetime64_dtype(self) or is_datetime64tz_dtype(self):
+ res_values = dispatch_to_index_op(op, self, other,
+ pd.DatetimeIndex)
+ return self._constructor(res_values, index=self.index,
+ name=res_name)
+
elif is_timedelta64_dtype(self):
res_values = dispatch_to_index_op(op, self, other,
pd.TimedeltaIndex)
@@ -1008,8 +1009,7 @@ def wrapper(self, other, axis=None):
elif isinstance(other, (np.ndarray, pd.Index)):
# do not check length of zerodim array
# as it will broadcast
- if (not is_scalar(lib.item_from_zerodim(other)) and
- len(self) != len(other)):
+ if other.ndim != 0 and len(self) != len(other):
raise ValueError('Lengths must match to compare')
res_values = na_op(self.values, np.asarray(other))
diff --git a/pandas/tests/test_base.py b/pandas/tests/test_base.py
index 4b5ad336139b0..40400e7394ffd 100644
--- a/pandas/tests/test_base.py
+++ b/pandas/tests/test_base.py
@@ -10,7 +10,7 @@
import pandas as pd
import pandas.compat as compat
from pandas.core.dtypes.common import (
- is_object_dtype, is_datetimetz,
+ is_object_dtype, is_datetimetz, is_datetime64_dtype,
needs_i8_conversion)
import pandas.util.testing as tm
from pandas import (Series, Index, DatetimeIndex, TimedeltaIndex,
@@ -296,14 +296,16 @@ def test_none_comparison(self):
# result = None != o # noqa
# assert result.iat[0]
# assert result.iat[1]
-
- result = None > o
- assert not result.iat[0]
- assert not result.iat[1]
-
- result = o < None
- assert not result.iat[0]
- assert not result.iat[1]
+ if not (is_datetime64_dtype(o) or is_datetimetz(o)):
+ # Following DatetimeIndex (and Timestamp) convention,
+ # inequality comparisons with Series[datetime64] raise
+ result = None > o
+ assert not result.iat[0]
+ assert not result.iat[1]
+
+ result = o < None
+ assert not result.iat[0]
+ assert not result.iat[1]
def test_ndarray_compat_properties(self):
From f48e6d52c793dcdf8fb1092d78e7eda94be5c904 Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Tue, 20 Feb 2018 14:02:26 -0800
Subject: [PATCH 02/13] remove unsupported case
---
pandas/tests/indexes/datetimes/test_partial_slicing.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pandas/tests/indexes/datetimes/test_partial_slicing.py b/pandas/tests/indexes/datetimes/test_partial_slicing.py
index 6bb4229883525..f263ac78cd343 100644
--- a/pandas/tests/indexes/datetimes/test_partial_slicing.py
+++ b/pandas/tests/indexes/datetimes/test_partial_slicing.py
@@ -2,7 +2,7 @@
import pytest
-from datetime import datetime, date
+from datetime import datetime
import numpy as np
import pandas as pd
import operator as op
@@ -349,7 +349,7 @@ def test_loc_datetime_length_one(self):
@pytest.mark.parametrize('datetimelike', [
Timestamp('20130101'), datetime(2013, 1, 1),
- date(2013, 1, 1), np.datetime64('2013-01-01T00:00', 'ns')])
+ np.datetime64('2013-01-01T00:00', 'ns')])
@pytest.mark.parametrize('op,expected', [
(op.lt, [True, False, False, False]),
(op.le, [True, True, False, False]),
From bd909b06002b6a4bee3b33e0b5d4913f7593ae9b Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Tue, 20 Feb 2018 15:41:31 -0800
Subject: [PATCH 03/13] fix error in older numpys
---
pandas/core/indexes/datetimes.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py
index cc9ce1f3fd5eb..554e519e09beb 100644
--- a/pandas/core/indexes/datetimes.py
+++ b/pandas/core/indexes/datetimes.py
@@ -142,6 +142,9 @@ def wrapper(self, other):
else:
o_mask = other.view('i8') == libts.iNaT
+ # for older numpys we need to be careful not to pass a Series
+ # as a mask below
+ o_mask = com._values_from_object(o_mask)
if o_mask.any():
result[o_mask] = nat_result
From 003c4ff856d89fb6f03607ba522719caa393e5ce Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Tue, 20 Feb 2018 21:48:28 -0800
Subject: [PATCH 04/13] Fixup copy/paste
---
pandas/core/ops.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pandas/core/ops.py b/pandas/core/ops.py
index 3fa0cc02d64fd..4363caf1aaf9e 100644
--- a/pandas/core/ops.py
+++ b/pandas/core/ops.py
@@ -953,7 +953,7 @@ def na_op(x, y):
method = getattr(x, name, None)
if method is not None:
with np.errstate(all='ignore'):
- result = getattr(x, name)(y)
+ result = method(y)
if result is NotImplemented:
raise TypeError("invalid type comparison")
else:
From f942be918738ece803b0368c08720a2f25b50afd Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Tue, 20 Feb 2018 21:57:02 -0800
Subject: [PATCH 05/13] unwrap Series to try to debug appveyor breakage
---
pandas/core/indexes/base.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index 7dfa34bd634ad..68fde295c5302 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -3949,6 +3949,10 @@ def _evaluate_compare(self, other):
result = _comp_method_OBJECT_ARRAY(
op, self.values, other)
else:
+ if isinstance(other, ABCSeries):
+ # Windows builds with some numpy versions (1.13)
+ # require specifically unwrapping Series GH#19800
+ other = other.values
with np.errstate(all='ignore'):
result = op(self.values, np.asarray(other))
From 59c96e82b6617cdb0e64945fb7b95593e7475878 Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Wed, 21 Feb 2018 08:52:32 -0800
Subject: [PATCH 06/13] port test for None comparison
---
pandas/tests/test_base.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/pandas/tests/test_base.py b/pandas/tests/test_base.py
index 40400e7394ffd..6247079e4ac3a 100644
--- a/pandas/tests/test_base.py
+++ b/pandas/tests/test_base.py
@@ -296,9 +296,14 @@ def test_none_comparison(self):
# result = None != o # noqa
# assert result.iat[0]
# assert result.iat[1]
- if not (is_datetime64_dtype(o) or is_datetimetz(o)):
+ if (is_datetime64_dtype(o) or is_datetimetz(o)):
# Following DatetimeIndex (and Timestamp) convention,
# inequality comparisons with Series[datetime64] raise
+ with pytest.raises(TypeError):
+ None > o
+ with pytest.raises(TypeError):
+ o > None
+ else:
result = None > o
assert not result.iat[0]
assert not result.iat[1]
From 25534c6fd766299847737b3ccec09d95becc6abc Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Wed, 21 Feb 2018 08:59:32 -0800
Subject: [PATCH 07/13] troubleshoot appveyor build
---
pandas/core/indexes/base.py | 4 ----
pandas/core/indexes/datetimes.py | 2 +-
2 files changed, 1 insertion(+), 5 deletions(-)
diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index 68fde295c5302..7dfa34bd634ad 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -3949,10 +3949,6 @@ def _evaluate_compare(self, other):
result = _comp_method_OBJECT_ARRAY(
op, self.values, other)
else:
- if isinstance(other, ABCSeries):
- # Windows builds with some numpy versions (1.13)
- # require specifically unwrapping Series GH#19800
- other = other.values
with np.errstate(all='ignore'):
result = op(self.values, np.asarray(other))
diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py
index 554e519e09beb..a5b2bbd7f977c 100644
--- a/pandas/core/indexes/datetimes.py
+++ b/pandas/core/indexes/datetimes.py
@@ -137,7 +137,7 @@ def wrapper(self, other):
result = func(np.asarray(other))
result = com._values_from_object(result)
- if isinstance(other, Index):
+ if isinstance(other, (Index, ABCSeries)):
o_mask = other.values.view('i8') == libts.iNaT
else:
o_mask = other.view('i8') == libts.iNaT
From 3d159984806def30981814a2e43f6f4f99b00ad6 Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Wed, 21 Feb 2018 13:45:13 -0800
Subject: [PATCH 08/13] continue troubleshooting Appveyor
---
pandas/core/indexes/datetimes.py | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py
index a5b2bbd7f977c..5affde78a3209 100644
--- a/pandas/core/indexes/datetimes.py
+++ b/pandas/core/indexes/datetimes.py
@@ -137,8 +137,17 @@ def wrapper(self, other):
result = func(np.asarray(other))
result = com._values_from_object(result)
- if isinstance(other, (Index, ABCSeries)):
+ if isinstance(other, Index):
o_mask = other.values.view('i8') == libts.iNaT
+ elif isinstance(other, ABCSeries):
+ try:
+ # GH#19800 On 32-bit Windows builds this fails, but we can
+ # infer that the mask should be all-False
+ view = other.values.view('i8')
+ except ValueError:
+ o_mask = np.zeros(shape=other.shape, dtype=bool)
+ else:
+ o_mask = view == libts.iNaT
else:
o_mask = other.view('i8') == libts.iNaT
From db487e8ba5f8efcdae8c949e70d0bca188c3d44e Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Wed, 21 Feb 2018 21:21:55 -0800
Subject: [PATCH 09/13] test for date comparison
---
pandas/tests/series/test_arithmetic.py | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/pandas/tests/series/test_arithmetic.py b/pandas/tests/series/test_arithmetic.py
index f727edf8fb7d8..1c723982143d2 100644
--- a/pandas/tests/series/test_arithmetic.py
+++ b/pandas/tests/series/test_arithmetic.py
@@ -79,6 +79,22 @@ def test_ser_cmp_result_names(self, names, op):
class TestTimestampSeriesComparison(object):
+ def test_dt64ser_cmp_date_invalid(self):
+ # GH#19800 datetime.date comparison raises to
+ # match DatetimeIndex/Timestamp/datetime
+ ser = pd.Series(pd.date_range('20010101', periods=10), name='dates')
+ date = ser.iloc[0].to_pydatetime().date()
+ assert not (ser == date).any()
+ assert (ser != date).all()
+ with pytest.raises(TypeError):
+ ser > date
+ with pytest.raises(TypeError):
+ ser < date
+ with pytest.raises(TypeError):
+ ser >= date
+ with pytest.raises(TypeError):
+ ser <= date
+
def test_dt64ser_cmp_period_scalar(self):
ser = Series(pd.period_range('2000-01-01', periods=10, freq='D'))
val = Period('2000-01-04', freq='D')
From ba290ab450f9a882ebed115f4bbac97db585051f Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Thu, 22 Feb 2018 09:04:43 -0800
Subject: [PATCH 10/13] just use isna
---
pandas/core/indexes/datetimes.py | 18 +-----------------
1 file changed, 1 insertion(+), 17 deletions(-)
diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py
index 5affde78a3209..88b3f6155d98c 100644
--- a/pandas/core/indexes/datetimes.py
+++ b/pandas/core/indexes/datetimes.py
@@ -137,23 +137,7 @@ def wrapper(self, other):
result = func(np.asarray(other))
result = com._values_from_object(result)
- if isinstance(other, Index):
- o_mask = other.values.view('i8') == libts.iNaT
- elif isinstance(other, ABCSeries):
- try:
- # GH#19800 On 32-bit Windows builds this fails, but we can
- # infer that the mask should be all-False
- view = other.values.view('i8')
- except ValueError:
- o_mask = np.zeros(shape=other.shape, dtype=bool)
- else:
- o_mask = view == libts.iNaT
- else:
- o_mask = other.view('i8') == libts.iNaT
-
- # for older numpys we need to be careful not to pass a Series
- # as a mask below
- o_mask = com._values_from_object(o_mask)
+ o_mask = np.array(isna(other))
if o_mask.any():
result[o_mask] = nat_result
From f8d5bddabdce0d90ef10ddfa9de1013138a4ceff Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Sat, 24 Feb 2018 08:42:50 -0800
Subject: [PATCH 11/13] comments
---
pandas/core/indexes/datetimes.py | 2 ++
pandas/core/ops.py | 2 ++
2 files changed, 4 insertions(+)
diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py
index 88b3f6155d98c..8d70ef7a30036 100644
--- a/pandas/core/indexes/datetimes.py
+++ b/pandas/core/indexes/datetimes.py
@@ -137,6 +137,8 @@ def wrapper(self, other):
result = func(np.asarray(other))
result = com._values_from_object(result)
+ # Make sure to pass an array to result[...]; indexing with
+ # Series breaks with older version of numpy
o_mask = np.array(isna(other))
if o_mask.any():
result[o_mask] = nat_result
diff --git a/pandas/core/ops.py b/pandas/core/ops.py
index 4363caf1aaf9e..08837f386aac9 100644
--- a/pandas/core/ops.py
+++ b/pandas/core/ops.py
@@ -987,6 +987,8 @@ def wrapper(self, other, axis=None):
name=res_name)
if is_datetime64_dtype(self) or is_datetime64tz_dtype(self):
+ # Dispatch to DatetimeIndex to ensure identical
+ # Series/Index behavior
res_values = dispatch_to_index_op(op, self, other,
pd.DatetimeIndex)
return self._constructor(res_values, index=self.index,
From b618a827d5a77e3268ed58e6778de6edc72721f2 Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Sat, 24 Feb 2018 11:05:46 -0800
Subject: [PATCH 12/13] comment
---
pandas/tests/series/test_arithmetic.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/pandas/tests/series/test_arithmetic.py b/pandas/tests/series/test_arithmetic.py
index 057b5db92c96d..ec0d7296e540e 100644
--- a/pandas/tests/series/test_arithmetic.py
+++ b/pandas/tests/series/test_arithmetic.py
@@ -90,7 +90,8 @@ def test_ser_cmp_result_names(self, names, op):
class TestTimestampSeriesComparison(object):
def test_dt64ser_cmp_date_invalid(self):
# GH#19800 datetime.date comparison raises to
- # match DatetimeIndex/Timestamp/datetime
+ # match DatetimeIndex/Timestamp. This also matches the behavior
+ # of stdlib datetime.datetime
ser = pd.Series(pd.date_range('20010101', periods=10), name='dates')
date = ser.iloc[0].to_pydatetime().date()
assert not (ser == date).any()
From ff516344f83bea946c2743810bea459f3182a265 Mon Sep 17 00:00:00 2001
From: jbrockmendel <jbrockmendel@gmail.com>
Date: Wed, 28 Feb 2018 20:04:31 -0800
Subject: [PATCH 13/13] Update ops.py
---
pandas/core/ops.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pandas/core/ops.py b/pandas/core/ops.py
index 0ded8556c924d..931e91b941a7e 100644
--- a/pandas/core/ops.py
+++ b/pandas/core/ops.py
@@ -10,7 +10,7 @@
import numpy as np
import pandas as pd
-from pandas._libs import lib, algos as libalgos, ops as libops
+from pandas._libs import algos as libalgos, ops as libops
from pandas import compat
from pandas.util._decorators import Appender