From ea54ddfde1cec3fc576136b0c561c53ffd221e21 Mon Sep 17 00:00:00 2001 From: Javier Sorribes Camargo Date: Sat, 25 Feb 2017 15:22:01 -0600 Subject: [PATCH 1/6] BUG: Overflow bug fixed for nansum --- pandas/core/nanops.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/core/nanops.py b/pandas/core/nanops.py index 0cc3a2d039b5e..f2f6a219400c6 100644 --- a/pandas/core/nanops.py +++ b/pandas/core/nanops.py @@ -2,6 +2,7 @@ import functools import numpy as np import operator +import os try: import bottleneck as bn @@ -130,7 +131,9 @@ def _bn_ok_dtype(dt, name): if name == 'nansum': if dt.itemsize < 8: return False - + elif os.name == 'nt': # bottleneck has overflow issue in Windows + return False + return True return False From 508bc4d93d30320dac528b61621bc55d890f1f61 Mon Sep 17 00:00:00 2001 From: Javier Sorribes Camargo Date: Sat, 25 Feb 2017 15:14:08 -0600 Subject: [PATCH 2/6] TST: Add numsum overflow test --- pandas/tests/test_nanops.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index 937c20d009b6b..7cba8d4ffcdbe 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -327,6 +327,12 @@ def test_nansum(self): self.check_funs(nanops.nansum, np.sum, allow_str=False, allow_date=False, allow_tdelta=True, check_dtype=False) + def test_nansum_overflow(self): + s = Series([2**31]) + self.assertEqual(s.sum(),2147483648) + s = Series([2**31-1,1]) + self.assertEqual(s.sum(),2147483648) + def test_nanmean(self): self.check_funs(nanops.nanmean, np.mean, allow_complex=False, allow_obj=False, allow_str=False, allow_date=False, From c40f6dc27c149eca26db71c56d675b928644d834 Mon Sep 17 00:00:00 2001 From: Javier Sorribes Camargo Date: Sat, 25 Feb 2017 21:23:10 -0600 Subject: [PATCH 3/6] CLN: Fix linting of ci/lint.sh --- pandas/core/nanops.py | 4 ++-- pandas/tests/test_nanops.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pandas/core/nanops.py b/pandas/core/nanops.py index f2f6a219400c6..75120f1d8e488 100644 --- a/pandas/core/nanops.py +++ b/pandas/core/nanops.py @@ -131,9 +131,9 @@ def _bn_ok_dtype(dt, name): if name == 'nansum': if dt.itemsize < 8: return False - elif os.name == 'nt': # bottleneck has overflow issue in Windows + elif os.name == 'nt': # bottleneck has overflow issue in Windows return False - + return True return False diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index 7cba8d4ffcdbe..f1c64ca96a80c 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -329,10 +329,10 @@ def test_nansum(self): def test_nansum_overflow(self): s = Series([2**31]) - self.assertEqual(s.sum(),2147483648) - s = Series([2**31-1,1]) - self.assertEqual(s.sum(),2147483648) - + self.assertEqual(s.sum(), 2147483648) + s = Series([2**31 - 1, 1]) + self.assertEqual(s.sum(), 2147483648) + def test_nanmean(self): self.check_funs(nanops.nanmean, np.mean, allow_complex=False, allow_obj=False, allow_str=False, allow_date=False, From af7aebbc48391c6766c6d5e7052c919f9c54a523 Mon Sep 17 00:00:00 2001 From: Javier Sorribes Camargo Date: Sat, 25 Feb 2017 21:51:07 -0600 Subject: [PATCH 4/6] BUG: Fix nansum overflow for Windows --- doc/source/whatsnew/v0.20.0.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/whatsnew/v0.20.0.txt b/doc/source/whatsnew/v0.20.0.txt index c94429b469641..184733da787af 100644 --- a/doc/source/whatsnew/v0.20.0.txt +++ b/doc/source/whatsnew/v0.20.0.txt @@ -632,3 +632,5 @@ Bug Fixes - Bug in ``pd.melt()`` where passing a tuple value for ``value_vars`` caused a ``TypeError`` (:issue:`15348`) - Bug in ``.eval()`` which caused multiline evals to fail with local variables not on the first line (:issue:`15342`) - Bug in ``pd.read_msgpack`` which did not allow to load dataframe with an index of type ``CategoricalIndex`` (:issue:`15487`) + +- Bug in ``nanops.py`` due to bottleneck, which produces overflow in nansum on Windows (:issue:`15453`) From 933590375cfc50096b6de06443273fc57dfeebcf Mon Sep 17 00:00:00 2001 From: Javier Sorribes Camargo Date: Tue, 28 Feb 2017 01:35:44 -0600 Subject: [PATCH 5/6] BUG: Disable bottleneck for nansum to avoid overflow --- pandas/core/nanops.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pandas/core/nanops.py b/pandas/core/nanops.py index 75120f1d8e488..6f2c2af7d43f3 100644 --- a/pandas/core/nanops.py +++ b/pandas/core/nanops.py @@ -129,10 +129,7 @@ def _bn_ok_dtype(dt, name): # bottleneck does not properly upcast during the sum # so can overflow if name == 'nansum': - if dt.itemsize < 8: - return False - elif os.name == 'nt': # bottleneck has overflow issue in Windows - return False + return False return True return False From 7b95a881c6de9367cbe5afa4e23bdfa9c6c6360b Mon Sep 17 00:00:00 2001 From: Javier Sorribes Camargo Date: Wed, 1 Mar 2017 23:49:16 -0600 Subject: [PATCH 6/6] CLN: Remove import os, since not using os --- pandas/core/nanops.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/core/nanops.py b/pandas/core/nanops.py index 6f2c2af7d43f3..151cbd0dd6032 100644 --- a/pandas/core/nanops.py +++ b/pandas/core/nanops.py @@ -2,7 +2,6 @@ import functools import numpy as np import operator -import os try: import bottleneck as bn