COMPAT: Iteration should always yield a python scalar

xref #10904
closes #13236
closes #13256
xref #14216

Author: jreback

doc/source/whatsnew/v0.21.0.txt

@@ -188,6 +188,53 @@ the target. Now, a ``ValueError`` will be raised when such an input is passed in
    ...
    ValueError: Cannot operate inplace if there is no assignment
 
+.. _whatsnew_0210.api_breaking.iteration_scalars:
+
+Iteration of Series/Index will now return python scalars
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Previously, when using certain iteration methods for a ``Series`` with dtype ``int`` or ``float``, you would receive a ``numpy`` scalar, e.g. a ``np.int64``, rather than a python ``int``. Issue (:issue:`10904`) corrected this for ``Series.tolist()`` and ``list(Series)``. This change makes all iteration methods consistent, in particular, for ``__iter__()`` and ``.map()``; note that this only affect int/float dtypes. (:issue:`13236`, :issue:`13258`, :issue:`14216`).
+
+.. ipython:: python
+
+   s = Series([1, 2, 3])
+   s
+
+Previously:
+
+.. code-block:: python
+
+   In [2]: type(list(s)[0])
+   Out[2]: numpy.int64
+
+New Behaviour:
+
+.. ipython:: python
+
+   type(list(s)[0])
+
+Furthermore this will now correctly box the results of iteration for :func:`DataFrame.to_dict` as well.
+
+.. ipython:: python
+
+   d = {'a':[1], 'b':['b']}
+   df = DataFrame(d)
+
+Previously:
+
+.. code-block:: python
+
+   In [8]: type(df.to_dict()['a'][0])
+   Out[8]: numpy.int64
+
+New Behaviour:
+
+.. ipython:: python
+
+   type(df.to_dict()['a'][0])
+
+.. _whatsnew_0210.api_breaking.dtype_conversions:
+
 Dtype Conversions
 ^^^^^^^^^^^^^^^^^
 

pandas/core/base.py

@@ -8,7 +8,12 @@
 
 from pandas.core.dtypes.missing import isna
 from pandas.core.dtypes.generic import ABCDataFrame, ABCSeries, ABCIndexClass
-from pandas.core.dtypes.common import is_object_dtype, is_list_like, is_scalar
+from pandas.core.dtypes.common import (
+    is_object_dtype,
+    is_list_like,
+    is_scalar,
+    is_datetimelike)
+
 from pandas.util._validators import validate_bool_kwarg
 
 from pandas.core import common as com
@@ -18,7 +23,8 @@
 from pandas.compat import PYPY
 from pandas.util._decorators import (Appender, cache_readonly,
                                      deprecate_kwarg, Substitution)
-from pandas.core.common import AbstractMethodError
+from pandas.core.common import AbstractMethodError, _maybe_box_datetimelike
+
 from pandas.core.accessor import DirNamesMixin
 
 _shared_docs = dict()
@@ -884,6 +890,21 @@ def argmin(self, axis=None):
         """
         return nanops.nanargmin(self.values)
 
+    def tolist(self):
+        """
+        return a list of the values; box to scalars
+        """
+        return list(self.__iter__())
+
+    def __iter__(self):
+        """
+        provide iteration over the values; box to scalars
+        """
+        if is_datetimelike(self):
+            return (_maybe_box_datetimelike(x) for x in self._values)
+        else:
+            return iter(self._values.tolist())
+
     @cache_readonly
     def hasnans(self):
         """ return if I have any nans; enables various perf speedups """

pandas/core/categorical.py

@@ -399,6 +399,12 @@ def itemsize(self):
         """ return the size of a single category """
         return self.categories.itemsize
 
+    def tolist(self):
+        """
+        return a list of my values
+        """
+        return np.array(self).tolist()
+
     def reshape(self, new_shape, *args, **kwargs):
         """
         .. deprecated:: 0.19.0

pandas/core/indexes/base.py

@@ -585,12 +585,6 @@ def memory_usage(self, deep=False):
         return result
 
     # ops compat
-    def tolist(self):
-        """
-        return a list of the Index values
-        """
-        return list(self.values)
-
     @deprecate_kwarg(old_arg_name='n', new_arg_name='repeats')
     def repeat(self, repeats, *args, **kwargs):
         """
@@ -1601,9 +1595,6 @@ def is_all_dates(self):
             return False
         return is_datetime_array(_ensure_object(self.values))
 
-    def __iter__(self):
-        return iter(self.values)
-
     def __reduce__(self):
         d = dict(data=self._data)
         d.update(self._get_attributes_dict())

pandas/core/indexes/category.py

@@ -253,6 +253,10 @@ def get_values(self):
         """ return the underlying data as an ndarray """
         return self._data.get_values()
 
+    def __iter__(self):
+        """ iterate like Categorical """
+        return self._data.__iter__()
+
     @property
     def codes(self):
         return self._data.codes

pandas/core/series.py

@@ -19,7 +19,6 @@
     is_integer, is_integer_dtype,
     is_float_dtype,
     is_extension_type, is_datetimetz,
-    is_datetimelike,
     is_datetime64tz_dtype,
     is_timedelta64_dtype,
     is_list_like,
@@ -1095,14 +1094,6 @@ def to_string(self, buf=None, na_rep='NaN', float_format=None, header=True,
                 with open(buf, 'w') as f:
                     f.write(result)
 
-    def __iter__(self):
-        """ provide iteration over the values of the Series
-        box values if necessary """
-        if is_datetimelike(self):
-            return (_maybe_box_datetimelike(x) for x in self._values)
-        else:
-            return iter(self._values)
-
     def iteritems(self):
         """
         Lazily iterate over (index, value) tuples
@@ -1118,10 +1109,6 @@ def keys(self):
         """Alias for index"""
         return self.index
 
-    def tolist(self):
-        """ Convert Series to a nested list """
-        return list(self.asobject)
-
     def to_dict(self, into=dict):
         """
         Convert Series to {label -> value} dict or dict-like object.

pandas/core/sparse/array.py

@@ -407,8 +407,18 @@ def to_dense(self, fill=None):
         return self.values
 
     def __iter__(self):
+        if np.issubdtype(self.dtype, np.floating):
+            boxer = float
+        elif np.issubdtype(self.dtype, np.integer):
+            boxer = int
+        else:
+            boxer = lambda x: x
+
         for i in range(len(self)):
-            yield self._get_val_at(i)
+            r = self._get_val_at(i)
+
+            # box em
+            yield boxer(r)
 
     def __getitem__(self, key):
         """

pandas/tests/frame/test_api.py

@@ -9,7 +9,7 @@
 import sys
 from distutils.version import LooseVersion
 
-from pandas.compat import range, lrange
+from pandas.compat import range, lrange, long
 from pandas import compat
 
 from numpy.random import randn
@@ -205,7 +205,7 @@ def test_itertuples(self):
                          'ints': lrange(5)}, columns=['floats', 'ints'])
 
         for tup in df.itertuples(index=False):
-            assert isinstance(tup[1], np.integer)
+            assert isinstance(tup[1], (int, long))
 
         df = self.klass(data={"a": [1, 2, 3], "b": [4, 5, 6]})
         dfaa = df[['a', 'a']]

pandas/tests/frame/test_convert_to.py

@@ -5,6 +5,7 @@
 import numpy as np
 
 from pandas import compat
+from pandas.compat import long
 from pandas import (DataFrame, Series, MultiIndex, Timestamp,
                     date_range)
 
@@ -236,3 +237,15 @@ def test_to_records_datetimeindex_with_tz(self, tz):
 
         # both converted to UTC, so they are equal
         tm.assert_numpy_array_equal(result, expected)
+
+    def test_to_dict_box_scalars(self):
+        # 14216
+        # make sure that we are boxing properly
+        d = {'a': [1], 'b': ['b']}
+
+        result = DataFrame(d).to_dict()
+        assert isinstance(list(result['a'])[0], (int, long))
+        assert isinstance(list(result['b'])[0], (int, long))
+
+        result = DataFrame(d).to_dict(orient='records')
+        assert isinstance(result[0]['a'], (int, long))

pandas/tests/series/test_io.py

@@ -10,7 +10,7 @@
 
 from pandas import Series, DataFrame
 
-from pandas.compat import StringIO, u, long
+from pandas.compat import StringIO, u
 from pandas.util.testing import (assert_series_equal, assert_almost_equal,
                                  assert_frame_equal, ensure_clean)
 import pandas.util.testing as tm
@@ -178,37 +178,3 @@ def test_to_dict(self, mapping):
         from_method = Series(ts.to_dict(collections.Counter))
         from_constructor = Series(collections.Counter(ts.iteritems()))
         tm.assert_series_equal(from_method, from_constructor)
-
-
-class TestSeriesToList(TestData):
-
-    def test_tolist(self):
-        rs = self.ts.tolist()
-        xp = self.ts.values.tolist()
-        assert_almost_equal(rs, xp)
-
-        # datetime64
-        s = Series(self.ts.index)
-        rs = s.tolist()
-        assert self.ts.index[0] == rs[0]
-
-    def test_tolist_np_int(self):
-        # GH10904
-        for t in ['int8', 'int16', 'int32', 'int64']:
-            s = pd.Series([1], dtype=t)
-            assert isinstance(s.tolist()[0], (int, long))
-
-    def test_tolist_np_uint(self):
-        # GH10904
-        for t in ['uint8', 'uint16']:
-            s = pd.Series([1], dtype=t)
-            assert isinstance(s.tolist()[0], int)
-        for t in ['uint32', 'uint64']:
-            s = pd.Series([1], dtype=t)
-            assert isinstance(s.tolist()[0], long)
-
-    def test_tolist_np_float(self):
-        # GH10904
-        for t in ['float16', 'float32', 'float64']:
-            s = pd.Series([1], dtype=t)
-            assert isinstance(s.tolist()[0], float)