From 0a8edfa5b16ca04866540ae5ea965404263bc3cf Mon Sep 17 00:00:00 2001 From: Alexis Mignon Date: Thu, 16 Jun 2016 15:11:46 +0200 Subject: [PATCH 1/8] BUG: Fix a bug occuring when using DataFrame.to_records with unicode column names in python 2. (issue #11879) --- pandas/core/frame.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 69def7502a6f7..1d366793d515e 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -1062,10 +1062,11 @@ def to_records(self, index=True, convert_datetime64=True): count += 1 elif index_names[0] is None: index_names = ['index'] - names = lmap(str, index_names) + lmap(str, self.columns) + names = (lmap(compat.text_type, index_names) + + lmap(compat.text_type, self.columns)) else: arrays = [self[c].get_values() for c in self.columns] - names = lmap(str, self.columns) + names = lmap(compat.text_type, self.columns) dtype = np.dtype([(x, v.dtype) for x, v in zip(names, arrays)]) return np.rec.fromarrays(arrays, dtype=dtype, names=names) From dd64e915c3f0ff70b8535360f2137f3324f9ed91 Mon Sep 17 00:00:00 2001 From: Alexis Mignon Date: Thu, 16 Jun 2016 16:47:17 +0200 Subject: [PATCH 2/8] BUG: Add a non regression test for to_records() with unicode column names in python 2 --- pandas/tests/frame/test_convert_to.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pandas/tests/frame/test_convert_to.py b/pandas/tests/frame/test_convert_to.py index 53083a602e183..396e013a1c4a7 100644 --- a/pandas/tests/frame/test_convert_to.py +++ b/pandas/tests/frame/test_convert_to.py @@ -179,3 +179,11 @@ def test_to_records_with_unicode_index(self): .to_records() expected = np.rec.array([('x', 'y')], dtype=[('a', 'O'), ('b', 'O')]) tm.assert_almost_equal(result, expected) + + def test_to_records_with_unicode_column_names(self): + # Issue #11879. to_records used to raise an exception when used + # with column names containing non ascii caracters in Python 2 + try: + DataFrame(columns=[u"accented_name_é"]).to_records() + except UnicodeEncodeError: + self.fail("to_records() raised a UnicodeEncodeError exception") From 35a3569f3bf24cb27b35e8f644cda7d3069fa821 Mon Sep 17 00:00:00 2001 From: Alexis Mignon Date: Tue, 27 Dec 2016 04:04:29 +0100 Subject: [PATCH 3/8] Changed the way dtype is specified in to_records Changed the way dtype is specified in to_records in order to allow unicode field names. --- pandas/core/frame.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 166ff48a0fb20..1fb717b4d4c6d 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -1110,8 +1110,8 @@ def to_records(self, index=True, convert_datetime64=True): arrays = [self[c].get_values() for c in self.columns] names = lmap(compat.text_type, self.columns) - dtype = np.dtype([(x, v.dtype) for x, v in zip(names, arrays)]) - return np.rec.fromarrays(arrays, dtype=dtype, names=names) + formats = [v.dtype for v in arrays] + return np.rec.fromarrays(arrays, dtype={'names': names, 'formats': formats}) @classmethod def from_items(cls, items, columns=None, orient='columns'): From 587b0f28149f446c75dcefc05f10ffdbb038d04e Mon Sep 17 00:00:00 2001 From: Alexis Mignon Date: Tue, 27 Dec 2016 04:12:26 +0100 Subject: [PATCH 4/8] BUG: Reformatted the test as requested --- pandas/tests/frame/test_convert_to.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pandas/tests/frame/test_convert_to.py b/pandas/tests/frame/test_convert_to.py index 396e013a1c4a7..47c687a45d5fd 100644 --- a/pandas/tests/frame/test_convert_to.py +++ b/pandas/tests/frame/test_convert_to.py @@ -183,7 +183,9 @@ def test_to_records_with_unicode_index(self): def test_to_records_with_unicode_column_names(self): # Issue #11879. to_records used to raise an exception when used # with column names containing non ascii caracters in Python 2 - try: - DataFrame(columns=[u"accented_name_é"]).to_records() - except UnicodeEncodeError: - self.fail("to_records() raised a UnicodeEncodeError exception") + result = DataFrame(data={u"accented_name_é":[1.0]}).to_records() + # Note that numpy allows for unicode field names but dtypes need + # to be specified using dictionnary intsead of list of tuples. + expected = np.rec.array([(0, 1.0)], + dtype={"names": ["index", u"accented_name_é"], + "formats": [' Date: Tue, 27 Dec 2016 04:59:05 +0100 Subject: [PATCH 5/8] [BUG] Added forgotten assert in test --- pandas/tests/frame/test_convert_to.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/frame/test_convert_to.py b/pandas/tests/frame/test_convert_to.py index 47c687a45d5fd..44454c7018935 100644 --- a/pandas/tests/frame/test_convert_to.py +++ b/pandas/tests/frame/test_convert_to.py @@ -183,9 +183,10 @@ def test_to_records_with_unicode_index(self): def test_to_records_with_unicode_column_names(self): # Issue #11879. to_records used to raise an exception when used # with column names containing non ascii caracters in Python 2 - result = DataFrame(data={u"accented_name_é":[1.0]}).to_records() + result = DataFrame(data={u"accented_name_é": [1.0]}).to_records() # Note that numpy allows for unicode field names but dtypes need # to be specified using dictionnary intsead of list of tuples. expected = np.rec.array([(0, 1.0)], dtype={"names": ["index", u"accented_name_é"], "formats": [' Date: Tue, 27 Dec 2016 05:34:31 +0100 Subject: [PATCH 6/8] CLN: Make the test complient with flake8 --- pandas/tests/frame/test_convert_to.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pandas/tests/frame/test_convert_to.py b/pandas/tests/frame/test_convert_to.py index 44454c7018935..fb82b0598bb0a 100644 --- a/pandas/tests/frame/test_convert_to.py +++ b/pandas/tests/frame/test_convert_to.py @@ -186,7 +186,9 @@ def test_to_records_with_unicode_column_names(self): result = DataFrame(data={u"accented_name_é": [1.0]}).to_records() # Note that numpy allows for unicode field names but dtypes need # to be specified using dictionnary intsead of list of tuples. - expected = np.rec.array([(0, 1.0)], - dtype={"names": ["index", u"accented_name_é"], - "formats": [' Date: Tue, 27 Dec 2016 05:38:27 +0100 Subject: [PATCH 7/8] CLN: Remove flake8 warnings --- pandas/core/frame.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 1fb717b4d4c6d..3ef8964d9b31e 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -1111,7 +1111,10 @@ def to_records(self, index=True, convert_datetime64=True): names = lmap(compat.text_type, self.columns) formats = [v.dtype for v in arrays] - return np.rec.fromarrays(arrays, dtype={'names': names, 'formats': formats}) + return np.rec.fromarrays( + arrays, + dtype={'names': names, 'formats': formats} + ) @classmethod def from_items(cls, items, columns=None, orient='columns'): From a82784481328170ae0665e128d2a622039735836 Mon Sep 17 00:00:00 2001 From: Alexis Mignon Date: Tue, 3 Jan 2017 17:12:42 +0100 Subject: [PATCH 8/8] Added bug fix entry in what's new --- doc/source/whatsnew/v0.20.0.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v0.20.0.txt b/doc/source/whatsnew/v0.20.0.txt index 65b62601c7022..c3c3e85f79f02 100644 --- a/doc/source/whatsnew/v0.20.0.txt +++ b/doc/source/whatsnew/v0.20.0.txt @@ -84,3 +84,4 @@ Performance Improvements Bug Fixes ~~~~~~~~~ +- Bug in ``pd.DataFrame.to_records`` which failed with unicode caracters in column names (:issue:`11879`)