CLN: Py2/3-compatible dict keys/items/values.

+ many other fixups

Author: jtratner

doc/make.py

@@ -259,7 +259,7 @@ def _get_config():
         func = funcd.get(arg)
         if func is None:
             raise SystemExit('Do not know how to handle %s; valid args are %s' % (
-                arg, funcd.keys()))
+                arg, list(funcd.keys())))
         func()
 else:
     small_docs = False

pandas/core/array.py

@@ -2,7 +2,6 @@
 Isolate pandas's exposure to NumPy
 """
 
-from pandas.util import compat
 import numpy as np
 import six
 
@@ -18,7 +17,7 @@
 
 _lift_types = []
 
-for _k, _v in compat.iteritems(_dtypes):
+for _k, _v in _dtypes.items():
     for _i in _v:
         _lift_types.append(_k + str(_i))
 

pandas/core/common.py

@@ -2,10 +2,10 @@
 Misc tools for implementing data structures
 """
 
-from pandas.util.py3compat import range, long
-import itertools
 import re
 from datetime import datetime
+import codecs
+import csv
 
 from numpy.lib.format import read_array, write_array
 import numpy as np
@@ -15,15 +15,13 @@
 import pandas.tslib as tslib
 
 from pandas.util import py3compat
-import codecs
-import csv
+from pandas.util.py3compat import StringIO, BytesIO, range, long
+from six.moves import zip, map
+import six
 
-from pandas.util.py3compat import StringIO, BytesIO
 
 from pandas.core.config import get_option
 from pandas.core import array as pa
-import six
-from six.moves import map
 
 # XXX: HACK for NumPy 1.5.1 to suppress warnings
 try:
@@ -1366,7 +1364,7 @@ def iterpairs(seq):
     seq_it_next = iter(seq)
     next(seq_it_next)
 
-    return itertools.izip(seq_it, seq_it_next)
+    return zip(seq_it, seq_it_next)
 
 
 def split_ranges(mask):
@@ -1992,7 +1990,7 @@ def _pprint_dict(seq, _nest_lvl=0,**kwds):
 
     nitems = get_option("max_seq_items") or len(seq)
 
-    for k, v in seq.items()[:nitems]:
+    for k, v in list(seq.items())[:nitems]:
         pairs.append(pfmt % (pprint_thing(k,_nest_lvl+1,**kwds),
                              pprint_thing(v,_nest_lvl+1,**kwds)))
 
@@ -2048,7 +2046,7 @@ def as_escaped_unicode(thing,escape_chars=escape_chars):
                 translate.update(escape_chars)
             else:
                 translate = escape_chars
-            escape_chars = escape_chars.keys()
+            escape_chars = list(escape_chars.keys())
         else:
             escape_chars = escape_chars or tuple()
         for c in escape_chars:

pandas/core/config.py

@@ -128,8 +128,8 @@ def _set_option(*args, **kwargs):
 
     # if 1 kwarg then it must be silent=True or silent=False
     if nkwargs:
-        k, = kwargs.keys()
-        v, = kwargs.values()
+        k, = list(kwargs.keys())
+        v, = list(kwargs.values())
 
         if k != 'silent':
             raise ValueError("the only allowed keyword argument is 'silent', "
@@ -209,7 +209,7 @@ def __getattr__(self, key):
             return _get_option(prefix)
 
     def __dir__(self):
-        return self.d.keys()
+        return list(self.d.keys())
 
 # For user convenience,  we'd like to have the available options described
 # in the docstring. For dev convenience we'd like to generate the docstrings
@@ -232,7 +232,7 @@ def __call__(self, *args, **kwds):
     @property
     def __doc__(self):
         opts_desc = _describe_option('all', _print_desc=False)
-        opts_list = pp_options_list(_registered_options.keys())
+        opts_list = pp_options_list(list(_registered_options.keys()))
         return self.__doc_tmpl__.format(opts_desc=opts_desc,
                                         opts_list=opts_list)
 
@@ -351,7 +351,7 @@ def __init__(self, *args):
            errmsg =  "Need to invoke as option_context(pat,val,[(pat,val),..))."
            raise AssertionError(errmsg)
 
-        ops = zip(args[::2], args[1::2])
+        ops = list(zip(args[::2], args[1::2]))
         undo = []
         for pat, val in ops:
             undo.append((pat, _get_option(pat, silent=True)))

pandas/core/frame.py

@@ -496,7 +496,7 @@ def _init_dict(self, data, index, columns, dtype=None):
             data = dict((k, v) for k, v in compat.iteritems(data) if k in columns)
 
             if index is None:
-                index = extract_index(data.values())
+                index = extract_index(list(data.values()))
             else:
                 index = _ensure_index(index)
 
@@ -521,9 +521,9 @@ def _init_dict(self, data, index, columns, dtype=None):
                 data_names.append(k)
                 arrays.append(v)
         else:
-            keys = data.keys()
+            keys = list(data.keys())
             if not isinstance(data, OrderedDict):
-                keys = _try_sort(data.keys())
+                keys = _try_sort(list(data.keys()))
             columns = data_names = Index(keys)
             arrays = [data[k] for k in columns]
 
@@ -954,10 +954,10 @@ def from_dict(cls, data, orient='columns', dtype=None):
         if orient == 'index':
             if len(data) > 0:
                 # TODO speed up Series case
-                if isinstance(data.values()[0], (Series, dict)):
+                if isinstance(list(data.values())[0], (Series, dict)):
                     data = _from_nested_dict(data)
                 else:
-                    data, index = data.values(), data.keys()
+                    data, index = list(data.values()), list(data.keys())
         elif orient != 'columns':  # pragma: no cover
             raise ValueError('only recognize index or columns for orient')
 
@@ -3600,7 +3600,7 @@ def replace(self, to_replace=None, value=None, inplace=False, limit=None,
                 to_replace = regex
                 regex = True
 
-            items = to_replace.items()
+            items = list(to_replace.items())
             keys, values = zip(*items)
 
             are_mappings = [isinstance(v, (dict, Series)) for v in values]
@@ -3615,8 +3615,8 @@ def replace(self, to_replace=None, value=None, inplace=False, limit=None,
                 value_dict = {}
 
                 for k, v in items:
-                    to_rep_dict[k] = v.keys()
-                    value_dict[k] = v.values()
+                    to_rep_dict[k] = list(v.keys())
+                    value_dict[k] = list(v.values())
 
                 to_replace, value = to_rep_dict, value_dict
             else:
@@ -5735,7 +5735,7 @@ def extract_index(data):
                 indexes.append(v.index)
             elif isinstance(v, dict):
                 have_dicts = True
-                indexes.append(v.keys())
+                indexes.append(list(v.keys()))
             elif isinstance(v, (list, tuple, np.ndarray)):
                 have_raw_arrays = True
                 raw_lengths.append(len(v))
@@ -5895,7 +5895,7 @@ def _list_of_series_to_arrays(data, columns, coerce_float=False, dtype=None):
 
 def _list_of_dict_to_arrays(data, columns, coerce_float=False, dtype=None):
     if columns is None:
-        gen = (x.keys() for x in data)
+        gen = (list(x.keys()) for x in data)
         columns = lib.fast_unique_multiple_list_gen(gen)
 
     # assure that they are of the base dict class and not of derived

pandas/core/groupby.py

@@ -1438,8 +1438,8 @@ def aggregate(self, func_or_funcs, *args, **kwargs):
 
     def _aggregate_multiple_funcs(self, arg):
         if isinstance(arg, dict):
-            columns = arg.keys()
-            arg = arg.items()
+            columns = list(arg.keys())
+            arg = list(arg.items())
         elif any(isinstance(x, (tuple, list)) for x in arg):
             arg = [(x, x) if not isinstance(x, (tuple, list)) else x
                    for x in arg]
@@ -1731,7 +1731,7 @@ def aggregate(self, arg, *args, **kwargs):
                     result[col] = colg.aggregate(agg_how)
                     keys.append(col)
 
-            if isinstance(result.values()[0], DataFrame):
+            if isinstance(list(result.values())[0], DataFrame):
                 from pandas.tools.merge import concat
                 result = concat([result[k] for k in keys], keys=keys, axis=1)
             else:

pandas/core/index.py

@@ -2704,7 +2704,7 @@ def _get_combined_index(indexes, intersect=False):
 
 
 def _get_distinct_indexes(indexes):
-    return dict((id(x), x) for x in indexes).values()
+    return list(dict((id(x), x) for x in indexes).values())
 
 
 def _union_indexes(indexes):

pandas/core/panel.py

@@ -284,7 +284,7 @@ def _init_dict(self, data, axes, dtype=None):
             data = OrderedDict((k, v) for k, v
                                in compat.iteritems(data) if k in haxis)
         else:
-            ks = data.keys()
+            ks = list(data.keys())
             if not isinstance(data,OrderedDict):
                 ks = _try_sort(ks)
             haxis = Index(ks)
@@ -360,7 +360,7 @@ def from_dict(cls, data, intersect=False, orient='items', dtype=None):
             raise ValueError('Orientation must be one of {items, minor}.')
 
         d = cls._homogenize_dict(cls, data, intersect=intersect, dtype=dtype)
-        ks = d['data'].keys()
+        ks = list(d['data'].keys())
         if not isinstance(d['data'],OrderedDict):
             ks = list(sorted(ks))
         d[cls._info_axis] = Index(ks)

pandas/core/reshape.py

@@ -543,9 +543,9 @@ def _stack_multi_columns(frame, level=-1, dropna=True):
 
     # tuple list excluding level for grouping columns
     if len(frame.columns.levels) > 2:
-        tuples = zip(*[lev.values.take(lab)
+        tuples = list(zip(*[lev.values.take(lab)
                        for lev, lab in zip(this.columns.levels[:-1],
-                                           this.columns.labels[:-1])])
+                                           this.columns.labels[:-1])]))
         unique_groups = [key for key, _ in itertools.groupby(tuples)]
         new_names = this.columns.names[:-1]
         new_columns = MultiIndex.from_tuples(unique_groups, names=new_names)
@@ -747,8 +747,8 @@ def lreshape(data, groups, dropna=True, label=None):
     reshaped : DataFrame
     """
     if isinstance(groups, dict):
-        keys = groups.keys()
-        values = groups.values()
+        keys = list(groups.keys())
+        values = list(groups.values())
     else:
         keys, values = zip(*groups)
 

pandas/core/strings.py

@@ -1,6 +1,7 @@
 import numpy as np
 
 from six.moves import zip
+import six
 from pandas.core.common import isnull
 from pandas.core.series import Series
 import re
@@ -282,16 +283,18 @@ def str_repeat(arr, repeats):
     if np.isscalar(repeats):
         def rep(x):
             try:
-                return str.__mul__(x, repeats)
+                return six.binary_type.__mul__(x, repeats)
             except TypeError:
                 return six.text_type.__mul__(x, repeats)
+
         return _na_map(rep, arr)
     else:
         def rep(x, r):
             try:
-                return str.__mul__(x, r)
+                return six.binary_type.__mul__(x, r)
             except TypeError:
                 return six.text_type.__mul__(x, r)
+
         repeats = np.asarray(repeats, dtype=object)
         result = lib.vec_binop(arr, repeats, rep)
         return result

pandas/io/common.py

@@ -10,21 +10,23 @@
 
 if py3compat.PY3:
     from urllib.request import urlopen
+    _urlopen = urlopen
     from urllib.parse import urlparse as parse_url
     import urllib.parse as compat_parse
-    from urllib.parse import uses_relative, uses_netloc, uses_params
+    from urllib.parse import uses_relative, uses_netloc, uses_params, urlencode
     from urllib.error import URLError
     from http.client import HTTPException
 else:
     from urllib2 import urlopen as _urlopen
+    from urllib import urlencode
     from urlparse import urlparse as parse_url
     from urlparse import uses_relative, uses_netloc, uses_params
     from urllib2 import URLError
     from httplib import HTTPException
     from contextlib import contextmanager, closing
     from functools import wraps
 
-    @wraps(_urlopen)
+    # @wraps(_urlopen)
     @contextmanager
     def urlopen(*args, **kwargs):
         with closing(_urlopen(*args, **kwargs)) as f:
@@ -80,8 +82,7 @@ def get_filepath_or_buffer(filepath_or_buffer, encoding=None):
     """
 
     if _is_url(filepath_or_buffer):
-        from urllib2 import urlopen
-        filepath_or_buffer = urlopen(filepath_or_buffer)
+        req = _urlopen(filepath_or_buffer)
         if py3compat.PY3:  # pragma: no cover
             if encoding:
                 errors = 'strict'
@@ -101,7 +102,7 @@ def get_filepath_or_buffer(filepath_or_buffer, encoding=None):
             raise ImportError("boto is required to handle s3 files")
         # Assuming AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
         # are environment variables
-        parsed_url = urlparse.urlparse(filepath_or_buffer)
+        parsed_url = parse_url(filepath_or_buffer)
         conn = boto.connect_s3()
         b = conn.get_bucket(parsed_url.netloc)
         k = boto.s3.key.Key(b)

pandas/io/data.py

@@ -6,7 +6,6 @@
 import warnings
 import tempfile
 import datetime as dt
-import urllib
 import time
 
 from collections import defaultdict
@@ -17,7 +16,7 @@
 from pandas import Panel, DataFrame, Series, read_csv, concat
 from pandas.core.common import PandasError
 from pandas.io.parsers import TextParser
-from pandas.io.common import urlopen, ZipFile
+from pandas.io.common import urlopen, ZipFile, urlencode
 from pandas.util.testing import _network_error_classes
 import six
 from six.moves import map, zip
@@ -115,7 +114,7 @@ def get_quote_yahoo(symbols):
 
     # for codes see: http://www.gummy-stuff.org/Yahoo-data.htm
     request = ''.join(six.itervalues(_yahoo_codes))  # code request string
-    header = _yahoo_codes.keys()
+    header = list(_yahoo_codes.keys())
 
     data = defaultdict(list)
 
@@ -202,7 +201,7 @@ def _get_hist_google(sym, start, end, retry_count, pause):
     google_URL = 'http://www.google.com/finance/historical?'
 
     # www.google.com/finance/historical?q=GOOG&startdate=Jun+9%2C+2011&enddate=Jun+8%2C+2013&output=csv
-    url = google_URL + urllib.urlencode({"q": sym,
+    url = google_URL + urlencode({"q": sym,
                                          "startdate": start.strftime('%b %d, '
                                                                      '%Y'),
                                          "enddate": end.strftime('%b %d, %Y'),

pandas/io/excel.py

@@ -5,14 +5,14 @@
 #----------------------------------------------------------------------
 # ExcelFile class
 
-from pandas.util.py3compat import range
 import datetime
 import numpy as np
 
 from pandas.io.parsers import TextParser
 from pandas.tseries.period import Period
 from pandas import json
 from six.moves import map, zip, reduce
+from pandas.util.py3compat import range
 import six
 
 def read_excel(path_or_buf, sheetname, kind=None, **kwds):