ENH: Int64Index class, Cython codegen, refactoring to enable transparent use

Author: wesm

pandas/core/daterange.py

@@ -295,6 +295,9 @@ def union(self, other):
         else:
             return Index.union(self, other)
 
+    def _wrap_union_result(self, other, result):
+        return Index(result)
+
     def tz_normalize(self, tz):
         """
         Convert DateRange from one time zone to another (using pytz)

pandas/core/frame.py

@@ -1575,7 +1575,8 @@ def combine(self, other, func, fill_value=None):
             this = self.reindex(new_index)
             other = other.reindex(new_index)
 
-        new_columns = _try_sort(set(this.columns + other.columns))
+        # sorts if possible
+        new_columns = this.columns.union(other.columns)
         do_fill = fill_value is not None
 
         result = {}

pandas/core/groupby.py

@@ -115,7 +115,7 @@ def groups(self):
             to_groupby = Index(to_groupby)
 
             axis = self.obj._get_axis(self.axis)
-            self._groups = _tseries.groupby(axis, to_groupby)
+            self._groups = axis.groupby(to_groupby)
 
         return self._groups
 
@@ -477,7 +477,7 @@ def __init__(self, index, grouper=None, name=None, level=None):
             else:
                 self.grouper = labels
 
-        self.index = index.values
+        self.index = np.asarray(index.values, dtype=object)
 
         # no level passed
         if not isinstance(self.grouper, np.ndarray):

pandas/core/index.py

@@ -42,7 +42,7 @@ class Index(np.ndarray):
     """
     def __new__(cls, data, dtype=None, copy=False):
         if isinstance(data, np.ndarray):
-            if issubclass(data.dtype.type, np.integer):
+            if dtype is None and issubclass(data.dtype.type, np.integer):
                 subarr = data.astype(np.int64)
                 return subarr.view(Int64Index)
             subarr = np.array(data, dtype=object, copy=copy)
@@ -57,27 +57,33 @@ def __new__(cls, data, dtype=None, copy=False):
             subarr[:] = data
         return subarr.view(cls)
 
+    @property
+    def dtype(self):
+        return np.dtype('object')
+
     def summary(self):
         if len(self) > 0:
             index_summary = ', %s to %s' % (str(self[0]), str(self[-1]))
         else:
             index_summary = ''
-        return 'Index: %s entries%s' % (len(self), index_summary)
+
+        name = type(self).__name__
+        return '%s: %s entries%s' % (name, len(self), index_summary)
 
     @property
     def values(self):
         return np.asarray(self)
 
     @cache_readonly
     def is_monotonic(self):
-        return lib.is_monotonic(self)
+        return lib.is_monotonic_object(self)
 
     _indexMap = None
     @property
     def indexMap(self):
         "{label -> location}"
         if self._indexMap is None:
-            self._indexMap = lib.map_indices_buf(self)
+            self._indexMap = lib.map_indices_object(self)
             self._verify_integrity()
 
         return self._indexMap
@@ -258,7 +264,14 @@ def union(self, other):
         except Exception:
             pass
 
-        return Index(result)
+        # for subclasses
+        return self._wrap_union_result(other, result)
+
+    def _wrap_union_result(self, other, result):
+        if type(self) == type(other):
+            return type(self)(result)
+        else:
+            return Index(result)
 
     def intersection(self, other):
         """
@@ -354,16 +367,24 @@ def get_indexer(self, target, method=None):
 
         method = self._get_method(method)
 
+        if self.dtype != target.dtype:
+            target = Index(target, dtype=object)
+
         if method == 'pad':
-            indexer = lib.pad(self, target, self.indexMap, target.indexMap)
+            indexer = lib.pad_object(self, target, self.indexMap,
+                                     target.indexMap)
         elif method == 'backfill':
-            indexer = lib.backfill(self, target, self.indexMap, target.indexMap)
+            indexer = lib.backfill_object(self, target, self.indexMap,
+                                          target.indexMap)
         elif method is None:
-            indexer = lib.merge_indexer(target, self.indexMap)
+            indexer = lib.merge_indexer_object(target, self.indexMap)
         else:
             raise ValueError('unrecognized method: %s' % method)
         return indexer
 
+    def groupby(self, to_groupby):
+        return lib.groupby(self, to_groupby)
+
     def _get_method(self, method):
         if method:
             method = method.lower()
@@ -487,6 +508,23 @@ def copy(self, order='C'):
 
 class Int64Index(Index):
 
+    def __new__(cls, data, dtype=None, copy=False):
+        if isinstance(data, np.ndarray):
+            subarr = np.array(data, dtype=np.int64, copy=copy)
+        elif np.isscalar(data):
+            raise ValueError('Index(...) must be called with a collection '
+                             'of some kind, %s was passed' % repr(data))
+        else:
+            # other iterable of some kind
+            if not isinstance(data, (list, tuple)):
+                data = list(data)
+            subarr = np.asarray(data, dtype=np.int64)
+        return subarr.view(cls)
+
+    @property
+    def dtype(self):
+        return np.dtype('int64')
+
     @cache_readonly
     def is_monotonic(self):
         return lib.is_monotonic_int64(self)
@@ -513,26 +551,36 @@ def equals(self, other):
         if self is other:
             return True
 
-        if not isinstance(other, Int64Index):
-            return False
+        # if not isinstance(other, Int64Index):
+        #     return False
 
         return np.array_equal(self, other)
 
     def get_indexer(self, target, method=None):
         target = _ensure_index(target)
 
+        if self.dtype != target.dtype:
+            this = Index(self, dtype=object)
+            target = Index(target, dtype=object)
+            return this.get_indexer(target, method=method)
+
         method = self._get_method(method)
 
         if method == 'pad':
-            indexer = lib.pad(self, target, self.indexMap, target.indexMap)
+            indexer = lib.pad_int64(self, target, self.indexMap,
+                                    target.indexMap)
         elif method == 'backfill':
-            indexer = lib.backfill(self, target, self.indexMap, target.indexMap)
+            indexer = lib.backfill_int64(self, target, self.indexMap,
+                                         target.indexMap)
         elif method is None:
             indexer = lib.merge_indexer_int64(target, self.indexMap)
         else:
             raise ValueError('unrecognized method: %s' % method)
         return indexer
-    get_indexer.__doc__ = Index.get_indexer__doc__
+    get_indexer.__doc__ = Index.get_indexer.__doc__
+
+    def groupby(self, to_groupby):
+        return lib.groupby(self.values.astype(object), to_groupby)
 
 class DateIndex(Index):
     pass
@@ -965,13 +1013,14 @@ def get_indexer(self, target, method=None):
         self_index = self.get_tuple_index()
 
         if method == 'pad':
-            indexer = lib.pad(self_index, target_index, self_index.indexMap,
-                              target.indexMap)
+            indexer = lib.pad_object(self_index, target_index,
+                                     self_index.indexMap, target.indexMap)
         elif method == 'backfill':
-            indexer = lib.backfill(self_index, target_index, self_index.indexMap,
-                                   target.indexMap)
+            indexer = lib.backfill_object(self_index, target_index,
+                                          self_index.indexMap, target.indexMap)
         else:
-            indexer = lib.merge_indexer(target_index, self_index.indexMap)
+            indexer = lib.merge_indexer_object(target_index,
+                                               self_index.indexMap)
         return indexer
 
     def reindex(self, target, method=None):

pandas/core/internals.py

@@ -650,7 +650,8 @@ def join_on(self, other, on, axis=1, lsuffix=None, rsuffix=None):
         this, other = self._maybe_rename_join(other, lsuffix, rsuffix)
 
         other_axis = other.axes[axis]
-        indexer = lib.merge_indexer(on.astype(object), other_axis.indexMap)
+        indexer = lib.merge_indexer_object(on.astype(object),
+                                           other_axis.indexMap)
 
         # TODO: deal with length-0 case? or does it fall out?
         mask = indexer == -1

pandas/core/series.py

@@ -1239,8 +1239,8 @@ def map(self, arg):
             if isinstance(arg, dict):
                 arg = Series(arg)
 
-            indexer = lib.merge_indexer(self.values.astype(object),
-                                        arg.index.indexMap)
+            indexer = lib.merge_indexer_object(self.values.astype(object),
+                                               arg.index.indexMap)
 
             new_values = common.take_1d(np.asarray(arg), indexer)
             return Series(new_values, index=self.index)

pandas/io/pytables.py

@@ -663,12 +663,12 @@ def _read_panel_table(self, group, where=None):
 
             # need a better algorithm
             tuple_index = long_index.get_tuple_index()
-            index_map = lib.map_indices_buf(tuple_index)
+            index_map = lib.map_indices_object(tuple_index)
 
             unique_tuples = lib.fast_unique(tuple_index)
             unique_tuples = _asarray_tuplesafe(unique_tuples)
 
-            indexer = lib.merge_indexer(unique_tuples, index_map)
+            indexer = lib.merge_indexer_object(unique_tuples, index_map)
 
             new_index = long_index.take(indexer)
             new_values = lp.values.take(indexer, axis=0)

pandas/src/common.pyx

@@ -80,76 +80,6 @@ PyDateTime_IMPORT
 # initialize numpy
 import_array()
 
-cpdef map_indices(ndarray index):
-    '''
-    Produce a dict mapping the values of the input array to their respective
-    locations.
-
-    Example:
-        array(['hi', 'there']) --> {'hi' : 0 , 'there' : 1}
-
-    Better to do this with Cython because of the enormous speed boost.
-    '''
-    cdef int i, length
-    cdef flatiter iter
-    cdef dict result
-    cdef object idx
-
-    result = {}
-
-    iter = <flatiter> PyArray_IterNew(index)
-    length = PyArray_SIZE(index)
-
-    for i from 0 <= i < length:
-        idx = PyArray_GETITEM(index, PyArray_ITER_DATA(iter))
-        result[idx] = i
-        PyArray_ITER_NEXT(iter)
-
-    return result
-
-@cython.wraparound(False)
-@cython.boundscheck(False)
-cpdef map_indices_buf(ndarray[object] index):
-    '''
-    Produce a dict mapping the values of the input array to their respective
-    locations.
-
-    Example:
-        array(['hi', 'there']) --> {'hi' : 0 , 'there' : 1}
-
-    Better to do this with Cython because of the enormous speed boost.
-    '''
-    cdef Py_ssize_t i, length
-    cdef dict result = {}
-
-    length = len(index)
-
-    for i from 0 <= i < length:
-        result[index[i]] = i
-
-    return result
-
-@cython.wraparound(False)
-@cython.boundscheck(False)
-cpdef map_indices_int64(ndarray[int64_t] index):
-    '''
-    Produce a dict mapping the values of the input array to their respective
-    locations.
-
-    Example:
-        array(['hi', 'there']) --> {'hi' : 0 , 'there' : 1}
-
-    Better to do this with Cython because of the enormous speed boost.
-    '''
-    cdef Py_ssize_t i, length
-    cdef dict result = {}
-
-    length = len(index)
-
-    for i from 0 <= i < length:
-        result[index[i]] = i
-
-    return result
 
 cpdef map_indices_list(list index):
     '''
@@ -171,6 +101,7 @@ cpdef map_indices_list(list index):
 
     return result
 
+
 from libc.stdlib cimport malloc, free
 
 cdef class MultiMap:
@@ -235,40 +166,3 @@ def isAllDates(ndarray[object, ndim=1] arr):
             return False
 
     return True
-
-def is_monotonic(ndarray[object] arr):
-    cdef:
-        Py_ssize_t i, n
-        object prev, cur
-
-    n = len(arr)
-
-    if n < 2:
-        return True
-
-    prev = arr[0]
-    for i from 1 <= i < n:
-        cur = arr[i]
-        if cur < prev:
-            return False
-        prev = cur
-    return True
-
-
-def is_monotonic_int64(ndarray[int64_t] arr):
-    cdef:
-        Py_ssize_t i, n
-        int64_t prev, cur
-
-    n = len(arr)
-
-    if n < 2:
-        return True
-
-    prev = arr[0]
-    for i from 1 <= i < n:
-        cur = arr[i]
-        if cur < prev:
-            return False
-        prev = cur
-    return True