introducing UInt8HashTable and Int8HashTable

Author: realead

pandas/_libs/hashtable.pxd

@@ -3,19 +3,23 @@ from numpy cimport intp_t, ndarray
 from pandas._libs.khash cimport (
     float32_t,
     float64_t,
+    int8_t,
     int16_t,
     int32_t,
     int64_t,
     kh_float32_t,
     kh_float64_t,
+    kh_int8_t,
     kh_int16_t,
     kh_int32_t,
     kh_int64_t,
     kh_pymap_t,
     kh_str_t,
+    kh_uint8_t,
     kh_uint16_t,
     kh_uint32_t,
     kh_uint64_t,
+    uint8_t,
     uint16_t,
     uint32_t,
     uint64_t,
@@ -62,6 +66,18 @@ cdef class Int16HashTable(HashTable):
     cpdef get_item(self, int16_t val)
     cpdef set_item(self, int16_t key, Py_ssize_t val)
 
+cdef class UInt8HashTable(HashTable):
+    cdef kh_uint8_t *table
+
+    cpdef get_item(self, uint8_t val)
+    cpdef set_item(self, uint8_t key, Py_ssize_t val)
+
+cdef class Int8HashTable(HashTable):
+    cdef kh_int8_t *table
+
+    cpdef get_item(self, int8_t val)
+    cpdef set_item(self, int8_t key, Py_ssize_t val)
+
 cdef class Float64HashTable(HashTable):
     cdef kh_float64_t *table
 

pandas/_libs/hashtable_class_helper.pxi.in

@@ -10,12 +10,14 @@ WARNING: DO NOT edit .pxi FILE directly, .pxi is generated from .pxi.in
 # name
 cimported_types = ['float32',
                    'float64',
+                   'int8',
                    'int16',
                    'int32',
                    'int64',
                    'pymap',
                    'str',
                    'strbox',
+                   'uint8',
                    'uint16',
                    'uint32',
                    'uint64']
@@ -51,10 +53,12 @@ dtypes = [('Float64', 'float64', 'float64_t'),
           ('Int64', 'int64', 'int64_t'),
           ('Int32', 'int32', 'int32_t'),
           ('Int16', 'int16', 'int16_t'),
+          ('Int8', 'int8', 'int8_t'),
           ('String', 'string', 'char *'),
           ('UInt64', 'uint64', 'uint64_t'),
           ('UInt32', 'uint32', 'uint32_t'),
-          ('UInt16', 'uint16', 'uint16_t')]
+          ('UInt16', 'uint16', 'uint16_t'),
+          ('UInt8', 'uint8', 'uint8_t')]
 }}
 
 {{for name, dtype, c_type in dtypes}}
@@ -83,9 +87,11 @@ ctypedef fused vector_data:
     Int64VectorData
     Int32VectorData
     Int16VectorData
+    Int8VectorData
     UInt64VectorData
     UInt32VectorData
     UInt16VectorData
+    UInt8VectorData
     Float64VectorData
     Float32VectorData
     StringVectorData
@@ -107,7 +113,9 @@ dtypes = [('Float64', 'float64', 'float64_t'),
           ('UInt32', 'uint32', 'uint32_t'),
           ('Int32', 'int32', 'int32_t'),
           ('UInt16', 'uint16', 'uint16_t'),
-          ('Int16', 'int16', 'int16_t')]
+          ('Int16', 'int16', 'int16_t'),
+          ('UInt8', 'uint8', 'uint8_t'),
+          ('Int8', 'int8', 'int8_t')]
 
 }}
 
@@ -303,7 +311,9 @@ dtypes = [('Float64', 'float64', True, 'np.nan'),
           ('UInt32', 'uint32', False, 0),
           ('Int32', 'int32', False, 0),
           ('UInt16', 'uint16', False, 0),
-          ('Int16', 'int16', False, 0)]
+          ('Int16', 'int16', False, 0),
+          ('UInt8', 'uint8', False, 0),
+          ('Int8', 'int8', False, 0)]
 
 }}
 

pandas/_libs/hashtable_func_helper.pxi.in

@@ -12,10 +12,12 @@ dtypes = [('float64', 'float64', 'float64_t'),
           ('uint64', 'uint64', 'uint64_t'),
           ('uint32', 'uint32', 'uint32_t'),
           ('uint16', 'uint16', 'uint16_t'),
+          ('uint8', 'uint8', 'uint8_t'),
           ('object', 'pymap', 'object'),
           ('int64', 'int64', 'int64_t'),
           ('int32', 'int32', 'int32_t'),
-          ('int16', 'int16', 'int16_t')]
+          ('int16', 'int16', 'int16_t'),
+          ('int8', 'int8', 'int8_t')]
 
 }}
 
@@ -284,9 +286,11 @@ dtypes = [('float64', 'float64_t', 'float64', 'float64'),
           ('int64', 'int64_t', 'int64', 'int64'),
           ('int32', 'int32_t', 'int32', 'int32'),
           ('int16', 'int16_t', 'int16', 'int16'),
+          ('int8', 'int8_t', 'int8', 'int8'),
           ('uint64', 'uint64_t', 'uint64', 'uint64'),
           ('uint32', 'uint32_t', 'uint32', 'uint32'),
           ('uint16', 'uint16_t', 'uint16', 'uint16'),
+          ('uint8', 'uint8_t', 'uint8', 'uint8'),
           ('object', 'object', 'pymap', 'object_')]
 }}
 

pandas/_libs/khash.pxd

@@ -2,9 +2,11 @@ from cpython.object cimport PyObject
 from numpy cimport (
     float32_t,
     float64_t,
+    int8_t,
     int16_t,
     int32_t,
     int64_t,
+    uint8_t,
     uint16_t,
     uint32_t,
     uint64_t,

pandas/_libs/khash_for_primitive_helper.pxi.in

@@ -15,6 +15,8 @@ primitive_types = [('int64', 'int64_t'),
                    ('float32', 'float32_t'),
                    ('int16', 'int16_t'),
                    ('uint16', 'uint16_t'),
+                   ('int8', 'int8_t'),
+                   ('uint8', 'uint8_t'),
                   ]
 }}
 

pandas/_libs/src/klib/khash.h

@@ -135,6 +135,10 @@ typedef unsigned int khint16_t;
 typedef unsigned short khint16_t;
 #endif
 
+#if UCHAR_MAX == 0xffu
+typedef unsigned char khint8_t;
+#endif
+
 typedef double khfloat64_t;
 typedef double khfloat32_t;
 
@@ -633,6 +637,18 @@ PANDAS_INLINE khint_t __ac_Wang_hash(khint_t key)
 #define KHASH_MAP_INIT_UINT16(name, khval_t)								\
 	KHASH_INIT(name, khint16_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
 
+/*! @function
+  @abstract     Instantiate a hash map containing 8bit-integer keys
+  @param  name  Name of the hash table [symbol]
+  @param  khval_t  Type of values [type]
+ */
+#define KHASH_MAP_INIT_INT8(name, khval_t)								\
+	KHASH_INIT(name, khint8_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
+
+#define KHASH_MAP_INIT_UINT8(name, khval_t)								\
+	KHASH_INIT(name, khint8_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
+
+
 
 typedef const char *kh_cstr_t;
 /*! @function
@@ -660,6 +676,8 @@ typedef const char *kh_cstr_t;
 #define kh_exist_uint32(h, k) (kh_exist(h, k))
 #define kh_exist_int16(h, k) (kh_exist(h, k))
 #define kh_exist_uint16(h, k) (kh_exist(h, k))
+#define kh_exist_int8(h, k) (kh_exist(h, k))
+#define kh_exist_uint8(h, k) (kh_exist(h, k))
 
 KHASH_MAP_INIT_STR(str, size_t)
 KHASH_MAP_INIT_INT(int32, size_t)
@@ -668,6 +686,8 @@ KHASH_MAP_INIT_INT64(int64, size_t)
 KHASH_MAP_INIT_UINT64(uint64, size_t)
 KHASH_MAP_INIT_INT16(int16, size_t)
 KHASH_MAP_INIT_UINT16(uint16, size_t)
+KHASH_MAP_INIT_INT16(int8, size_t)
+KHASH_MAP_INIT_UINT16(uint8, size_t)
 
 
 #endif /* __AC_KHASH_H */

pandas/tests/libs/test_hashtable.py

@@ -17,6 +17,8 @@
         (ht.Float32HashTable, np.float32),
         (ht.Int16HashTable, np.int16),
         (ht.UInt16HashTable, np.uint16),
+        (ht.Int8HashTable, np.int8),
+        (ht.UInt8HashTable, np.uint8),
     ],
 )
 class TestHashTable:
@@ -77,7 +79,10 @@ def test_lookup(self, table_type, dtype):
         tm.assert_numpy_array_equal(result, expected)
 
     def test_lookup_wrong(self, table_type, dtype):
-        N = 512
+        if dtype in (np.int8, np.uint8):
+            N = 100
+        else:
+            N = 512
         table = table_type()
         keys = (np.arange(N) + N).astype(dtype)
         table.map_locations(keys)
@@ -86,7 +91,10 @@ def test_lookup_wrong(self, table_type, dtype):
         assert np.all(result == -1)
 
     def test_unique(self, table_type, dtype):
-        N = 1000
+        if dtype in (np.int8, np.uint8):
+            N = 88
+        else:
+            N = 1000
         table = table_type()
         expected = (np.arange(N) + N).astype(dtype)
         keys = np.repeat(expected, 5)
@@ -157,6 +165,8 @@ def get_ht_function(fun_name, type_suffix):
         (np.float32, "float32"),
         (np.int16, "int16"),
         (np.uint16, "uint16"),
+        (np.int8, "int8"),
+        (np.uint8, "uint8"),
     ],
 )
 class TestHelpFunctions:
@@ -197,7 +207,10 @@ def test_ismember_no(self, dtype, type_suffix):
         tm.assert_numpy_array_equal(result, expected)
 
     def test_mode(self, dtype, type_suffix):
-        N = 11111
+        if dtype in (np.int8, np.uint8):
+            N = 53
+        else:
+            N = 11111
         mode = get_ht_function("mode", type_suffix)
         values = np.repeat(np.arange(N).astype(dtype), 5)
         values[0] = 42