gh-90751: memoryview now supports half-float (#96738)

Co-authored-by: Antoine Pitrou <[email protected]>

Author: corona10

Doc/whatsnew/3.12.rst

@@ -94,6 +94,9 @@ Other Language Changes
   length limitation <int_max_str_digits>` documentation.  The default limit
   is 4300 digits in string form.
 
+* :class:`memoryview` now supports the half-float type (the "e" format code).
+  (Contributed by Dong-hee Na and Antoine Pitrou in :gh:`90751`.)
+
 
 New Modules
 ===========

Lib/test/test_buffer.py

@@ -64,7 +64,7 @@
     '?':0, 'c':0, 'b':0, 'B':0,
     'h':0, 'H':0, 'i':0, 'I':0,
     'l':0, 'L':0, 'n':0, 'N':0,
-    'f':0, 'd':0, 'P':0
+    'e':0, 'f':0, 'd':0, 'P':0
 }
 
 # NumPy does not have 'n' or 'N':
@@ -89,7 +89,8 @@
     'i':(-(1<<31), 1<<31), 'I':(0, 1<<32),
     'l':(-(1<<31), 1<<31), 'L':(0, 1<<32),
     'q':(-(1<<63), 1<<63), 'Q':(0, 1<<64),
-    'f':(-(1<<63), 1<<63), 'd':(-(1<<1023), 1<<1023)
+    'e':(-65519, 65520),   'f':(-(1<<63), 1<<63),
+    'd':(-(1<<1023), 1<<1023)
 }
 
 def native_type_range(fmt):
@@ -98,6 +99,8 @@ def native_type_range(fmt):
         lh = (0, 256)
     elif fmt == '?':
         lh = (0, 2)
+    elif fmt == 'e':
+        lh = (-65519, 65520)
     elif fmt == 'f':
         lh = (-(1<<63), 1<<63)
     elif fmt == 'd':
@@ -125,7 +128,10 @@ def native_type_range(fmt):
     for fmt in fmtdict['@']:
         fmtdict['@'][fmt] = native_type_range(fmt)
 
+# Format codes suppported by the memoryview object
 MEMORYVIEW = NATIVE.copy()
+
+# Format codes suppported by array.array
 ARRAY = NATIVE.copy()
 for k in NATIVE:
     if not k in "bBhHiIlLfd":
@@ -164,7 +170,7 @@ def randrange_fmt(mode, char, obj):
             x = b'\x01'
     if char == '?':
         x = bool(x)
-    if char == 'f' or char == 'd':
+    if char in 'efd':
         x = struct.pack(char, x)
         x = struct.unpack(char, x)[0]
     return x
@@ -2246,7 +2252,7 @@ def test_py_buffer_to_contiguous(self):
         ###
         ###    Fortran output:
         ###    ---------------
-        ###       >>> fortran_buf = nd.tostring(order='F')
+        ###       >>> fortran_buf = nd.tobytes(order='F')
         ###       >>> fortran_buf
         ###       b'\x00\x04\x08\x01\x05\t\x02\x06\n\x03\x07\x0b'
         ###
@@ -2289,7 +2295,7 @@ def test_py_buffer_to_contiguous(self):
                 self.assertEqual(memoryview(y), memoryview(nd))
 
                 if numpy_array:
-                    self.assertEqual(b, na.tostring(order='C'))
+                    self.assertEqual(b, na.tobytes(order='C'))
 
             # 'F' request
             if f == 0: # 'C' to 'F'
@@ -2312,7 +2318,7 @@ def test_py_buffer_to_contiguous(self):
                 self.assertEqual(memoryview(y), memoryview(nd))
 
                 if numpy_array:
-                    self.assertEqual(b, na.tostring(order='F'))
+                    self.assertEqual(b, na.tobytes(order='F'))
 
             # 'A' request
             if f == ND_FORTRAN:
@@ -2336,7 +2342,7 @@ def test_py_buffer_to_contiguous(self):
                 self.assertEqual(memoryview(y), memoryview(nd))
 
                 if numpy_array:
-                    self.assertEqual(b, na.tostring(order='A'))
+                    self.assertEqual(b, na.tobytes(order='A'))
 
         # multi-dimensional, non-contiguous input
         nd = ndarray(list(range(12)), shape=[3, 4], flags=ND_WRITABLE|ND_PIL)

Lib/test/test_memoryview.py

@@ -13,6 +13,7 @@
 import io
 import copy
 import pickle
+import struct
 
 from test.support import import_helper
 
@@ -527,6 +528,14 @@ def test_ctypes_cast(self):
                 m[2:] = memoryview(p6).cast(format)[2:]
                 self.assertEqual(d.value, 0.6)
 
+    def test_half_float(self):
+        half_data = struct.pack('eee', 0.0, -1.5, 1.5)
+        float_data = struct.pack('fff', 0.0, -1.5, 1.5)
+        half_view = memoryview(half_data).cast('e')
+        float_view = memoryview(float_data).cast('f')
+        self.assertEqual(half_view.nbytes * 2, float_view.nbytes)
+        self.assertListEqual(half_view.tolist(), float_view.tolist())
+
     def test_memoryview_hex(self):
         # Issue #9951: memoryview.hex() segfaults with non-contiguous buffers.
         x = b'0' * 200000

Misc/NEWS.d/next/Core and Builtins/2022-09-11-00-37-50.gh-issue-90751.VE8-zf.rst

@@ -0,0 +1,2 @@
+:class:`memoryview` now supports half-floats.
+Patch by Dong-hee Na and Antoine Pitrou.

Objects/memoryobject.c

@@ -1135,6 +1135,7 @@ get_native_fmtchar(char *result, const char *fmt)
     case 'n': case 'N': size = sizeof(Py_ssize_t); break;
     case 'f': size = sizeof(float); break;
     case 'd': size = sizeof(double); break;
+    case 'e': size = sizeof(float) / 2; break;
     case '?': size = sizeof(_Bool); break;
     case 'P': size = sizeof(void *); break;
     }
@@ -1178,6 +1179,7 @@ get_native_fmtstr(const char *fmt)
     case 'N': RETURN("N");
     case 'f': RETURN("f");
     case 'd': RETURN("d");
+    case 'e': RETURN("e");
     case '?': RETURN("?");
     case 'P': RETURN("P");
     }
@@ -1697,6 +1699,12 @@ unpack_single(PyMemoryViewObject *self, const char *ptr, const char *fmt)
 
     CHECK_RELEASED_AGAIN(self);
 
+#if PY_LITTLE_ENDIAN
+    int endian = 1;
+#else
+    int endian = 0;
+#endif
+
     switch (fmt[0]) {
 
     /* signed integers and fast path for 'B' */
@@ -1725,6 +1733,7 @@ unpack_single(PyMemoryViewObject *self, const char *ptr, const char *fmt)
     /* floats */
     case 'f': UNPACK_SINGLE(d, ptr, float); goto convert_double;
     case 'd': UNPACK_SINGLE(d, ptr, double); goto convert_double;
+    case 'e': d = PyFloat_Unpack2(ptr, endian); goto convert_double;
 
     /* bytes object */
     case 'c': goto convert_bytes;
@@ -1786,6 +1795,11 @@ pack_single(PyMemoryViewObject *self, char *ptr, PyObject *item, const char *fmt
     double d;
     void *p;
 
+#if PY_LITTLE_ENDIAN
+    int endian = 1;
+#else
+    int endian = 0;
+#endif
     switch (fmt[0]) {
     /* signed integers */
     case 'b': case 'h': case 'i': case 'l':
@@ -1862,17 +1876,22 @@ pack_single(PyMemoryViewObject *self, char *ptr, PyObject *item, const char *fmt
         break;
 
     /* floats */
-    case 'f': case 'd':
+    case 'f': case 'd': case 'e':
         d = PyFloat_AsDouble(item);
         if (d == -1.0 && PyErr_Occurred())
             goto err_occurred;
         CHECK_RELEASED_INT_AGAIN(self);
         if (fmt[0] == 'f') {
             PACK_SINGLE(ptr, d, float);
         }
-        else {
+        else if (fmt[0] == 'd') {
             PACK_SINGLE(ptr, d, double);
         }
+        else {
+            if (PyFloat_Pack2(d, ptr, endian) < 0) {
+                goto err_occurred;
+            }
+        }
         break;
 
     /* bool */
@@ -1882,7 +1901,7 @@ pack_single(PyMemoryViewObject *self, char *ptr, PyObject *item, const char *fmt
             return -1; /* preserve original error */
         CHECK_RELEASED_INT_AGAIN(self);
         PACK_SINGLE(ptr, ld, _Bool);
-         break;
+        break;
 
     /* bytes object */
     case 'c':
@@ -2748,6 +2767,17 @@ unpack_cmp(const char *p, const char *q, char fmt,
     /* XXX DBL_EPSILON? */
     case 'f': CMP_SINGLE(p, q, float); return equal;
     case 'd': CMP_SINGLE(p, q, double); return equal;
+    case 'e': {
+#if PY_LITTLE_ENDIAN
+        int endian = 1;
+#else
+        int endian = 0;
+#endif
+        /* Note: PyFloat_Unpack2 should never fail */
+        double u = PyFloat_Unpack2(p, endian);
+        double v = PyFloat_Unpack2(q, endian);
+        return (u == v);
+    }
 
     /* bytes object */
     case 'c': return *p == *q;