pythongh-46376: add (failing) tests for ctypes/pointer/cast/set_contents

Previous attempt to fix pythongh-46376 was incomplete and
overall it didn't succeed, and was reverted in 3.11.
However, we have discovered some dangerous issues with
ctypes, that aren't fixed or documented anywhere.
This commit adds tests (@expectedfailure) so at least
developers are aware of situations where memory might
be corrupted, leaked or when changing a pointer value
might have no effect.
Hopefully, we should be able to remove @expectedfailure
in the future, with new shiny ctypes implementation or
at least a bugfix.

Author: s1kpp

.gitignore

@@ -124,6 +124,8 @@ Tools/unicode/data/
 /config.status.lineno
 # hendrikmuhs/ccache-action@v1
 /.ccache
+# clangd and it's debugger
+/.cache
 /platform
 /profile-clean-stamp
 /profile-run-stamp

Lib/test/test_ctypes/test_arrays.py

@@ -1,8 +1,10 @@
+import gc
 import ctypes
 import sys
 import unittest
 import warnings
-from ctypes import (Structure, Array, sizeof, addressof,
+import weakref
+from ctypes import (Structure, Array, sizeof, addressof, POINTER, pointer,
                     create_string_buffer, create_unicode_buffer,
                     c_char, c_wchar, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint,
                     c_long, c_ulonglong, c_float, c_double, c_longdouble)
@@ -249,6 +251,32 @@ def test_deprecation(self):
         with self.assertWarns(DeprecationWarning):
             CharArray = ctypes.ARRAY(c_char, 3)
 
+    def test_ptr_reuse(self):
+        w = weakref.WeakValueDictionary()
+        arr = 3 * POINTER(c_short)
+
+        vals = arr(
+            pointer(w.setdefault(10, c_short(10))),
+            pointer(w.setdefault(11, c_short(11))),
+            None,
+        )
+        gc.collect()
+
+        self.assertEqual(vals[0].contents.value, 10)
+        self.assertEqual(vals[1].contents.value, 11)
+
+        vals[2] = vals[0]
+
+        self.assertEqual(vals[2].contents.value, 10)
+
+        vals[2][0] = w.setdefault(12, c_short(12))
+        vals[2].contents = w.setdefault(13, c_short(13))
+
+        gc.collect()
+
+        self.assertEqual(vals[2].contents.value, 13)
+        self.assertEqual(vals[0].contents.value, 12)
+
 
 if __name__ == '__main__':
     unittest.main()

Lib/test/test_ctypes/test_cast.py

@@ -1,8 +1,11 @@
+import gc
 import sys
 import unittest
-from ctypes import (Structure, Union, POINTER, cast, sizeof, addressof,
-                    c_void_p, c_char_p, c_wchar_p,
-                    c_byte, c_short, c_int)
+from ctypes import (Structure, Union, pointer, POINTER, sizeof, addressof,
+                    c_void_p, c_char_p, c_wchar_p, cast,
+                    c_byte, c_short, c_int, c_int16,
+                    _pointer_type_cache)
+import weakref
 
 
 class Test(unittest.TestCase):
@@ -95,6 +98,303 @@ class MyUnion(Union):
             _fields_ = [("a", c_int)]
         self.assertRaises(TypeError, cast, array, MyUnion)
 
+    @unittest.expectedFailure  # gh-46376, gh-107131, gh-107940, gh-108222
+    def test_cast_detached_structure(self):
+        class Struct(Structure):
+            _fields_ = [("a", POINTER(c_int))]
+
+        x = Struct(a=pointer(c_int(23)))
+        self.assertEqual(x.a.contents.value, 23)
+
+        p = cast(x.a, POINTER(c_int))  # intentinally, cast to same
+        p.contents = c_int(32)
+
+        self.assertEqual(x.a.contents.value, 32)
+
+    @unittest.expectedFailure  # gh-46376, gh-107131, gh-107940, gh-108222
+    def test_casted_structure(self):
+        class Struct(Structure):
+            _fields_ = [("a", POINTER(c_int))]
+
+        x = Struct(a=pointer(c_int(23)))
+        self.assertEqual(x.a.contents.value, 23)
+
+        p = cast(pointer(x), POINTER(POINTER(c_int)))  # intentinally, cast to same
+        p.contents = pointer(c_int(32))
+
+        self.assertEqual(x.a.contents.value, 32)
+
+        # to avoid leaking when tests are run several times
+        # clean up the types left in the cache.
+        del _pointer_type_cache[Struct]
+
+    @unittest.expectedFailure  # gh-46376, gh-107131, gh-107940, gh-108222
+    def test_pointer_array(self):
+        w = weakref.WeakValueDictionary()
+        arr = 3 * POINTER(c_short)
+
+        vals = arr(
+            w.setdefault("10", pointer(c_short(10))),
+            w.setdefault("11", pointer(c_short(11))),
+            None,
+        )
+
+        self.assertEqual(vals[0].contents.value, 10)
+        self.assertEqual(vals[1].contents.value, 11)
+
+        class Struct(Structure):
+            _fields_ = [('a', c_int16)]
+
+        v = cast(pointer(vals), POINTER(Struct * 3)).contents
+        v[2] = v[0]
+
+        self.assertTrue(vals[2])
+        self.assertEqual(vals[2].contents.value, 10)
+
+        vals[2].contents = w.setdefault("12", pointer(c_short(12)))
+
+        self.assertEqual(vals[2].contents.value, 12)
+        self.assertEqual(vals[0].contents.value, 12)
+
+        gc.collect()
+        self.assertNotIn("10", w)
+
+    def test_pointer_identity1(self):
+        class Struct(Structure):
+            _fields_ = [('a', c_int16)]
+
+        Struct3 = 3 * Struct
+        c_array = (2 * Struct3)(
+            Struct3(Struct(a=1), Struct(a=2), Struct(a=3)),
+            Struct3(Struct(a=4), Struct(a=5), Struct(a=6))
+        )
+        self.assertEqual(c_array[0][0].a, 1)
+        self.assertEqual(c_array[0][1].a, 2)
+        self.assertEqual(c_array[0][2].a, 3)
+        self.assertEqual(c_array[1][0].a, 4)
+        self.assertEqual(c_array[1][1].a, 5)
+        self.assertEqual(c_array[1][2].a, 6)
+
+        p_obj = cast(pointer(c_array), POINTER(pointer(c_array)._type_))
+        obj = p_obj.contents
+        self.assertEqual(obj[0][0].a, 1)
+        self.assertEqual(obj[0][1].a, 2)
+        self.assertEqual(obj[0][2].a, 3)
+        self.assertEqual(obj[1][0].a, 4)
+        self.assertEqual(obj[1][1].a, 5)
+        self.assertEqual(obj[1][2].a, 6)
+
+        p_obj = cast(pointer(c_array[0]), POINTER(pointer(c_array)._type_))
+        obj = p_obj.contents
+
+        self.assertEqual(obj[0][0].a, 1)
+        self.assertEqual(obj[0][1].a, 2)
+        self.assertEqual(obj[0][2].a, 3)
+        self.assertEqual(obj[1][0].a, 4)
+        self.assertEqual(obj[1][1].a, 5)
+        self.assertEqual(obj[1][2].a, 6)
+
+    @unittest.expectedFailure  # gh-46376, gh-107131, gh-107940, gh-108222
+    def test_cast_structure_array(self):
+        class Struct(Structure):
+            _fields_ = [('a', c_int16)]
+
+        Struct3 = 3 * Struct
+        c_array = (2 * Struct3)(
+            Struct3(Struct(a=1), Struct(a=2), Struct(a=3)),
+            Struct3(Struct(a=4), Struct(a=5), Struct(a=6))
+        )
+        obj = cast(pointer(c_array), POINTER(pointer(c_array)._type_)).contents
+
+        c_array[0][0] = Struct(a=100)
+        self.assertEqual(c_array[0][0].a, 100)
+        self.assertEqual(obj[0][0].a, 100)
+
+        cast(pointer(c_array), POINTER(c_short)).contents = c_short(200)
+        self.assertEqual(c_array[0][0].a, 200)
+        self.assertEqual(obj[0][0].a, 200)
+
+    def test_pointer_identity2(self):
+        class Struct(Structure):
+            _fields_ = [('a', c_int16)]
+
+        StructPointer = POINTER(Struct)
+        s1 = Struct(a=10)
+        s2 = Struct(a=20)
+        s3 = Struct(a=30)
+        pointer_array = (3 * StructPointer)(pointer(s1), pointer(s2), pointer(s3))
+        self.assertEqual(pointer_array[0][0].a, 10)
+        self.assertEqual(pointer_array[1][0].a, 20)
+        self.assertEqual(pointer_array[2][0].a, 30)
+        self.assertEqual(pointer_array[0].contents.a, 10)
+        self.assertEqual(pointer_array[1].contents.a, 20)
+        self.assertEqual(pointer_array[2].contents.a, 30)
+
+        p_obj = cast(pointer(pointer_array[0]), POINTER(pointer(pointer_array)._type_))
+        obj = p_obj.contents
+        self.assertEqual(obj[0][0].a, 10)
+        self.assertEqual(obj[1][0].a, 20)
+        self.assertEqual(obj[2][0].a, 30)
+        self.assertEqual(obj[0].contents.a, 10)
+        self.assertEqual(obj[1].contents.a, 20)
+        self.assertEqual(obj[2].contents.a, 30)
+
+    @unittest.expectedFailure  # gh-46376, gh-107131, gh-107940, gh-108222
+    def test_cast_structure_pointer_array(self):
+        class Struct(Structure):
+            _fields_ = [('a', c_int16)]
+
+        StructPointer = POINTER(Struct)
+        s1 = Struct(a=10)
+        s2 = Struct(a=20)
+        s3 = Struct(a=30)
+        pointer_array = (3 * StructPointer)(pointer(s1), pointer(s2), pointer(s3))
+        obj = cast(pointer(pointer_array[0]),
+                   POINTER(pointer(pointer_array)._type_)).contents
+
+        pointer_array[0][0].a = 50
+        self.assertEqual(obj[0][0].a, 50)
+
+        c = cast(pointer(pointer_array[0]), POINTER(POINTER(c_short)))
+        c.contents = pointer(c_short(100))
+        self.assertEqual(pointer_array[0].contents.a, 100)
+        self.assertEqual(obj[0][0].a, 100)
+
+    @unittest.expectedFailure  # gh-46376, gh-107131, gh-107940, gh-108222
+    def test_pointer_identity3(self):
+        class Struct(Structure):
+            _fields_ = [('a', c_int16)]
+
+        class StructWithPointers(Structure):
+            _fields_ = [("s1", POINTER(Struct)), ("s2", POINTER(Struct))]
+
+        s1 = Struct(a=10)
+        s2 = Struct(a=20)
+        s3 = Struct(a=30)
+
+        StructPointer = POINTER(Struct)
+
+        pointer_array = (3 * StructPointer)(pointer(s1), pointer(s2), pointer(s3))
+
+        struct = StructWithPointers(s1=pointer(s1), s2=pointer(s2))
+        p_obj = pointer(struct)
+        obj = p_obj.contents
+        self.assertEqual(addressof(obj), addressof(struct))  # assertIs?
+
+        self.assertEqual(obj.s1[0].a, 10)
+        self.assertEqual(obj.s2[0].a, 20)
+        self.assertEqual(obj.s1.contents.a, 10)
+        self.assertEqual(obj.s2.contents.a, 20)
+
+        p_obj = cast(pointer(struct), POINTER(pointer(pointer_array)._type_))
+        obj = p_obj.contents
+        self.assertEqual(addressof(obj), addressof(struct))
+        self.assertEqual(addressof(obj[0]), addressof(struct))
+        self.assertEqual(addressof(obj[0]), addressof(struct.s1))
+        self.assertEqual(addressof(obj[1]), addressof(struct.s2))
+        self.assertEqual(addressof(obj[0].contents), addressof(struct.s1.contents))
+        self.assertEqual(addressof(obj[0][0]), addressof(pointer_array[0].contents))
+        self.assertEqual(addressof(obj[1][0]), addressof(pointer_array[1].contents))
+
+        self.assertEqual(obj[0][0].a, 10)
+        self.assertEqual(obj[1][0].a, 20)
+        self.assertEqual(obj[0].contents.a, 10)
+        self.assertEqual(obj[1].contents.a, 20)
+
+        obj[0][0].a = 23
+        self.assertEqual(pointer_array[0].contents.a, 23)
+
+        obj[0].contents.a = 32
+        self.assertEqual(pointer_array[0].contents.a, 32)
+
+        obj[1].contents = Struct(a=42)
+        self.assertEqual(pointer_array[1].contents.a, 42)
+
+    @unittest.expectedFailure  # gh-46376, gh-107131, gh-107940, gh-108222
+    def test_pointer_set_contents(self):
+        class Struct(Structure):
+            _fields_ = [('a', c_int16)]
+
+        w = weakref.WeakValueDictionary()
+        p = pointer(w.setdefault(23, Struct(a=23)))
+        self.assertIs(p._type_, Struct)
+        self.assertEqual(cast(p, POINTER(c_int16)).contents._type_, 'h')
+
+        pp = pointer(p)
+        self.assertIs(pp._type_, POINTER(Struct))
+
+        cast(pp, POINTER(POINTER(c_int16))).contents.contents = w.setdefault(
+            "k32",
+            c_int16(32)
+        )
+
+        self.assertEqual(addressof(p.contents), addressof(pp.contents.contents))
+        self.assertEqual(pp[0].contents.a, 32)
+        self.assertEqual(pp.contents[0].a, 32)
+        self.assertEqual(pp.contents.contents.a, 32)
+        del pp
+        gc.collect()
+
+        # if c_int16(32) was destroyed both of these two asserts should fail
+        self.assertIn("k32", w)   # but fail here first, so we don't use-after-fee
+        self.assertEqual(p[0].a, 32)
+
+        self.assertEqual(p.contents.a, 32)
+        self.assertEqual(cast(p, POINTER(c_int16)).contents.value, 32)
+
+    @unittest.expectedFailure  # gh-46376, gh-107131, gh-107940, gh-108222
+    def test_p2p_set_contents(self):
+        class StructA(Structure):
+            _fields_ = [('a', POINTER(c_int))]
+        class StructB(Structure):
+            _fields_ = [('b', POINTER(c_int))]
+
+        p = pointer(StructA(a=pointer(c_int(23))))
+        self.assertEqual(p.contents.a.contents.value, 23)
+
+        p.contents.a.contents.value = 32
+        self.assertEqual(p.contents.a.contents.value, 32)
+
+        w = weakref.WeakValueDictionary()
+        p[0].a = pointer(w.setdefault("1", c_int(100)))
+
+        self.assertEqual(p.contents.a.contents.value, 100)
+        self.assertEqual(p[0].a.contents.value, 100)
+
+        pp = cast(p, POINTER(POINTER(c_int)))
+        pp.contents.contents = w.setdefault("2", c_int(200))
+        gc.collect()
+
+        self.assertEqual(p.contents.a.contents.value, 200)
+        self.assertEqual(cast(pp, POINTER(StructA)).contents.a.contents.value, 200)
+        self.assertEqual(cast(pp, POINTER(StructA))[0].a.contents.value, 200)
+
+        del pp
+        gc.collect()
+
+        self.assertEqual(p.contents.a.contents.value, 200)
+        self.assertEqual(p[0].a.contents.value, 200)
+
+        x = StructA(a=pointer(c_int(23)))
+        r = cast(pointer(x), POINTER(POINTER(c_int)))
+        r.contents.contents = w.setdefault("3", c_int(300))
+        del r
+        gc.collect()
+        # if c_int(300) was destroyed both of these two asserts should fail
+        self.assertIn("3", w)   # but fail here first, so we don't use-after-fee
+        self.assertEqual(x.a[0], 300)
+
+        cast(pointer(x), POINTER(StructB)).contents.b = pointer(
+            w.setdefault("4", c_int(400))
+        )
+        gc.collect()
+        self.assertEqual(x.a[0], 400)
+
+        # to avoid leaking when tests are run several times
+        # clean up the types left in the cache.
+        del _pointer_type_cache[StructA]
+        del _pointer_type_cache[StructB]
+
 
 if __name__ == "__main__":
     unittest.main()