pythongh-46376: add (failing) tests for ctypes/pointer/cast/set_contents

Previous attempt to fix pythongh-46376 was incomplete and
overall it didn't succeed, and was reverted. However,
we have discovered some dangerous issues with ctypes,
that aren't fixed or documented anywhere.
This commit adds tests (@expectedfailure) so at least
developers are aware of situations where memory might
be corrupted, leaked or when changing a pointer value
might have no effect.
Hopefully, we should be able to remove @expectedfailure
in the future, with new shiny ctypes implementation or
at least a bugfix.

Author: s1kpp

.gitignore

@@ -124,6 +124,8 @@ Tools/unicode/data/
 /config.status.lineno
 # hendrikmuhs/ccache-action@v1
 /.ccache
+# clangd and it's debugger
+/.cache
 /platform
 /profile-clean-stamp
 /profile-run-stamp

Lib/test/test_ctypes/test_arrays.py

@@ -2,7 +2,7 @@
 import sys
 import unittest
 import warnings
-from ctypes import (Structure, Array, sizeof, addressof,
+from ctypes import (Structure, Array, sizeof, addressof, POINTER, pointer,
                     create_string_buffer, create_unicode_buffer,
                     c_char, c_wchar, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint,
                     c_long, c_ulonglong, c_float, c_double, c_longdouble)
@@ -249,6 +249,24 @@ def test_deprecation(self):
         with self.assertWarns(DeprecationWarning):
             CharArray = ctypes.ARRAY(c_char, 3)
 
+    @unittest.expectedFailure  # gh-46376, gh-107131, gh-107940, gh-108222
+    def test_ptr_reuse(self):
+        arr = 3 * POINTER(c_short)
+
+        vals = arr(pointer(c_short(10)), pointer(c_short(11)), None)
+
+        self.assertEqual(vals[0].contents.value, 10)
+        self.assertEqual(vals[1].contents.value, 11)
+
+        vals[2] = vals[0]
+
+        self.assertEqual(vals[2].contents.value, 10)
+
+        vals[2].contents = c_short(12)
+
+        self.assertEqual(vals[2].contents.value, 12)
+        self.assertEqual(vals[0].contents.value, 12)
+
 
 if __name__ == '__main__':
     unittest.main()

Lib/test/test_ctypes/test_cast.py

@@ -1,8 +1,11 @@
+import gc
 import sys
 import unittest
-from ctypes import (Structure, Union, POINTER, cast, sizeof, addressof,
-                    c_void_p, c_char_p, c_wchar_p,
-                    c_byte, c_short, c_int)
+from ctypes import (Structure, Union, pointer, POINTER, sizeof, addressof,
+                    c_void_p, c_char_p, c_wchar_p, cast,
+                    c_byte, c_short, c_int, c_int16,
+                    _pointer_type_cache)
+import weakref
 
 
 class Test(unittest.TestCase):
@@ -95,6 +98,297 @@ class MyUnion(Union):
             _fields_ = [("a", c_int)]
         self.assertRaises(TypeError, cast, array, MyUnion)
 
+    @unittest.expectedFailure  # gh-46376, gh-107131, gh-107940, gh-108222
+    def test_cast_detached_structure(self):
+        class Struct(Structure):
+            _fields_ = [("a", POINTER(c_int))]
+
+        x = Struct(a=pointer(c_int(23)))
+        self.assertEqual(x.a.contents.value, 23)
+
+        p = cast(x.a, POINTER(c_int))  # intentinally, cast to same
+        p.contents = c_int(32)
+
+        self.assertEqual(x.a.contents.value, 32)
+
+    @unittest.expectedFailure  # gh-46376, gh-107131, gh-107940, gh-108222
+    def test_casted_structure(self):
+        class Struct(Structure):
+            _fields_ = [("a", POINTER(c_int))]
+
+        x = Struct(a=pointer(c_int(23)))
+        self.assertEqual(x.a.contents.value, 23)
+
+        p = cast(pointer(x), POINTER(POINTER(c_int)))  # intentinally, cast to same
+        p.contents = pointer(c_int(32))
+
+        self.assertEqual(x.a.contents.value, 32)
+
+        # to avoid leaking when tests are run several times
+        # clean up the types left in the cache.
+        del _pointer_type_cache[Struct]
+
+    @unittest.expectedFailure  # gh-46376, gh-107131, gh-107940, gh-108222
+    def test_pointer_array(self):
+        w = weakref.WeakValueDictionary()
+        arr = 3 * POINTER(c_short)
+
+        vals = arr(
+            w.setdefault("10", pointer(c_short(10))),
+            w.setdefault("11", pointer(c_short(11))),
+            None,
+        )
+
+        self.assertEqual(vals[0].contents.value, 10)
+        self.assertEqual(vals[1].contents.value, 11)
+
+        class Struct(Structure):
+            _fields_ = [('a', c_int16)]
+
+        v = cast(pointer(vals), POINTER(Struct * 3)).contents
+        v[2] = v[0]
+
+        self.assertTrue(vals[2])
+        self.assertEqual(vals[2].contents.value, 10)
+
+        vals[2].contents = w.setdefault("12", pointer(c_short(12)))
+
+        self.assertEqual(vals[2].contents.value, 12)
+        self.assertEqual(vals[0].contents.value, 12)
+
+        gc.collect()
+        self.assertNotIn("10", w)
+
+    def test_pointer_identity1(self):
+        class Struct(Structure):
+            _fields_ = [('a', c_int16)]
+
+        Struct3 = 3 * Struct
+        c_array = (2 * Struct3)(
+            Struct3(Struct(a=1), Struct(a=2), Struct(a=3)),
+            Struct3(Struct(a=4), Struct(a=5), Struct(a=6))
+        )
+        self.assertEqual(c_array[0][0].a, 1)
+        self.assertEqual(c_array[0][1].a, 2)
+        self.assertEqual(c_array[0][2].a, 3)
+        self.assertEqual(c_array[1][0].a, 4)
+        self.assertEqual(c_array[1][1].a, 5)
+        self.assertEqual(c_array[1][2].a, 6)
+
+        p_obj = cast(pointer(c_array), POINTER(pointer(c_array)._type_))
+        obj = p_obj.contents
+        self.assertEqual(obj[0][0].a, 1)
+        self.assertEqual(obj[0][1].a, 2)
+        self.assertEqual(obj[0][2].a, 3)
+        self.assertEqual(obj[1][0].a, 4)
+        self.assertEqual(obj[1][1].a, 5)
+        self.assertEqual(obj[1][2].a, 6)
+
+        p_obj = cast(pointer(c_array[0]), POINTER(pointer(c_array)._type_))
+        obj = p_obj.contents
+
+        self.assertEqual(obj[0][0].a, 1)
+        self.assertEqual(obj[0][1].a, 2)
+        self.assertEqual(obj[0][2].a, 3)
+        self.assertEqual(obj[1][0].a, 4)
+        self.assertEqual(obj[1][1].a, 5)
+        self.assertEqual(obj[1][2].a, 6)
+
+    @unittest.expectedFailure  # gh-46376, gh-107131, gh-107940, gh-108222
+    def test_cast_structure_array(self):
+        class Struct(Structure):
+            _fields_ = [('a', c_int16)]
+
+        Struct3 = 3 * Struct
+        c_array = (2 * Struct3)(
+            Struct3(Struct(a=1), Struct(a=2), Struct(a=3)),
+            Struct3(Struct(a=4), Struct(a=5), Struct(a=6))
+        )
+        obj = cast(pointer(c_array), POINTER(pointer(c_array)._type_)).contents
+
+        c_array[0][0] = Struct(a=100)
+        self.assertEqual(c_array[0][0].a, 100)
+        self.assertEqual(obj[0][0].a, 100)
+
+        cast(pointer(c_array), POINTER(c_short)).contents = c_short(200)
+        self.assertEqual(c_array[0][0].a, 200)
+        self.assertEqual(obj[0][0].a, 200)
+
+    def test_pointer_identity2(self):
+        class Struct(Structure):
+            _fields_ = [('a', c_int16)]
+
+        StructPointer = POINTER(Struct)
+        s1 = Struct(a=10)
+        s2 = Struct(a=20)
+        s3 = Struct(a=30)
+        pointer_array = (3 * StructPointer)(pointer(s1), pointer(s2), pointer(s3))
+        self.assertEqual(pointer_array[0][0].a, 10)
+        self.assertEqual(pointer_array[1][0].a, 20)
+        self.assertEqual(pointer_array[2][0].a, 30)
+        self.assertEqual(pointer_array[0].contents.a, 10)
+        self.assertEqual(pointer_array[1].contents.a, 20)
+        self.assertEqual(pointer_array[2].contents.a, 30)
+
+        p_obj = cast(pointer(pointer_array[0]), POINTER(pointer(pointer_array)._type_))
+        obj = p_obj.contents
+        self.assertEqual(obj[0][0].a, 10)
+        self.assertEqual(obj[1][0].a, 20)
+        self.assertEqual(obj[2][0].a, 30)
+        self.assertEqual(obj[0].contents.a, 10)
+        self.assertEqual(obj[1].contents.a, 20)
+        self.assertEqual(obj[2].contents.a, 30)
+
+    @unittest.expectedFailure  # gh-46376, gh-107131, gh-107940, gh-108222
+    def test_cast_structure_pointer_array(self):
+        class Struct(Structure):
+            _fields_ = [('a', c_int16)]
+
+        StructPointer = POINTER(Struct)
+        s1 = Struct(a=10)
+        s2 = Struct(a=20)
+        s3 = Struct(a=30)
+        pointer_array = (3 * StructPointer)(pointer(s1), pointer(s2), pointer(s3))
+        obj = cast(pointer(pointer_array[0]),
+                   POINTER(pointer(pointer_array)._type_)).contents
+
+        pointer_array[0][0].a = 50
+        self.assertEqual(obj[0][0].a, 50)
+
+        c = cast(pointer(pointer_array[0]), POINTER(POINTER(c_short)))
+        c.contents = pointer(c_short(100))
+        self.assertEqual(pointer_array[0].contents.a, 100)
+        self.assertEqual(obj[0][0].a, 100)
+
+    @unittest.expectedFailure  # gh-46376, gh-107131, gh-107940, gh-108222
+    def test_pointer_identity3(self):
+        class Struct(Structure):
+            _fields_ = [('a', c_int16)]
+
+        class StructWithPointers(Structure):
+            _fields_ = [("s1", POINTER(Struct)), ("s2", POINTER(Struct))]
+
+        s1 = Struct(a=10)
+        s2 = Struct(a=20)
+        s3 = Struct(a=30)
+
+        StructPointer = POINTER(Struct)
+
+        pointer_array = (3 * StructPointer)(pointer(s1), pointer(s2), pointer(s3))
+
+        struct = StructWithPointers(s1=pointer(s1), s2=pointer(s2))
+        p_obj = pointer(struct)
+        obj = p_obj.contents
+        self.assertEqual(addressof(obj), addressof(struct))  # assertIs?
+
+        self.assertEqual(obj.s1[0].a, 10)
+        self.assertEqual(obj.s2[0].a, 20)
+        self.assertEqual(obj.s1.contents.a, 10)
+        self.assertEqual(obj.s2.contents.a, 20)
+
+        p_obj = cast(pointer(struct), POINTER(pointer(pointer_array)._type_))
+        obj = p_obj.contents
+        self.assertEqual(addressof(obj), addressof(struct))
+        self.assertEqual(addressof(obj[0]), addressof(struct))
+        self.assertEqual(addressof(obj[0]), addressof(struct.s1))
+        self.assertEqual(addressof(obj[1]), addressof(struct.s2))
+        self.assertEqual(addressof(obj[0].contents), addressof(struct.s1.contents))
+        self.assertEqual(addressof(obj[0][0]), addressof(pointer_array[0].contents))
+        self.assertEqual(addressof(obj[1][0]), addressof(pointer_array[1].contents))
+
+        self.assertEqual(obj[0][0].a, 10)
+        self.assertEqual(obj[1][0].a, 20)
+        self.assertEqual(obj[0].contents.a, 10)
+        self.assertEqual(obj[1].contents.a, 20)
+
+        obj[0][0].a = 23
+        self.assertEqual(pointer_array[0].contents.a, 23)
+
+        obj[0].contents.a = 32
+        self.assertEqual(pointer_array[0].contents.a, 32)
+
+        obj[1].contents = Struct(a=42)
+        self.assertEqual(pointer_array[1].contents.a, 42)
+
+    def test_pointer_set_contents(self):
+        class Struct(Structure):
+            _fields_ = [('a', c_int16)]
+
+        w = weakref.WeakValueDictionary()
+        p = pointer(w.setdefault(23, Struct(a=23)))
+        self.assertIs(p._type_, Struct)
+        self.assertEqual(cast(p, POINTER(c_int16)).contents._type_, 'h')
+
+        pp = pointer(p)
+        self.assertIs(pp._type_, POINTER(Struct))
+
+        cast(pp, POINTER(POINTER(c_int16))).contents.contents = w.setdefault(
+            32,
+            c_int16(32)
+        )
+
+        self.assertEqual(addressof(p.contents), addressof(pp.contents.contents))
+        self.assertEqual(pp[0].contents.a, 32)
+        self.assertEqual(pp.contents[0].a, 32)
+        self.assertEqual(pp.contents.contents.a, 32)
+        del pp
+        gc.collect()
+
+        self.assertEqual(p[0].a, 32)
+        self.assertEqual(p.contents.a, 32)
+        self.assertEqual(cast(p, POINTER(c_int16)).contents.value, 32)
+
+    @unittest.expectedFailure  # gh-46376, gh-107131, gh-107940, gh-108222
+    def test_p2p_set_contents(self):
+        class StructA(Structure):
+            _fields_ = [('a', POINTER(c_int))]
+        class StructB(Structure):
+            _fields_ = [('b', POINTER(c_int))]
+
+        p = pointer(StructA(a=pointer(c_int(23))))
+        self.assertEqual(p.contents.a.contents.value, 23)
+
+        p.contents.a.contents.value = 32
+        self.assertEqual(p.contents.a.contents.value, 32)
+
+        w = weakref.WeakValueDictionary()
+        p[0].a = pointer(w.setdefault("1", c_int(100)))
+
+        self.assertEqual(p.contents.a.contents.value, 100)
+        self.assertEqual(p[0].a.contents.value, 100)
+
+        pp = cast(p, POINTER(POINTER(c_int)))
+        pp.contents.contents = w.setdefault("2", c_int(200))
+        gc.collect()
+
+        self.assertEqual(p.contents.a.contents.value, 200)
+        self.assertEqual(cast(pp, POINTER(StructA)).contents.a.contents.value, 200)
+        self.assertEqual(cast(pp, POINTER(StructA))[0].a.contents.value, 200)
+
+        del pp
+        gc.collect()
+
+        self.assertEqual(p.contents.a.contents.value, 200)
+        self.assertEqual(p[0].a.contents.value, 200)
+
+        x = StructA(a=pointer(c_int(23)))
+        r = cast(pointer(x), POINTER(POINTER(c_int)))
+        r.contents.contents = w.setdefault("3", c_int(300))
+        del r
+        gc.collect()
+        self.assertEqual(x.a[0], 300)
+
+        cast(pointer(x), POINTER(StructB)).contents.b = pointer(
+            w.setdefault("4", c_int(400))
+        )
+        gc.collect()
+        self.assertEqual(x.a[0], 400)
+
+        # to avoid leaking when tests are run several times
+        # clean up the types left in the cache.
+        del _pointer_type_cache[StructA]
+        del _pointer_type_cache[StructB]
+
 
 if __name__ == "__main__":
     unittest.main()