From ac1d0a3049dfbe80366fdeaf35d27a4402086d16 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 25 Jul 2024 16:32:00 +0200 Subject: [PATCH 01/94] Add classes that will implement the struct packing --- Lib/ctypes/_layout.py | 15 +++++++++++++++ Modules/_ctypes/stgdict.c | 24 ++++++++++++++++++------ 2 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 Lib/ctypes/_layout.py diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py new file mode 100644 index 00000000000000..449764c29498cf --- /dev/null +++ b/Lib/ctypes/_layout.py @@ -0,0 +1,15 @@ +import sys + +class _BaseLayout: + pass + +class WindowsLayout(_BaseLayout): + pass + +class GCCSysVLayout(_BaseLayout): + pass + +if sys.platform == 'win32': + NativeLayout = WindowsLayout +else: + NativeLayout = GCCSysVLayout diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 970f0a033fbb0b..a7cadd5469b210 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -293,17 +293,19 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct LayoutMode layout_mode = (pack > 0) ? LAYOUT_MODE_MS : LAYOUT_MODE_GCC_SYSV; #endif + PyObject *layout_class; if (PyObject_GetOptionalAttr(type, &_Py_ID(_layout_), &tmp) < 0) { return -1; } - if (tmp) { - if (!PyUnicode_Check(tmp)) { - PyErr_SetString(PyExc_TypeError, - "_layout_ must be a string"); - return -1; - } + if (!tmp) { + layout_class = _PyImport_GetModuleAttrString("ctypes._layout", + "NativeLayout"); + } + else if (PyUnicode_Check(tmp)) { if (PyUnicode_CompareWithASCIIString(tmp, "ms") == 0) { layout_mode = LAYOUT_MODE_MS; + layout_class = _PyImport_GetModuleAttrString("ctypes._layout", + "WindowsLayout"); } else if (PyUnicode_CompareWithASCIIString(tmp, "gcc-sysv") == 0) { layout_mode = LAYOUT_MODE_GCC_SYSV; @@ -312,13 +314,23 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct "_pack_ is not compatible with _layout_=\"gcc-sysv\""); return -1; } + layout_class = _PyImport_GetModuleAttrString("ctypes._layout", + "GCCSysVLayout"); } else { PyErr_Format(PyExc_ValueError, "unknown _layout_ %R", tmp); return -1; } + Py_DECREF(tmp); + } + else { + layout_class = tmp; + } + if (!layout_class) { + return -1; } + Py_DECREF(layout_class); if (PyObject_GetOptionalAttr(type, &_Py_ID(_align_), &tmp) < 0) { return -1; } From b35a97ca9b6c9df6a205bdfb7800f536f5d7c0ca Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 25 Jul 2024 16:57:28 +0200 Subject: [PATCH 02/94] Instantiate the layout and take `_align_` from it --- .../pycore_global_objects_fini_generated.h | 2 +- Include/internal/pycore_global_strings.h | 2 +- .../internal/pycore_runtime_init_generated.h | 2 +- .../internal/pycore_unicodeobject_generated.h | 8 ++++---- Lib/ctypes/_layout.py | 4 +++- Modules/_ctypes/stgdict.c | 17 ++++++++++++++++- 6 files changed, 26 insertions(+), 9 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index d9b46df507dfd7..b157f5f0179cb9 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -738,7 +738,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abc_impl)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abstract_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_active)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_align_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_anonymous_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_argtypes_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_as_parameter_)); @@ -787,6 +786,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aggregate_class)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(align)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(allow_code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arg)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 10773d7a6c7e3f..5ed0b2f85d6584 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -227,7 +227,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(_abc_impl) STRUCT_FOR_ID(_abstract_) STRUCT_FOR_ID(_active) - STRUCT_FOR_ID(_align_) STRUCT_FOR_ID(_anonymous_) STRUCT_FOR_ID(_argtypes_) STRUCT_FOR_ID(_as_parameter_) @@ -276,6 +275,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(after_in_parent) STRUCT_FOR_ID(aggregate_class) STRUCT_FOR_ID(alias) + STRUCT_FOR_ID(align) STRUCT_FOR_ID(allow_code) STRUCT_FOR_ID(append) STRUCT_FOR_ID(arg) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 618f8d0a36b6c3..56133a2f5c6a5a 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -736,7 +736,6 @@ extern "C" { INIT_ID(_abc_impl), \ INIT_ID(_abstract_), \ INIT_ID(_active), \ - INIT_ID(_align_), \ INIT_ID(_anonymous_), \ INIT_ID(_argtypes_), \ INIT_ID(_as_parameter_), \ @@ -785,6 +784,7 @@ extern "C" { INIT_ID(after_in_parent), \ INIT_ID(aggregate_class), \ INIT_ID(alias), \ + INIT_ID(align), \ INIT_ID(allow_code), \ INIT_ID(append), \ INIT_ID(arg), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index f848a002c3b5d1..b6fd81a3ca3937 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -708,10 +708,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(_align_); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_anonymous_); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -904,6 +900,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(align); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(allow_code); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 449764c29498cf..095a439cfd8ca3 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -1,7 +1,9 @@ import sys class _BaseLayout: - pass + def __init__(self, cls, fields, is_struct): + self.cls = cls + self.align = getattr(cls, '_align_', 1); class WindowsLayout(_BaseLayout): pass diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index a7cadd5469b210..d7beff0244b559 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -330,10 +330,25 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (!layout_class) { return -1; } + PyObject *layout = PyObject_Vectorcall( + layout_class, + 1 + (PyObject*[]){ + NULL, + type, + fields, + Py_GetConstantBorrowed( + isStruct ? Py_CONSTANT_TRUE : Py_CONSTANT_FALSE)}, + 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); Py_DECREF(layout_class); - if (PyObject_GetOptionalAttr(type, &_Py_ID(_align_), &tmp) < 0) { + if (!layout) { return -1; } + if (PyObject_GetOptionalAttr(layout, &_Py_ID(align), &tmp) < 0) { + Py_DECREF(layout); + return -1; + } + Py_DECREF(layout); if (tmp) { forced_alignment = PyLong_AsInt(tmp); Py_DECREF(tmp); From 2cc8b40c85cfe1e6fbc3221d2b70df776b27180c Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Sat, 27 Jul 2024 18:38:02 +0200 Subject: [PATCH 03/94] Get base size and align; etc. --- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 4 + Lib/ctypes/_layout.py | 33 ++++++- Modules/_ctypes/_ctypes.c | 2 +- Modules/_ctypes/stgdict.c | 99 ++++++++++--------- 7 files changed, 89 insertions(+), 52 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index b157f5f0179cb9..19f51029827232 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1006,6 +1006,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(intersection)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(interval)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(is_running)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(is_struct)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isatty)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isinstance)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isoformat)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 5ed0b2f85d6584..1bf5bdaf488b7a 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -495,6 +495,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(intersection) STRUCT_FOR_ID(interval) STRUCT_FOR_ID(is_running) + STRUCT_FOR_ID(is_struct) STRUCT_FOR_ID(isatty) STRUCT_FOR_ID(isinstance) STRUCT_FOR_ID(isoformat) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 56133a2f5c6a5a..1a1c8408623fb1 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1004,6 +1004,7 @@ extern "C" { INIT_ID(intersection), \ INIT_ID(interval), \ INIT_ID(is_running), \ + INIT_ID(is_struct), \ INIT_ID(isatty), \ INIT_ID(isinstance), \ INIT_ID(isoformat), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index b6fd81a3ca3937..839446c9bc7331 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1780,6 +1780,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(is_struct); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(isatty); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 095a439cfd8ca3..7dca44af56db16 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -1,9 +1,38 @@ import sys +import warnings + +from _ctypes import CField +import ctypes class _BaseLayout: - def __init__(self, cls, fields, is_struct): + def __init__(self, cls, fields, is_struct, base, **kwargs): + if kwargs: + warnings.warn(f'Unknown keyword arguments: {list(kwargs.keys())}') self.cls = cls - self.align = getattr(cls, '_align_', 1); + self.fields = fields + self.is_struct = is_struct + + if base: + self.size = self.offset = ctypes.sizeof(base) + base_align = ctypes.alignment(base) + else: + self.size = 0 + self.offset = 0 + base_align = 1 + + self.align = getattr(cls, '_align_', 1) + if self.align < 0: + raise ValueError('_align_ must be a non-negative integer') + elif self.align == 0: + # Setting `_align_ = 0` amounts to using the default alignment + self.align == 1 + + self.total_align = max(self.align, base_align) + + def __iter__(self): + for field in self.fields: + yield CField() + class WindowsLayout(_BaseLayout): pass diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index b55102639c6786..9c43fbe0e42358 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -5780,7 +5780,7 @@ _ctypes_add_types(PyObject *mod) * Simple classes */ - CREATE_TYPE(st->PyCField_Type, &cfield_spec, NULL, NULL); + MOD_ADD_TYPE(st->PyCField_Type, &cfield_spec, NULL, NULL); /************************************************* * diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index d7beff0244b559..d1461a442c2f64 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -246,7 +246,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct Py_ssize_t bitofs = 0; PyObject *tmp; int pack; - int forced_alignment = 1; Py_ssize_t ffi_ofs; int big_endian; int arrays_seen = 0; @@ -265,6 +264,31 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct big_endian = PY_BIG_ENDIAN; } + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + StgInfo *stginfo; + if (PyStgInfo_FromType(st, type, &stginfo) < 0) { + return -1; + } + if (!stginfo) { + PyErr_SetString(PyExc_TypeError, + "ctypes state is not initialized"); + return -1; + } + PyObject *base = (PyObject *)((PyTypeObject *)type)->tp_base; + StgInfo *baseinfo; + if (PyStgInfo_FromType(st, base, &baseinfo) < 0) { + return -1; + } + + /* If this structure/union is already marked final we cannot assign + _fields_ anymore. */ + + if (stginfo->flags & DICTFLAG_FINAL) {/* is final ? */ + PyErr_SetString(PyExc_AttributeError, + "_fields_ is final"); + return -1; + } + if (PyObject_GetOptionalAttr(type, &_Py_ID(_pack_), &tmp) < 0) { return -1; } @@ -330,42 +354,43 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (!layout_class) { return -1; } + PyObject *kwnames = PyTuple_Pack( + 2, + &_Py_ID(is_struct), + &_Py_ID(base)); + PyObject *base_arg = Py_NewRef(baseinfo ? base : Py_None); PyObject *layout = PyObject_Vectorcall( layout_class, 1 + (PyObject*[]){ NULL, + /* positional args */ type, fields, + /* keyword args */ Py_GetConstantBorrowed( - isStruct ? Py_CONSTANT_TRUE : Py_CONSTANT_FALSE)}, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); + isStruct ? Py_CONSTANT_TRUE : Py_CONSTANT_FALSE), + base_arg}, + 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); + Py_DECREF(kwnames); + Py_DECREF(base_arg); Py_DECREF(layout_class); if (!layout) { return -1; } - if (PyObject_GetOptionalAttr(layout, &_Py_ID(align), &tmp) < 0) { - Py_DECREF(layout); + tmp = PyObject_GetAttr(layout, &_Py_ID(align)); + Py_DECREF(layout); + if (!tmp) { return -1; } - Py_DECREF(layout); - if (tmp) { - forced_alignment = PyLong_AsInt(tmp); - Py_DECREF(tmp); - if (forced_alignment < 0) { - if (!PyErr_Occurred() || - PyErr_ExceptionMatches(PyExc_TypeError) || - PyErr_ExceptionMatches(PyExc_OverflowError)) - { - PyErr_SetString(PyExc_ValueError, - "_align_ must be a non-negative integer"); - } - return -1; + int forced_alignment = PyLong_AsInt(tmp); + Py_DECREF(tmp); + if (forced_alignment < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "_align_ must be a non-negative integer"); } - } - else { - /* Setting `_align_ = 0` amounts to using the default alignment */ - forced_alignment = 1; + return -1; } len = PySequence_Size(fields); @@ -377,39 +402,15 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct return -1; } - ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); - StgInfo *stginfo; - if (PyStgInfo_FromType(st, type, &stginfo) < 0) { - return -1; - } - if (!stginfo) { - PyErr_SetString(PyExc_TypeError, - "ctypes state is not initialized"); - return -1; - } - - /* If this structure/union is already marked final we cannot assign - _fields_ anymore. */ - - if (stginfo->flags & DICTFLAG_FINAL) {/* is final ? */ - PyErr_SetString(PyExc_AttributeError, - "_fields_ is final"); - return -1; - } - if (stginfo->format) { PyMem_Free(stginfo->format); stginfo->format = NULL; } - if (stginfo->ffi_type_pointer.elements) + if (stginfo->ffi_type_pointer.elements) { PyMem_Free(stginfo->ffi_type_pointer.elements); - - StgInfo *baseinfo; - if (PyStgInfo_FromType(st, (PyObject *)((PyTypeObject *)type)->tp_base, - &baseinfo) < 0) { - return -1; } + if (baseinfo) { stginfo->flags |= (baseinfo->flags & (TYPEFLAG_HASUNION | TYPEFLAG_HASBITFIELD)); From a9fd94ae8290a41d8a1d7a6adfd9e1b098f0db22 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Sat, 27 Jul 2024 18:53:55 +0200 Subject: [PATCH 04/94] Sorta start creating CField objects --- Lib/ctypes/_layout.py | 15 +++- Lib/test/test_ctypes/test_struct_fields.py | 2 +- Modules/_ctypes/cfield.c | 43 +++++++++- Modules/_ctypes/clinic/cfield.c.h | 98 ++++++++++++++++++++++ Modules/_ctypes/stgdict.c | 15 +++- 5 files changed, 169 insertions(+), 4 deletions(-) create mode 100644 Modules/_ctypes/clinic/cfield.c.h diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 7dca44af56db16..d9071f1a7e520b 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -31,7 +31,20 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): def __iter__(self): for field in self.fields: - yield CField() + field = tuple(field) + try: + name, ftype, bits = field + except ValueError: + name, ftype = field + bits = None + size = ctypes.sizeof(ftype) + offset = self.offset + yield CField( + name=name, + size=size, + offset=offset, + ) + self.offset += self.size class WindowsLayout(_BaseLayout): diff --git a/Lib/test/test_ctypes/test_struct_fields.py b/Lib/test/test_ctypes/test_struct_fields.py index 7adab794809def..1746312155b46f 100644 --- a/Lib/test/test_ctypes/test_struct_fields.py +++ b/Lib/test/test_ctypes/test_struct_fields.py @@ -60,7 +60,7 @@ def test_6(self): self.assertRaises(TypeError, CField) def test_cfield_type_flags(self): - self.assertTrue(CField.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + #self.assertTrue(CField.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) self.assertTrue(CField.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) def test_cfield_inheritance_hierarchy(self): diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 2c1fb9b862e12d..ea6dbc76046cec 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -20,6 +20,13 @@ #define CTYPES_CFIELD_CAPSULE_NAME_PYMEM "_ctypes/cfield.c pymem" +/*[clinic input] +module _ctypes +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=476a19c49b31a75c]*/ + +#include "clinic/cfield.c.h" + static void pymem_destructor(PyObject *ptr) { void *p = PyCapsule_GetPointer(ptr, CTYPES_CFIELD_CAPSULE_NAME_PYMEM); @@ -33,6 +40,10 @@ static void pymem_destructor(PyObject *ptr) /* PyCField_Type */ +/*[clinic input] +class _ctypes.CField "PyObject *" "PyObject" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=602817ea3ffc709c]*/ static inline Py_ssize_t round_down(Py_ssize_t numToRound, Py_ssize_t multiple) @@ -185,6 +196,35 @@ PyCField_FromDesc_msvc( return 0; } +/*[clinic input] +@classmethod +_ctypes.CField.__new__ as PyCField_new + + name: str + size: Py_ssize_t + offset: Py_ssize_t + +[clinic start generated code]*/ + +static PyObject * +PyCField_new_impl(PyTypeObject *type, const char *name, Py_ssize_t size, + Py_ssize_t offset) +/*[clinic end generated code: output=109dac517c651e4b input=25b7b4a2d83d7ef9]*/ +{ + PyTypeObject *tp = type; + CFieldObject* self = (CFieldObject *)tp->tp_alloc(tp, 0); + self->size = size; + self->offset = offset; + + self->setfunc = NULL; // XXX + self->getfunc = NULL; // XXX + self->index = 0; // XXX + self->proto = NULL; // XXX + + return (PyObject *)self; +} + + PyObject * PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, Py_ssize_t bitsize, @@ -398,6 +438,7 @@ PyCField_repr(CFieldObject *self) } static PyType_Slot cfield_slots[] = { + {Py_tp_new, PyCField_new}, {Py_tp_dealloc, PyCField_dealloc}, {Py_tp_repr, PyCField_repr}, {Py_tp_doc, (void *)PyDoc_STR("Structure/Union member")}, @@ -413,7 +454,7 @@ PyType_Spec cfield_spec = { .name = "_ctypes.CField", .basicsize = sizeof(CFieldObject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), + Py_TPFLAGS_IMMUTABLETYPE), .slots = cfield_slots, }; diff --git a/Modules/_ctypes/clinic/cfield.c.h b/Modules/_ctypes/clinic/cfield.c.h new file mode 100644 index 00000000000000..895994a0476a75 --- /dev/null +++ b/Modules/_ctypes/clinic/cfield.c.h @@ -0,0 +1,98 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +static PyObject * +PyCField_new_impl(PyTypeObject *type, const char *name, Py_ssize_t size, + Py_ssize_t offset); + +static PyObject * +PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(name), &_Py_ID(size), &_Py_ID(offset), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"name", "size", "offset", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "CField", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + const char *name; + Py_ssize_t size; + Py_ssize_t offset; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 3, 3, 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!PyUnicode_Check(fastargs[0])) { + _PyArg_BadArgument("CField", "argument 'name'", "str", fastargs[0]); + goto exit; + } + Py_ssize_t name_length; + name = PyUnicode_AsUTF8AndSize(fastargs[0], &name_length); + if (name == NULL) { + goto exit; + } + if (strlen(name) != (size_t)name_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(fastargs[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + size = ival; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(fastargs[2]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + offset = ival; + } + return_value = PyCField_new_impl(type, name, size, offset); + +exit: + return return_value; +} +/*[clinic end generated code: output=1f92a9742a307128 input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index d1461a442c2f64..95e0b70b8f21a5 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -379,8 +379,8 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct return -1; } tmp = PyObject_GetAttr(layout, &_Py_ID(align)); - Py_DECREF(layout); if (!tmp) { + Py_DECREF(layout); return -1; } int forced_alignment = PyLong_AsInt(tmp); @@ -390,6 +390,14 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct PyErr_SetString(PyExc_ValueError, "_align_ must be a non-negative integer"); } + Py_DECREF(layout); + return -1; + } + + PyObject *layout_fields = PySequence_Fast(layout, + "layout must return a sequence"); + Py_DECREF(layout); + if (!layout_fields) { return -1; } @@ -402,6 +410,11 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct return -1; } + if (len != PySequence_Fast_GET_SIZE(layout_fields)) { + PyErr_SetString(PyExc_ValueError, + "number of '_fields_' must match result of layout... for now."); + } + if (stginfo->format) { PyMem_Free(stginfo->format); stginfo->format = NULL; From c4d3c662cac35a27c6e74e8849d9139ce778e53c Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Sat, 27 Jul 2024 19:25:32 +0200 Subject: [PATCH 05/94] Turn constructor into initializer --- Modules/_ctypes/cfield.c | 27 ++++++++++----------------- Modules/_ctypes/ctypes.h | 7 ++++--- Modules/_ctypes/stgdict.c | 23 +++++++++++++++++++---- 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index ea6dbc76046cec..52f89d7d58e2c9 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -225,27 +225,24 @@ PyCField_new_impl(PyTypeObject *type, const char *name, Py_ssize_t size, } -PyObject * -PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, +int +PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, int big_endian, LayoutMode layout_mode) { PyTypeObject *tp = st->PyCField_Type; - CFieldObject* self = (CFieldObject *)tp->tp_alloc(tp, 0); if (self == NULL) { - return NULL; + return -1; } StgInfo *info; if (PyStgInfo_FromType(st, desc, &info) < 0) { - Py_DECREF(self); - return NULL; + return -1; } if (!info) { PyErr_SetString(PyExc_TypeError, "has no _stginfo_"); - Py_DECREF(self); - return NULL; + return -1; } PyObject* proto = desc; @@ -258,21 +255,18 @@ PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, if (PyCArrayTypeObject_Check(st, proto)) { StgInfo *ainfo; if (PyStgInfo_FromType(st, proto, &ainfo) < 0) { - Py_DECREF(self); - return NULL; + return -1; } if (ainfo && ainfo->proto) { StgInfo *iinfo; if (PyStgInfo_FromType(st, ainfo->proto, &iinfo) < 0) { - Py_DECREF(self); - return NULL; + return -1; } if (!iinfo) { PyErr_SetString(PyExc_TypeError, "has no _stginfo_"); - Py_DECREF(self); - return NULL; + return -1; } if (iinfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { struct fielddesc *fd = _ctypes_get_fielddesc("s"); @@ -323,14 +317,13 @@ PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, ); } if (result < 0) { - Py_DECREF(self); - return NULL; + return -1; } assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); if(big_endian && is_bitfield) { self->size = BUILD_SIZE(NUM_BITS(self->size), 8*info->size - LOW_BIT(self->size) - bitsize); } - return (PyObject *)self; + return 0; } static int diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index a794cfe86b5f42..10c4b23992ff54 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -221,8 +221,9 @@ typedef enum { LAYOUT_MODE_GCC_SYSV, } LayoutMode; -extern PyObject * -PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, +struct CFieldObject; +extern int +PyCField_InitFromDesc(ctypes_state *st, struct CFieldObject* self, PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, @@ -259,7 +260,7 @@ struct fielddesc { GETFUNC getfunc_swapped; }; -typedef struct { +typedef struct CFieldObject { PyObject_HEAD Py_ssize_t offset; Py_ssize_t size; diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 95e0b70b8f21a5..f430b4399c886c 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -548,6 +548,20 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } else bitsize = 0; + prop = PySequence_Fast_GET_ITEM(layout_fields, i); + if (!prop) { + Py_DECREF(pair); + return -1; + } + Py_INCREF(prop); + if (!PyType_IsSubtype(Py_TYPE(prop), st->PyCField_Type)) { + PyErr_Format(PyExc_TypeError, + "fields must be of type CField, got %T", prop); + Py_DECREF(pair); + return -1; + + } + if (isStruct) { const char *fieldfmt = info->format ? info->format : "B"; const char *fieldname = PyUnicode_AsUTF8(name); @@ -565,12 +579,13 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct /* construct the field now, as `prop->offset` is `offset` with corrected alignment */ - prop = PyCField_FromDesc(st, desc, i, + int res = PyCField_InitFromDesc(st, (CFieldObject*)prop, desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, pack, big_endian, layout_mode); - if (prop == NULL) { + if (res < 0) { Py_DECREF(pair); + Py_DECREF(prop); return -1; } @@ -621,11 +636,11 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct bitofs = 0; offset = 0; align = 0; - prop = PyCField_FromDesc(st, desc, i, + int res = PyCField_InitFromDesc(st, (CFieldObject*)prop, desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, pack, big_endian, layout_mode); - if (prop == NULL) { + if (res < 0) { Py_DECREF(pair); return -1; } From cbb308166111a4505b4134d301e9408fea6e1a8f Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 8 Aug 2024 16:10:37 +0200 Subject: [PATCH 06/94] Use goto for error handling --- Modules/_ctypes/stgdict.c | 103 ++++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 50 deletions(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index f430b4399c886c..20df66e0ef7cb4 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -250,12 +250,13 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct int big_endian; int arrays_seen = 0; - if (fields == NULL) + if (fields == NULL) { return 0; + } int rc = PyObject_HasAttrWithError(type, &_Py_ID(_swappedbytes_)); if (rc < 0) { - return -1; + goto error; } if (rc) { big_endian = !PY_BIG_ENDIAN; @@ -267,17 +268,17 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); StgInfo *stginfo; if (PyStgInfo_FromType(st, type, &stginfo) < 0) { - return -1; + goto error; } if (!stginfo) { PyErr_SetString(PyExc_TypeError, "ctypes state is not initialized"); - return -1; + goto error; } PyObject *base = (PyObject *)((PyTypeObject *)type)->tp_base; StgInfo *baseinfo; if (PyStgInfo_FromType(st, base, &baseinfo) < 0) { - return -1; + goto error; } /* If this structure/union is already marked final we cannot assign @@ -286,11 +287,11 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (stginfo->flags & DICTFLAG_FINAL) {/* is final ? */ PyErr_SetString(PyExc_AttributeError, "_fields_ is final"); - return -1; + goto error; } if (PyObject_GetOptionalAttr(type, &_Py_ID(_pack_), &tmp) < 0) { - return -1; + goto error; } if (tmp) { pack = PyLong_AsInt(tmp); @@ -303,7 +304,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct PyErr_SetString(PyExc_ValueError, "_pack_ must be a non-negative integer"); } - return -1; + goto error; } } else { @@ -319,7 +320,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct PyObject *layout_class; if (PyObject_GetOptionalAttr(type, &_Py_ID(_layout_), &tmp) < 0) { - return -1; + goto error; } if (!tmp) { layout_class = _PyImport_GetModuleAttrString("ctypes._layout", @@ -336,7 +337,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (pack > 0) { PyErr_SetString(PyExc_ValueError, "_pack_ is not compatible with _layout_=\"gcc-sysv\""); - return -1; + goto error; } layout_class = _PyImport_GetModuleAttrString("ctypes._layout", "GCCSysVLayout"); @@ -344,7 +345,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct else { PyErr_Format(PyExc_ValueError, "unknown _layout_ %R", tmp); - return -1; + goto error; } Py_DECREF(tmp); } @@ -352,7 +353,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct layout_class = tmp; } if (!layout_class) { - return -1; + goto error; } PyObject *kwnames = PyTuple_Pack( 2, @@ -376,12 +377,12 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct Py_DECREF(base_arg); Py_DECREF(layout_class); if (!layout) { - return -1; + goto error; } tmp = PyObject_GetAttr(layout, &_Py_ID(align)); if (!tmp) { Py_DECREF(layout); - return -1; + goto error; } int forced_alignment = PyLong_AsInt(tmp); Py_DECREF(tmp); @@ -391,14 +392,14 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct "_align_ must be a non-negative integer"); } Py_DECREF(layout); - return -1; + goto error; } PyObject *layout_fields = PySequence_Fast(layout, "layout must return a sequence"); Py_DECREF(layout); if (!layout_fields) { - return -1; + goto error; } len = PySequence_Size(fields); @@ -407,7 +408,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct PyErr_SetString(PyExc_TypeError, "'_fields_' must be a sequence of pairs"); } - return -1; + goto error; } if (len != PySequence_Fast_GET_SIZE(layout_fields)) { @@ -441,7 +442,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct stginfo->ffi_type_pointer.elements = PyMem_New(ffi_type *, baseinfo->length + len + 1); if (stginfo->ffi_type_pointer.elements == NULL) { PyErr_NoMemory(); - return -1; + goto error; } memset(stginfo->ffi_type_pointer.elements, 0, sizeof(ffi_type *) * (baseinfo->length + len + 1)); @@ -461,7 +462,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct stginfo->ffi_type_pointer.elements = PyMem_New(ffi_type *, len + 1); if (stginfo->ffi_type_pointer.elements == NULL) { PyErr_NoMemory(); - return -1; + goto error; } memset(stginfo->ffi_type_pointer.elements, 0, sizeof(ffi_type *) * (len + 1)); @@ -476,7 +477,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct stginfo->format = _ctypes_alloc_format_string(NULL, "B"); } if (stginfo->format == NULL) - return -1; + goto error; for (i = 0; i < len; ++i) { PyObject *name = NULL, *desc = NULL; @@ -488,7 +489,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct PyErr_SetString(PyExc_TypeError, "'_fields_' must be a sequence of (name, C type) pairs"); Py_XDECREF(pair); - return -1; + goto error; } if (PyCArrayTypeObject_Check(st, desc)) { arrays_seen = 1; @@ -497,14 +498,14 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct StgInfo *info; if (PyStgInfo_FromType(st, desc, &info) < 0) { Py_DECREF(pair); - return -1; + goto error; } if (info == NULL) { Py_DECREF(pair); PyErr_Format(PyExc_TypeError, "second item in _fields_ tuple (index %zd) must be a C type", i); - return -1; + goto error; } stginfo->ffi_type_pointer.elements[ffi_ofs + i] = &info->ffi_type_pointer; @@ -536,14 +537,14 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct "bit fields not allowed for type %s", ((PyTypeObject *)desc)->tp_name); Py_DECREF(pair); - return -1; + goto error; } if (bitsize <= 0 || bitsize > info->size * 8) { PyErr_Format(PyExc_ValueError, "number of bits invalid for bit field %R", name); Py_DECREF(pair); - return -1; + goto error; } } else bitsize = 0; @@ -551,14 +552,14 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct prop = PySequence_Fast_GET_ITEM(layout_fields, i); if (!prop) { Py_DECREF(pair); - return -1; + goto error; } Py_INCREF(prop); if (!PyType_IsSubtype(Py_TYPE(prop), st->PyCField_Type)) { PyErr_Format(PyExc_TypeError, "fields must be of type CField, got %T", prop); Py_DECREF(pair); - return -1; + goto error; } @@ -574,7 +575,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (fieldname == NULL) { Py_DECREF(pair); - return -1; + goto error; } /* construct the field now, as `prop->offset` is `offset` with @@ -586,7 +587,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (res < 0) { Py_DECREF(pair); Py_DECREF(prop); - return -1; + goto error; } /* number of bytes between the end of the last field and the start @@ -600,7 +601,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (stginfo->format == NULL) { Py_DECREF(pair); Py_DECREF(prop); - return -1; + goto error; } } @@ -611,7 +612,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct Py_DECREF(pair); Py_DECREF(prop); PyErr_NoMemory(); - return -1; + goto error; } sprintf(buf, "%s:%s:", fieldfmt, fieldname); @@ -628,7 +629,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (stginfo->format == NULL) { Py_DECREF(pair); Py_DECREF(prop); - return -1; + goto error; } } else /* union */ { field_size = 0; @@ -642,7 +643,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct pack, big_endian, layout_mode); if (res < 0) { Py_DECREF(pair); - return -1; + goto error; } union_size = max(size, union_size); } @@ -651,7 +652,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (-1 == PyObject_SetAttr(type, name, prop)) { Py_DECREF(prop); Py_DECREF(pair); - return -1; + goto error; } Py_DECREF(pair); Py_DECREF(prop); @@ -675,7 +676,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct stginfo->format = _ctypes_alloc_format_padding(ptr, padding); PyMem_Free(ptr); if (stginfo->format == NULL) { - return -1; + goto error; } } @@ -683,7 +684,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct stginfo->format = _ctypes_alloc_format_string(stginfo->format, "}"); PyMem_Free(ptr); if (stginfo->format == NULL) - return -1; + goto error; } stginfo->ffi_type_pointer.alignment = Py_SAFE_DOWNCAST(total_align, @@ -787,26 +788,26 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct int bitsize = 0; if (pair == NULL) { - return -1; + goto error; } if (!PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) { PyErr_SetString(PyExc_TypeError, "'_fields_' must be a sequence of (name, C type) pairs"); Py_DECREF(pair); - return -1; + goto error; } StgInfo *info; if (PyStgInfo_FromType(st, desc, &info) < 0) { Py_DECREF(pair); - return -1; + goto error; } if (info == NULL) { Py_DECREF(pair); PyErr_Format(PyExc_TypeError, "second item in _fields_ tuple (index %zd) must be a C type", i); - return -1; + goto error; } if (!PyCArrayTypeObject_Check(st, desc)) { @@ -820,14 +821,14 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct StgInfo *einfo; if (PyStgInfo_FromType(st, info->proto, &einfo) < 0) { Py_DECREF(pair); - return -1; + goto error; } if (einfo == NULL) { Py_DECREF(pair); PyErr_Format(PyExc_TypeError, "second item in _fields_ tuple (index %zd) must be a C type", i); - return -1; + goto error; } /* * We need one extra ffi_type to hold the struct, and one @@ -854,7 +855,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (type_block == NULL) { PyErr_NoMemory(); - return -1; + goto error; } /* * the first block takes up ffi_ofs + len + 1 which is the pointers * @@ -885,7 +886,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (pair == NULL) { PyMem_Free(type_block); - return -1; + goto error; } /* In theory, we made this call in the first pass, so it *shouldn't* * fail. However, you never know, and the code above might change @@ -898,14 +899,14 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct "'_fields_' must be a sequence of (name, C type) pairs"); Py_DECREF(pair); PyMem_Free(type_block); - return -1; + goto error; } StgInfo *info; if (PyStgInfo_FromType(st, desc, &info) < 0) { Py_DECREF(pair); PyMem_Free(type_block); - return -1; + goto error; } /* Possibly this check could be avoided, but see above comment. */ @@ -915,7 +916,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct PyErr_Format(PyExc_TypeError, "second item in _fields_ tuple (index %zd) must be a C type", i); - return -1; + goto error; } assert(element_index < (ffi_ofs + len)); /* will be used below */ @@ -929,7 +930,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (PyStgInfo_FromType(st, info->proto, &einfo) < 0) { Py_DECREF(pair); PyMem_Free(type_block); - return -1; + goto error; } if (einfo == NULL) { Py_DECREF(pair); @@ -937,7 +938,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct PyErr_Format(PyExc_TypeError, "second item in _fields_ tuple (index %zd) must be a C type", i); - return -1; + goto error; } element_types[element_index++] = &structs[struct_index]; structs[struct_index].size = length * einfo->ffi_type_pointer.size; @@ -972,9 +973,11 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (stginfo->flags & DICTFLAG_FINAL) { PyErr_SetString(PyExc_AttributeError, "Structure or union cannot contain itself"); - return -1; + goto error; } stginfo->flags |= DICTFLAG_FINAL; return MakeAnonFields(type); +error: + return -1; } From 1adb23a49b31140a32907eda6fcf967a721e3742 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 8 Aug 2024 16:14:31 +0200 Subject: [PATCH 07/94] Handle Py_DECREF(prop) in the error block --- Modules/_ctypes/stgdict.c | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 20df66e0ef7cb4..2cfa0f594959a4 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -250,6 +250,10 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct int big_endian; int arrays_seen = 0; + // The following are NULL or hold strong references. + // They're cleared on error. + PyObject *prop = NULL; + if (fields == NULL) { return 0; } @@ -482,9 +486,20 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct for (i = 0; i < len; ++i) { PyObject *name = NULL, *desc = NULL; PyObject *pair = PySequence_GetItem(fields, i); - PyObject *prop; Py_ssize_t bitsize = 0; + prop = PySequence_Fast_GET_ITEM(layout_fields, i); + if (!prop) { + goto error; + } + Py_INCREF(prop); + if (!PyType_IsSubtype(Py_TYPE(prop), st->PyCField_Type)) { + PyErr_Format(PyExc_TypeError, + "fields must be of type CField, got %T", prop); + goto error; + + } + if (!pair || !PyArg_ParseTuple(pair, "UO|n", &name, &desc, &bitsize)) { PyErr_SetString(PyExc_TypeError, "'_fields_' must be a sequence of (name, C type) pairs"); @@ -549,20 +564,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } else bitsize = 0; - prop = PySequence_Fast_GET_ITEM(layout_fields, i); - if (!prop) { - Py_DECREF(pair); - goto error; - } - Py_INCREF(prop); - if (!PyType_IsSubtype(Py_TYPE(prop), st->PyCField_Type)) { - PyErr_Format(PyExc_TypeError, - "fields must be of type CField, got %T", prop); - Py_DECREF(pair); - goto error; - - } - if (isStruct) { const char *fieldfmt = info->format ? info->format : "B"; const char *fieldname = PyUnicode_AsUTF8(name); @@ -586,7 +587,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct pack, big_endian, layout_mode); if (res < 0) { Py_DECREF(pair); - Py_DECREF(prop); goto error; } @@ -600,7 +600,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct PyMem_Free(ptr); if (stginfo->format == NULL) { Py_DECREF(pair); - Py_DECREF(prop); goto error; } } @@ -610,7 +609,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct buf = PyMem_Malloc(len + 2 + 1); if (buf == NULL) { Py_DECREF(pair); - Py_DECREF(prop); PyErr_NoMemory(); goto error; } @@ -628,7 +626,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (stginfo->format == NULL) { Py_DECREF(pair); - Py_DECREF(prop); goto error; } } else /* union */ { @@ -650,12 +647,11 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct total_align = max(align, total_align); if (-1 == PyObject_SetAttr(type, name, prop)) { - Py_DECREF(prop); Py_DECREF(pair); goto error; } Py_DECREF(pair); - Py_DECREF(prop); + Py_CLEAR(prop); } if (!isStruct) { @@ -979,5 +975,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct return MakeAnonFields(type); error: + Py_XDECREF(prop); return -1; } From 01ee4f89989ea044415038a93411391d2a3a4d6d Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 8 Aug 2024 16:19:54 +0200 Subject: [PATCH 08/94] Pass bit_size and name to CField --- Lib/ctypes/_layout.py | 5 ++-- Modules/_ctypes/cfield.c | 28 +++++++++++++----- Modules/_ctypes/clinic/cfield.c.h | 48 +++++++++++++++++++------------ Modules/_ctypes/ctypes.h | 3 ++ 4 files changed, 56 insertions(+), 28 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index d9071f1a7e520b..0e4d91d6dc7518 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -33,16 +33,17 @@ def __iter__(self): for field in self.fields: field = tuple(field) try: - name, ftype, bits = field + name, ftype, bit_size = field except ValueError: name, ftype = field - bits = None + bit_size = None size = ctypes.sizeof(ftype) offset = self.offset yield CField( name=name, size=size, offset=offset, + bit_size=-1 if bit_size is None else bit_size, ) self.offset += self.size diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 52f89d7d58e2c9..330818d07abb56 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -200,21 +200,31 @@ PyCField_FromDesc_msvc( @classmethod _ctypes.CField.__new__ as PyCField_new - name: str + name: object(subclass_of='&PyUnicode_Type') size: Py_ssize_t offset: Py_ssize_t + bit_size: Py_ssize_t = -1 [clinic start generated code]*/ static PyObject * -PyCField_new_impl(PyTypeObject *type, const char *name, Py_ssize_t size, - Py_ssize_t offset) -/*[clinic end generated code: output=109dac517c651e4b input=25b7b4a2d83d7ef9]*/ +PyCField_new_impl(PyTypeObject *type, PyObject *name, Py_ssize_t size, + Py_ssize_t offset, Py_ssize_t bit_size) +/*[clinic end generated code: output=c8973d95f5944d97 input=cf50946ed6d90492]*/ { PyTypeObject *tp = type; CFieldObject* self = (CFieldObject *)tp->tp_alloc(tp, 0); + if (PyUnicode_CheckExact(name)) { + self->name = Py_NewRef(name); + } else { + self->name = PyObject_Str(name); + if (!self->name) { + goto error; + } + } self->size = size; self->offset = offset; + self->bit_size = bit_size; self->setfunc = NULL; // XXX self->getfunc = NULL; // XXX @@ -222,6 +232,9 @@ PyCField_new_impl(PyTypeObject *type, const char *name, Py_ssize_t size, self->proto = NULL; // XXX return (PyObject *)self; +error: + Py_DECREF(self); + return NULL; } @@ -231,7 +244,6 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, PyObject *desc, Py_s Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, int big_endian, LayoutMode layout_mode) { - PyTypeObject *tp = st->PyCField_Type; if (self == NULL) { return -1; } @@ -404,8 +416,10 @@ PyCField_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - (void)PyCField_clear((CFieldObject *)self); - Py_TYPE(self)->tp_free((PyObject *)self); + CFieldObject *self_cf = (CFieldObject *)self; + (void)PyCField_clear(self_cf); + Py_CLEAR(self_cf->name); + Py_TYPE(self)->tp_free(self); Py_DECREF(tp); } diff --git a/Modules/_ctypes/clinic/cfield.c.h b/Modules/_ctypes/clinic/cfield.c.h index 895994a0476a75..1f0efa53ce6aef 100644 --- a/Modules/_ctypes/clinic/cfield.c.h +++ b/Modules/_ctypes/clinic/cfield.c.h @@ -10,8 +10,8 @@ preserve #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() static PyObject * -PyCField_new_impl(PyTypeObject *type, const char *name, Py_ssize_t size, - Py_ssize_t offset); +PyCField_new_impl(PyTypeObject *type, PyObject *name, Py_ssize_t size, + Py_ssize_t offset, Py_ssize_t bit_size); static PyObject * PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -19,14 +19,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 3 + #define NUM_KEYWORDS 4 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(size), &_Py_ID(offset), }, + .ob_item = { &_Py_ID(name), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(bit_size), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -35,21 +35,23 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "size", "offset", NULL}; + static const char * const _keywords[] = {"name", "size", "offset", "bit_size", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "CField", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[4]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); - const char *name; + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 3; + PyObject *name; Py_ssize_t size; Py_ssize_t offset; + Py_ssize_t bit_size = -1; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 3, 3, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 3, 4, 0, argsbuf); if (!fastargs) { goto exit; } @@ -57,15 +59,7 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) _PyArg_BadArgument("CField", "argument 'name'", "str", fastargs[0]); goto exit; } - Py_ssize_t name_length; - name = PyUnicode_AsUTF8AndSize(fastargs[0], &name_length); - if (name == NULL) { - goto exit; - } - if (strlen(name) != (size_t)name_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } + name = fastargs[0]; { Py_ssize_t ival = -1; PyObject *iobj = _PyNumber_Index(fastargs[1]); @@ -90,9 +84,25 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } offset = ival; } - return_value = PyCField_new_impl(type, name, size, offset); + if (!noptargs) { + goto skip_optional_pos; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(fastargs[3]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + bit_size = ival; + } +skip_optional_pos: + return_value = PyCField_new_impl(type, name, size, offset, bit_size); exit: return return_value; } -/*[clinic end generated code: output=1f92a9742a307128 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8e63c9e14920d7d4 input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 10c4b23992ff54..f8eb4c31016cb9 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -270,6 +270,9 @@ typedef struct CFieldObject { GETFUNC getfunc; /* getter function if proto is NULL */ SETFUNC setfunc; /* setter function if proto is NULL */ int anonymous; + + Py_ssize_t bit_size; + PyObject *name; } CFieldObject; /**************************************************************** From eb7666cc096f962b5908dbc9447d6e4f8fbf3159 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 8 Aug 2024 16:32:21 +0200 Subject: [PATCH 09/94] Use name from prop --- Modules/_ctypes/ctypes.h | 2 +- Modules/_ctypes/stgdict.c | 34 ++++++++++++++++++---------------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index f8eb4c31016cb9..44fa0f8a0c5004 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -272,7 +272,7 @@ typedef struct CFieldObject { int anonymous; Py_ssize_t bit_size; - PyObject *name; + PyObject *name; /* exact PyUnicode */ } CFieldObject; /**************************************************************** diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 2cfa0f594959a4..7b712e91f2add2 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -252,7 +252,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct // The following are NULL or hold strong references. // They're cleared on error. - PyObject *prop = NULL; + PyObject *prop_obj = NULL; if (fields == NULL) { return 0; @@ -484,23 +484,25 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; for (i = 0; i < len; ++i) { - PyObject *name = NULL, *desc = NULL; + PyObject *desc = NULL; PyObject *pair = PySequence_GetItem(fields, i); Py_ssize_t bitsize = 0; - prop = PySequence_Fast_GET_ITEM(layout_fields, i); - if (!prop) { + prop_obj = PySequence_Fast_GET_ITEM(layout_fields, i); + if (!prop_obj) { goto error; } - Py_INCREF(prop); - if (!PyType_IsSubtype(Py_TYPE(prop), st->PyCField_Type)) { + Py_INCREF(prop_obj); + if (!PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type)) { PyErr_Format(PyExc_TypeError, - "fields must be of type CField, got %T", prop); + "fields must be of type CField, got %T", prop_obj); goto error; } + CFieldObject *prop = (CFieldObject *)prop_obj; // borrow from prop_obj - if (!pair || !PyArg_ParseTuple(pair, "UO|n", &name, &desc, &bitsize)) { + PyObject *_dummy; + if (!pair || !PyArg_ParseTuple(pair, "UO|n", &_dummy, &desc, &bitsize)) { PyErr_SetString(PyExc_TypeError, "'_fields_' must be a sequence of (name, C type) pairs"); Py_XDECREF(pair); @@ -557,7 +559,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (bitsize <= 0 || bitsize > info->size * 8) { PyErr_Format(PyExc_ValueError, "number of bits invalid for bit field %R", - name); + prop->name); Py_DECREF(pair); goto error; } @@ -566,7 +568,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (isStruct) { const char *fieldfmt = info->format ? info->format : "B"; - const char *fieldname = PyUnicode_AsUTF8(name); + const char *fieldname = PyUnicode_AsUTF8(prop->name); char *ptr; Py_ssize_t len; char *buf; @@ -581,7 +583,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct /* construct the field now, as `prop->offset` is `offset` with corrected alignment */ - int res = PyCField_InitFromDesc(st, (CFieldObject*)prop, desc, i, + int res = PyCField_InitFromDesc(st, prop, desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, pack, big_endian, layout_mode); @@ -592,7 +594,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct /* number of bytes between the end of the last field and the start of this one */ - padding = ((CFieldObject *)prop)->offset - last_size; + padding = prop->offset - last_size; if (padding > 0) { ptr = stginfo->format; @@ -634,7 +636,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct bitofs = 0; offset = 0; align = 0; - int res = PyCField_InitFromDesc(st, (CFieldObject*)prop, desc, i, + int res = PyCField_InitFromDesc(st, prop, desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, pack, big_endian, layout_mode); @@ -646,12 +648,12 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } total_align = max(align, total_align); - if (-1 == PyObject_SetAttr(type, name, prop)) { + if (-1 == PyObject_SetAttr(type, prop->name, prop_obj)) { Py_DECREF(pair); goto error; } Py_DECREF(pair); - Py_CLEAR(prop); + Py_CLEAR(prop_obj); } if (!isStruct) { @@ -975,6 +977,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct return MakeAnonFields(type); error: - Py_XDECREF(prop); + Py_XDECREF(prop_obj); return -1; } From be6113ff26286b65633c01ddafa320ccf2fb8a44 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 8 Aug 2024 16:42:14 +0200 Subject: [PATCH 10/94] Use desc from prop --- Lib/ctypes/_layout.py | 1 + Modules/_ctypes/cfield.c | 10 +++++++--- Modules/_ctypes/clinic/cfield.c.h | 28 +++++++++++++++------------- Modules/_ctypes/ctypes.h | 1 + Modules/_ctypes/stgdict.c | 21 +++++++-------------- 5 files changed, 31 insertions(+), 30 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 0e4d91d6dc7518..4f5b417cd40e5a 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -41,6 +41,7 @@ def __iter__(self): offset = self.offset yield CField( name=name, + type=ftype, size=size, offset=offset, bit_size=-1 if bit_size is None else bit_size, diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 330818d07abb56..708340b388ecb4 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -201,6 +201,7 @@ PyCField_FromDesc_msvc( _ctypes.CField.__new__ as PyCField_new name: object(subclass_of='&PyUnicode_Type') + type as desc: object size: Py_ssize_t offset: Py_ssize_t bit_size: Py_ssize_t = -1 @@ -208,9 +209,9 @@ _ctypes.CField.__new__ as PyCField_new [clinic start generated code]*/ static PyObject * -PyCField_new_impl(PyTypeObject *type, PyObject *name, Py_ssize_t size, - Py_ssize_t offset, Py_ssize_t bit_size) -/*[clinic end generated code: output=c8973d95f5944d97 input=cf50946ed6d90492]*/ +PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *desc, + Py_ssize_t size, Py_ssize_t offset, Py_ssize_t bit_size) +/*[clinic end generated code: output=79ce818b0a937343 input=f7d8d0284e8fb708]*/ { PyTypeObject *tp = type; CFieldObject* self = (CFieldObject *)tp->tp_alloc(tp, 0); @@ -222,6 +223,7 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, Py_ssize_t size, goto error; } } + self->desc = Py_NewRef(desc); self->size = size; self->offset = offset; self->bit_size = bit_size; @@ -401,6 +403,7 @@ PyCField_traverse(CFieldObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); Py_VISIT(self->proto); + Py_VISIT(self->desc); return 0; } @@ -408,6 +411,7 @@ static int PyCField_clear(CFieldObject *self) { Py_CLEAR(self->proto); + Py_CLEAR(self->desc); return 0; } diff --git a/Modules/_ctypes/clinic/cfield.c.h b/Modules/_ctypes/clinic/cfield.c.h index 1f0efa53ce6aef..5db556610f3ef5 100644 --- a/Modules/_ctypes/clinic/cfield.c.h +++ b/Modules/_ctypes/clinic/cfield.c.h @@ -10,8 +10,8 @@ preserve #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() static PyObject * -PyCField_new_impl(PyTypeObject *type, PyObject *name, Py_ssize_t size, - Py_ssize_t offset, Py_ssize_t bit_size); +PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *desc, + Py_ssize_t size, Py_ssize_t offset, Py_ssize_t bit_size); static PyObject * PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -19,14 +19,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 4 + #define NUM_KEYWORDS 5 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(bit_size), }, + .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(bit_size), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -35,23 +35,24 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "size", "offset", "bit_size", NULL}; + static const char * const _keywords[] = {"name", "type", "size", "offset", "bit_size", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "CField", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[4]; + PyObject *argsbuf[5]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); - Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 3; + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 4; PyObject *name; + PyObject *desc; Py_ssize_t size; Py_ssize_t offset; Py_ssize_t bit_size = -1; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 3, 4, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 4, 5, 0, argsbuf); if (!fastargs) { goto exit; } @@ -60,9 +61,10 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto exit; } name = fastargs[0]; + desc = fastargs[1]; { Py_ssize_t ival = -1; - PyObject *iobj = _PyNumber_Index(fastargs[1]); + PyObject *iobj = _PyNumber_Index(fastargs[2]); if (iobj != NULL) { ival = PyLong_AsSsize_t(iobj); Py_DECREF(iobj); @@ -74,7 +76,7 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } { Py_ssize_t ival = -1; - PyObject *iobj = _PyNumber_Index(fastargs[2]); + PyObject *iobj = _PyNumber_Index(fastargs[3]); if (iobj != NULL) { ival = PyLong_AsSsize_t(iobj); Py_DECREF(iobj); @@ -89,7 +91,7 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } { Py_ssize_t ival = -1; - PyObject *iobj = _PyNumber_Index(fastargs[3]); + PyObject *iobj = _PyNumber_Index(fastargs[4]); if (iobj != NULL) { ival = PyLong_AsSsize_t(iobj); Py_DECREF(iobj); @@ -100,9 +102,9 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) bit_size = ival; } skip_optional_pos: - return_value = PyCField_new_impl(type, name, size, offset, bit_size); + return_value = PyCField_new_impl(type, name, desc, size, offset, bit_size); exit: return return_value; } -/*[clinic end generated code: output=8e63c9e14920d7d4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1eaa4a49317f1f01 input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 44fa0f8a0c5004..fd5d4110e2c026 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -273,6 +273,7 @@ typedef struct CFieldObject { Py_ssize_t bit_size; PyObject *name; /* exact PyUnicode */ + PyObject *desc; /* underlying ctype; any PyObject */ } CFieldObject; /**************************************************************** diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 7b712e91f2add2..00d591a565912a 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -484,7 +484,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; for (i = 0; i < len; ++i) { - PyObject *desc = NULL; PyObject *pair = PySequence_GetItem(fields, i); Py_ssize_t bitsize = 0; @@ -502,28 +501,22 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct CFieldObject *prop = (CFieldObject *)prop_obj; // borrow from prop_obj PyObject *_dummy; - if (!pair || !PyArg_ParseTuple(pair, "UO|n", &_dummy, &desc, &bitsize)) { + if (!pair || !PyArg_ParseTuple(pair, "UO|n", &_dummy, &_dummy, &bitsize)) { PyErr_SetString(PyExc_TypeError, "'_fields_' must be a sequence of (name, C type) pairs"); Py_XDECREF(pair); goto error; } - if (PyCArrayTypeObject_Check(st, desc)) { + if (PyCArrayTypeObject_Check(st, prop->desc)) { arrays_seen = 1; } StgInfo *info; - if (PyStgInfo_FromType(st, desc, &info) < 0) { + if (PyStgInfo_FromType(st, prop->desc, &info) < 0) { Py_DECREF(pair); goto error; } - if (info == NULL) { - Py_DECREF(pair); - PyErr_Format(PyExc_TypeError, - "second item in _fields_ tuple (index %zd) must be a C type", - i); - goto error; - } + assert(info); stginfo->ffi_type_pointer.elements[ffi_ofs + i] = &info->ffi_type_pointer; if (info->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) @@ -552,7 +545,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct default: PyErr_Format(PyExc_TypeError, "bit fields not allowed for type %s", - ((PyTypeObject *)desc)->tp_name); + ((PyTypeObject *)prop->desc)->tp_name); Py_DECREF(pair); goto error; } @@ -583,7 +576,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct /* construct the field now, as `prop->offset` is `offset` with corrected alignment */ - int res = PyCField_InitFromDesc(st, prop, desc, i, + int res = PyCField_InitFromDesc(st, prop, prop->desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, pack, big_endian, layout_mode); @@ -636,7 +629,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct bitofs = 0; offset = 0; align = 0; - int res = PyCField_InitFromDesc(st, prop, desc, i, + int res = PyCField_InitFromDesc(st, prop, prop->desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, pack, big_endian, layout_mode); From a7f05f92768a5c171ae099630d52f308f89c36de Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 8 Aug 2024 16:46:43 +0200 Subject: [PATCH 11/94] Use bit_size from prop; avoid pair --- Modules/_ctypes/ctypes.h | 6 +++--- Modules/_ctypes/stgdict.c | 29 +++++------------------------ 2 files changed, 8 insertions(+), 27 deletions(-) diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index fd5d4110e2c026..d6f3e68e03d421 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -271,9 +271,9 @@ typedef struct CFieldObject { SETFUNC setfunc; /* setter function if proto is NULL */ int anonymous; - Py_ssize_t bit_size; - PyObject *name; /* exact PyUnicode */ - PyObject *desc; /* underlying ctype; any PyObject */ + Py_ssize_t bit_size; /* -1 if not a bitfield */ + PyObject *name; /* exact PyUnicode */ + PyObject *desc; /* underlying ctype; must have StgInfo */ } CFieldObject; /**************************************************************** diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 00d591a565912a..8b7b9581b78917 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -484,9 +484,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; for (i = 0; i < len; ++i) { - PyObject *pair = PySequence_GetItem(fields, i); - Py_ssize_t bitsize = 0; - prop_obj = PySequence_Fast_GET_ITEM(layout_fields, i); if (!prop_obj) { goto error; @@ -499,21 +496,14 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } CFieldObject *prop = (CFieldObject *)prop_obj; // borrow from prop_obj + Py_ssize_t bitsize = prop->bit_size; - PyObject *_dummy; - if (!pair || !PyArg_ParseTuple(pair, "UO|n", &_dummy, &_dummy, &bitsize)) { - PyErr_SetString(PyExc_TypeError, - "'_fields_' must be a sequence of (name, C type) pairs"); - Py_XDECREF(pair); - goto error; - } if (PyCArrayTypeObject_Check(st, prop->desc)) { arrays_seen = 1; } StgInfo *info; if (PyStgInfo_FromType(st, prop->desc, &info) < 0) { - Py_DECREF(pair); goto error; } assert(info); @@ -523,7 +513,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct stginfo->flags |= TYPEFLAG_HASPOINTER; stginfo->flags |= info->flags & (TYPEFLAG_HASUNION | TYPEFLAG_HASBITFIELD); info->flags |= DICTFLAG_FINAL; /* mark field type final */ - if (PyTuple_Size(pair) == 3) { /* bits specified */ + if (bitsize != -1) { /* bits specified */ stginfo->flags |= TYPEFLAG_HASBITFIELD; switch(info->ffi_type_pointer.type) { case FFI_TYPE_UINT8: @@ -546,18 +536,17 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct PyErr_Format(PyExc_TypeError, "bit fields not allowed for type %s", ((PyTypeObject *)prop->desc)->tp_name); - Py_DECREF(pair); goto error; } if (bitsize <= 0 || bitsize > info->size * 8) { PyErr_Format(PyExc_ValueError, "number of bits invalid for bit field %R", - prop->name); - Py_DECREF(pair); + prop->name, bitsize); goto error; } - } else + } else { bitsize = 0; + } if (isStruct) { const char *fieldfmt = info->format ? info->format : "B"; @@ -570,7 +559,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (fieldname == NULL) { - Py_DECREF(pair); goto error; } @@ -581,7 +569,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct &size, &offset, &align, pack, big_endian, layout_mode); if (res < 0) { - Py_DECREF(pair); goto error; } @@ -594,7 +581,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct stginfo->format = _ctypes_alloc_format_padding(ptr, padding); PyMem_Free(ptr); if (stginfo->format == NULL) { - Py_DECREF(pair); goto error; } } @@ -603,7 +589,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct buf = PyMem_Malloc(len + 2 + 1); if (buf == NULL) { - Py_DECREF(pair); PyErr_NoMemory(); goto error; } @@ -620,7 +605,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct PyMem_Free(buf); if (stginfo->format == NULL) { - Py_DECREF(pair); goto error; } } else /* union */ { @@ -634,7 +618,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct &size, &offset, &align, pack, big_endian, layout_mode); if (res < 0) { - Py_DECREF(pair); goto error; } union_size = max(size, union_size); @@ -642,10 +625,8 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct total_align = max(align, total_align); if (-1 == PyObject_SetAttr(type, prop->name, prop_obj)) { - Py_DECREF(pair); goto error; } - Py_DECREF(pair); Py_CLEAR(prop_obj); } From ef24a0c8a409a44f3edd78f68bf2f0088961ac01 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 8 Aug 2024 16:55:27 +0200 Subject: [PATCH 12/94] Pass None for undefined bit_size, not -1 --- Lib/ctypes/_layout.py | 2 +- Modules/_ctypes/cfield.c | 36 +++++++++++++++++++++++++++---- Modules/_ctypes/clinic/cfield.c.h | 28 ++++++------------------ 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 4f5b417cd40e5a..f7b2d2a765b7c9 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -44,7 +44,7 @@ def __iter__(self): type=ftype, size=size, offset=offset, - bit_size=-1 if bit_size is None else bit_size, + bit_size=bit_size, ) self.offset += self.size diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 708340b388ecb4..4c579caf590e4a 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -204,17 +204,21 @@ _ctypes.CField.__new__ as PyCField_new type as desc: object size: Py_ssize_t offset: Py_ssize_t - bit_size: Py_ssize_t = -1 + bit_size as bit_size_obj: object = None [clinic start generated code]*/ static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *desc, - Py_ssize_t size, Py_ssize_t offset, Py_ssize_t bit_size) -/*[clinic end generated code: output=79ce818b0a937343 input=f7d8d0284e8fb708]*/ + Py_ssize_t size, Py_ssize_t offset, PyObject *bit_size_obj) +/*[clinic end generated code: output=0959e4b1c957425f input=418beea0b785cc4e]*/ { PyTypeObject *tp = type; + ctypes_state *st = get_module_state_by_class(tp); CFieldObject* self = (CFieldObject *)tp->tp_alloc(tp, 0); + if (!self) { + return NULL; + } if (PyUnicode_CheckExact(name)) { self->name = Py_NewRef(name); } else { @@ -223,10 +227,34 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *desc, goto error; } } + + if (bit_size_obj == Py_None) { + self->bit_size = -1; + } else { + self->bit_size = PyLong_AsSsize_t(bit_size_obj); + if (self->bit_size < 0) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_ValueError, + "number of bits invalid for bit field %R", + self->name); + } + goto error; + } + } + + StgInfo *info; + if (PyStgInfo_FromType(st, desc, &info) < 0) { + goto error; + } + if (info == NULL) { + PyErr_Format(PyExc_TypeError, + "type of field %R must be a C type", self->name); + goto error; + } + self->desc = Py_NewRef(desc); self->size = size; self->offset = offset; - self->bit_size = bit_size; self->setfunc = NULL; // XXX self->getfunc = NULL; // XXX diff --git a/Modules/_ctypes/clinic/cfield.c.h b/Modules/_ctypes/clinic/cfield.c.h index 5db556610f3ef5..4b99cbe91d7edd 100644 --- a/Modules/_ctypes/clinic/cfield.c.h +++ b/Modules/_ctypes/clinic/cfield.c.h @@ -11,7 +11,7 @@ preserve static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *desc, - Py_ssize_t size, Py_ssize_t offset, Py_ssize_t bit_size); + Py_ssize_t size, Py_ssize_t offset, PyObject *bit_size_obj); static PyObject * PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -45,14 +45,13 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *argsbuf[5]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); - Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 4; PyObject *name; PyObject *desc; Py_ssize_t size; Py_ssize_t offset; - Py_ssize_t bit_size = -1; + PyObject *bit_size_obj; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 4, 5, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 5, 5, 0, argsbuf); if (!fastargs) { goto exit; } @@ -86,25 +85,10 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } offset = ival; } - if (!noptargs) { - goto skip_optional_pos; - } - { - Py_ssize_t ival = -1; - PyObject *iobj = _PyNumber_Index(fastargs[4]); - if (iobj != NULL) { - ival = PyLong_AsSsize_t(iobj); - Py_DECREF(iobj); - } - if (ival == -1 && PyErr_Occurred()) { - goto exit; - } - bit_size = ival; - } -skip_optional_pos: - return_value = PyCField_new_impl(type, name, desc, size, offset, bit_size); + bit_size_obj = fastargs[4]; + return_value = PyCField_new_impl(type, name, desc, size, offset, bit_size_obj); exit: return return_value; } -/*[clinic end generated code: output=1eaa4a49317f1f01 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1bda1136bd0147ad input=a9049054013a1b77]*/ From 728ec0a6db208be91b3c47aa87a374ab245ae121 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 8 Aug 2024 17:21:41 +0200 Subject: [PATCH 13/94] Fix a refleak --- Modules/_ctypes/stgdict.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 8b7b9581b78917..98993aa8780773 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -253,6 +253,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct // The following are NULL or hold strong references. // They're cleared on error. PyObject *prop_obj = NULL; + PyObject *layout_fields = NULL; if (fields == NULL) { return 0; @@ -399,8 +400,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; } - PyObject *layout_fields = PySequence_Fast(layout, - "layout must return a sequence"); + layout_fields = PySequence_Fast(layout, "layout must return a sequence"); Py_DECREF(layout); if (!layout_fields) { goto error; @@ -629,6 +629,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } Py_CLEAR(prop_obj); } + Py_CLEAR(layout_fields); if (!isStruct) { size = union_size; @@ -952,5 +953,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct return MakeAnonFields(type); error: Py_XDECREF(prop_obj); + Py_XDECREF(layout_fields); return -1; } From 5ef81b3560c005b158de3b6cf2f6ca132109a04b Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 8 Aug 2024 17:34:43 +0200 Subject: [PATCH 14/94] Move bitsize check to CField initializer --- Modules/_ctypes/cfield.c | 49 +++++++++++++++++++++++++++++++-------- Modules/_ctypes/stgdict.c | 29 ----------------------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 4c579caf590e4a..3be3d4fabeac1a 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -228,6 +228,16 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *desc, } } + StgInfo *info; + if (PyStgInfo_FromType(st, desc, &info) < 0) { + goto error; + } + if (info == NULL) { + PyErr_Format(PyExc_TypeError, + "type of field %R must be a C type", self->name); + goto error; + } + if (bit_size_obj == Py_None) { self->bit_size = -1; } else { @@ -240,16 +250,35 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *desc, } goto error; } - } - - StgInfo *info; - if (PyStgInfo_FromType(st, desc, &info) < 0) { - goto error; - } - if (info == NULL) { - PyErr_Format(PyExc_TypeError, - "type of field %R must be a C type", self->name); - goto error; + switch(info->ffi_type_pointer.type) { + case FFI_TYPE_UINT8: + case FFI_TYPE_UINT16: + case FFI_TYPE_UINT32: + case FFI_TYPE_SINT64: + case FFI_TYPE_UINT64: + break; + + case FFI_TYPE_SINT8: + case FFI_TYPE_SINT16: + case FFI_TYPE_SINT32: + if (info->getfunc != _ctypes_get_fielddesc("c")->getfunc + && info->getfunc != _ctypes_get_fielddesc("u")->getfunc) + { + break; + } + _Py_FALLTHROUGH; /* else fall through */ + default: + PyErr_Format(PyExc_TypeError, + "bit fields not allowed for type %s", + ((PyTypeObject*)desc)->tp_name); + goto error; + } + if (self->bit_size <= 0 || self->bit_size > info->size * 8) { + PyErr_Format(PyExc_ValueError, + "number of bits invalid for bit field %R", + self->name); + goto error; + } } self->desc = Py_NewRef(desc); diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 98993aa8780773..3a6cd6e3bc9389 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -515,35 +515,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct info->flags |= DICTFLAG_FINAL; /* mark field type final */ if (bitsize != -1) { /* bits specified */ stginfo->flags |= TYPEFLAG_HASBITFIELD; - switch(info->ffi_type_pointer.type) { - case FFI_TYPE_UINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT64: - case FFI_TYPE_UINT64: - break; - - case FFI_TYPE_SINT8: - case FFI_TYPE_SINT16: - case FFI_TYPE_SINT32: - if (info->getfunc != _ctypes_get_fielddesc("c")->getfunc - && info->getfunc != _ctypes_get_fielddesc("u")->getfunc) - { - break; - } - _Py_FALLTHROUGH; /* else fall through */ - default: - PyErr_Format(PyExc_TypeError, - "bit fields not allowed for type %s", - ((PyTypeObject *)prop->desc)->tp_name); - goto error; - } - if (bitsize <= 0 || bitsize > info->size * 8) { - PyErr_Format(PyExc_ValueError, - "number of bits invalid for bit field %R", - prop->name, bitsize); - goto error; - } } else { bitsize = 0; } From 16c2d775508ba1c37748a5cd024ecfb6b56c6551 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 8 Aug 2024 17:42:30 +0200 Subject: [PATCH 15/94] desc is proto --- Modules/_ctypes/cfield.c | 24 ++++++++++-------------- Modules/_ctypes/clinic/cfield.c.h | 19 ++++++++++++------- Modules/_ctypes/ctypes.h | 5 ++--- Modules/_ctypes/stgdict.c | 8 ++++---- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 3be3d4fabeac1a..3a31784a519a6f 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -201,7 +201,7 @@ PyCField_FromDesc_msvc( _ctypes.CField.__new__ as PyCField_new name: object(subclass_of='&PyUnicode_Type') - type as desc: object + type as proto: object size: Py_ssize_t offset: Py_ssize_t bit_size as bit_size_obj: object = None @@ -209,9 +209,9 @@ _ctypes.CField.__new__ as PyCField_new [clinic start generated code]*/ static PyObject * -PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *desc, +PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, Py_ssize_t size, Py_ssize_t offset, PyObject *bit_size_obj) -/*[clinic end generated code: output=0959e4b1c957425f input=418beea0b785cc4e]*/ +/*[clinic end generated code: output=fd7dec6f142d5f41 input=60fae319c86d236e]*/ { PyTypeObject *tp = type; ctypes_state *st = get_module_state_by_class(tp); @@ -229,7 +229,7 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *desc, } StgInfo *info; - if (PyStgInfo_FromType(st, desc, &info) < 0) { + if (PyStgInfo_FromType(st, proto, &info) < 0) { goto error; } if (info == NULL) { @@ -270,7 +270,7 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *desc, default: PyErr_Format(PyExc_TypeError, "bit fields not allowed for type %s", - ((PyTypeObject*)desc)->tp_name); + ((PyTypeObject*)proto)->tp_name); goto error; } if (self->bit_size <= 0 || self->bit_size > info->size * 8) { @@ -281,14 +281,13 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *desc, } } - self->desc = Py_NewRef(desc); + self->proto = Py_NewRef(proto); self->size = size; self->offset = offset; self->setfunc = NULL; // XXX self->getfunc = NULL; // XXX self->index = 0; // XXX - self->proto = NULL; // XXX return (PyObject *)self; error: @@ -298,7 +297,7 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *desc, int -PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, PyObject *desc, Py_ssize_t index, +PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, Py_ssize_t index, Py_ssize_t *pfield_size, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, int big_endian, LayoutMode layout_mode) @@ -306,8 +305,9 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, PyObject *desc, Py_s if (self == NULL) { return -1; } + assert(self->proto); StgInfo *info; - if (PyStgInfo_FromType(st, desc, &info) < 0) { + if (PyStgInfo_FromType(st, self->proto, &info) < 0) { return -1; } if (!info) { @@ -316,7 +316,7 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, PyObject *desc, Py_s return -1; } - PyObject* proto = desc; + PyObject* proto = self->proto; /* Field descriptors for 'c_char * n' are be scpecial cased to return a Python string instead of an Array object instance... @@ -356,8 +356,6 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, PyObject *desc, Py_s self->getfunc = getfunc; self->index = index; - self->proto = Py_NewRef(proto); - int is_bitfield = !!bitsize; if(!is_bitfield) { assert(info->size >= 0); @@ -460,7 +458,6 @@ PyCField_traverse(CFieldObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); Py_VISIT(self->proto); - Py_VISIT(self->desc); return 0; } @@ -468,7 +465,6 @@ static int PyCField_clear(CFieldObject *self) { Py_CLEAR(self->proto); - Py_CLEAR(self->desc); return 0; } diff --git a/Modules/_ctypes/clinic/cfield.c.h b/Modules/_ctypes/clinic/cfield.c.h index 4b99cbe91d7edd..f9f930cc47d901 100644 --- a/Modules/_ctypes/clinic/cfield.c.h +++ b/Modules/_ctypes/clinic/cfield.c.h @@ -10,7 +10,7 @@ preserve #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() static PyObject * -PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *desc, +PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, Py_ssize_t size, Py_ssize_t offset, PyObject *bit_size_obj); static PyObject * @@ -45,13 +45,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *argsbuf[5]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 4; PyObject *name; - PyObject *desc; + PyObject *proto; Py_ssize_t size; Py_ssize_t offset; - PyObject *bit_size_obj; + PyObject *bit_size_obj = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 5, 5, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 4, 5, 0, argsbuf); if (!fastargs) { goto exit; } @@ -60,7 +61,7 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto exit; } name = fastargs[0]; - desc = fastargs[1]; + proto = fastargs[1]; { Py_ssize_t ival = -1; PyObject *iobj = _PyNumber_Index(fastargs[2]); @@ -85,10 +86,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } offset = ival; } + if (!noptargs) { + goto skip_optional_pos; + } bit_size_obj = fastargs[4]; - return_value = PyCField_new_impl(type, name, desc, size, offset, bit_size_obj); +skip_optional_pos: + return_value = PyCField_new_impl(type, name, proto, size, offset, bit_size_obj); exit: return return_value; } -/*[clinic end generated code: output=1bda1136bd0147ad input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ffea985028c6e110 input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index d6f3e68e03d421..6d59f922504aef 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -223,7 +223,7 @@ typedef enum { struct CFieldObject; extern int -PyCField_InitFromDesc(ctypes_state *st, struct CFieldObject* self, PyObject *desc, Py_ssize_t index, +PyCField_InitFromDesc(ctypes_state *st, struct CFieldObject* self, Py_ssize_t index, Py_ssize_t *pfield_size, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, @@ -266,14 +266,13 @@ typedef struct CFieldObject { Py_ssize_t size; Py_ssize_t index; /* Index into CDataObject's object array */ - PyObject *proto; /* a type or NULL */ + PyObject *proto; /* underlying ctype; must have StgInfo */ GETFUNC getfunc; /* getter function if proto is NULL */ SETFUNC setfunc; /* setter function if proto is NULL */ int anonymous; Py_ssize_t bit_size; /* -1 if not a bitfield */ PyObject *name; /* exact PyUnicode */ - PyObject *desc; /* underlying ctype; must have StgInfo */ } CFieldObject; /**************************************************************** diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 3a6cd6e3bc9389..649303ffb6e20c 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -498,12 +498,12 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct CFieldObject *prop = (CFieldObject *)prop_obj; // borrow from prop_obj Py_ssize_t bitsize = prop->bit_size; - if (PyCArrayTypeObject_Check(st, prop->desc)) { + if (PyCArrayTypeObject_Check(st, prop->proto)) { arrays_seen = 1; } StgInfo *info; - if (PyStgInfo_FromType(st, prop->desc, &info) < 0) { + if (PyStgInfo_FromType(st, prop->proto, &info) < 0) { goto error; } assert(info); @@ -535,7 +535,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct /* construct the field now, as `prop->offset` is `offset` with corrected alignment */ - int res = PyCField_InitFromDesc(st, prop, prop->desc, i, + int res = PyCField_InitFromDesc(st, prop, i, &field_size, bitsize, &bitofs, &size, &offset, &align, pack, big_endian, layout_mode); @@ -584,7 +584,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct bitofs = 0; offset = 0; align = 0; - int res = PyCField_InitFromDesc(st, prop, prop->desc, i, + int res = PyCField_InitFromDesc(st, prop, i, &field_size, bitsize, &bitofs, &size, &offset, &align, pack, big_endian, layout_mode); From 14fd92d860938d3cd2405223a7be16292431a6ec Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 8 Aug 2024 17:46:04 +0200 Subject: [PATCH 16/94] Remove the unused TYPEFLAG_HASBITFIELD and TYPEFLAG_HASUNION --- Modules/_ctypes/_ctypes.c | 7 ++++--- Modules/_ctypes/ctypes.h | 2 -- Modules/_ctypes/stgdict.c | 10 +--------- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 9c43fbe0e42358..078e756da51f91 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -664,9 +664,6 @@ StructUnionType_init(PyObject *self, PyObject *args, PyObject *kwds, int isStruc Py_DECREF(attrdict); return -1; } - if (!isStruct) { - info->flags |= TYPEFLAG_HASUNION; - } info->format = _ctypes_alloc_format_string(NULL, "B"); if (info->format == NULL) { @@ -2534,6 +2531,10 @@ converters_from_argtypes(ctypes_state *st, PyObject *ob) return -1; } + // TYPEFLAG_HASUNION and TYPEFLAG_HASBITFIELD used to be set + // if there were any unions/bitfields; + // if the check is re-enabled we either need to loop here or + // restore the flag if (stginfo != NULL) { if (stginfo->flags & TYPEFLAG_HASUNION) { Py_DECREF(converters); diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 6d59f922504aef..299a9e78bbce9b 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -383,8 +383,6 @@ PyObject *_ctypes_callproc(ctypes_state *st, #define TYPEFLAG_ISPOINTER 0x100 #define TYPEFLAG_HASPOINTER 0x200 -#define TYPEFLAG_HASUNION 0x400 -#define TYPEFLAG_HASBITFIELD 0x800 #define DICTFLAG_FINAL 0x1000 diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 649303ffb6e20c..77d45d8a5b4407 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -429,13 +429,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct PyMem_Free(stginfo->ffi_type_pointer.elements); } - if (baseinfo) { - stginfo->flags |= (baseinfo->flags & - (TYPEFLAG_HASUNION | TYPEFLAG_HASBITFIELD)); - } - if (!isStruct) { - stginfo->flags |= TYPEFLAG_HASUNION; - } if (baseinfo) { size = offset = baseinfo->size; align = baseinfo->align; @@ -511,10 +504,9 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct stginfo->ffi_type_pointer.elements[ffi_ofs + i] = &info->ffi_type_pointer; if (info->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) stginfo->flags |= TYPEFLAG_HASPOINTER; - stginfo->flags |= info->flags & (TYPEFLAG_HASUNION | TYPEFLAG_HASBITFIELD); info->flags |= DICTFLAG_FINAL; /* mark field type final */ if (bitsize != -1) { /* bits specified */ - stginfo->flags |= TYPEFLAG_HASBITFIELD; + // empty } else { bitsize = 0; } From e3337c23b5bb18eddc2042c0a169722a40a0288f Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 8 Aug 2024 18:03:15 +0200 Subject: [PATCH 17/94] Use -1 for "not is_bitfield" --- Modules/_ctypes/cfield.c | 2 +- Modules/_ctypes/stgdict.c | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 3a31784a519a6f..39ce63add620bb 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -356,7 +356,7 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, Py_ssize_t index, self->getfunc = getfunc; self->index = index; - int is_bitfield = !!bitsize; + int is_bitfield = bitsize >= 0; if(!is_bitfield) { assert(info->size >= 0); // assert: no overflow; diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 77d45d8a5b4407..8b1b8d8d3861ac 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -505,11 +505,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (info->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) stginfo->flags |= TYPEFLAG_HASPOINTER; info->flags |= DICTFLAG_FINAL; /* mark field type final */ - if (bitsize != -1) { /* bits specified */ - // empty - } else { - bitsize = 0; - } if (isStruct) { const char *fieldfmt = info->format ? info->format : "B"; From f3c678d19d482c1238b4c7d14c811970af0927ca Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 9 Aug 2024 15:58:34 +0200 Subject: [PATCH 18/94] Handle _swappedbytes_ in Python --- Lib/ctypes/_layout.py | 3 +++ Modules/_ctypes/cfield.c | 16 ++++++++++++---- Modules/_ctypes/clinic/cfield.c.h | 29 ++++++++++++++++++++--------- Modules/_ctypes/ctypes.h | 3 ++- Modules/_ctypes/stgdict.c | 16 ++-------------- 5 files changed, 39 insertions(+), 28 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index f7b2d2a765b7c9..48c3fdc126be46 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -27,6 +27,8 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): # Setting `_align_ = 0` amounts to using the default alignment self.align == 1 + self.swapped_bytes = hasattr(cls, '_swappedbytes_') + self.total_align = max(self.align, base_align) def __iter__(self): @@ -45,6 +47,7 @@ def __iter__(self): size=size, offset=offset, bit_size=bit_size, + swapped_bytes=self.swapped_bytes, ) self.offset += self.size diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 39ce63add620bb..5972a17ef39755 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -205,13 +205,15 @@ _ctypes.CField.__new__ as PyCField_new size: Py_ssize_t offset: Py_ssize_t bit_size as bit_size_obj: object = None + swapped_bytes: bool = False [clinic start generated code]*/ static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, - Py_ssize_t size, Py_ssize_t offset, PyObject *bit_size_obj) -/*[clinic end generated code: output=fd7dec6f142d5f41 input=60fae319c86d236e]*/ + Py_ssize_t size, Py_ssize_t offset, PyObject *bit_size_obj, + int swapped_bytes) +/*[clinic end generated code: output=0eb12310fd6fd816 input=a71fc465f25b600a]*/ { PyTypeObject *tp = type; ctypes_state *st = get_module_state_by_class(tp); @@ -285,6 +287,12 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, self->size = size; self->offset = offset; + if (swapped_bytes) { + self->big_endian = !PY_BIG_ENDIAN; + } else { + self->big_endian = PY_BIG_ENDIAN; + } + self->setfunc = NULL; // XXX self->getfunc = NULL; // XXX self->index = 0; // XXX @@ -300,7 +308,7 @@ int PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, Py_ssize_t index, Py_ssize_t *pfield_size, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int big_endian, LayoutMode layout_mode) + int pack, LayoutMode layout_mode) { if (self == NULL) { return -1; @@ -389,7 +397,7 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, Py_ssize_t index, return -1; } assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); - if(big_endian && is_bitfield) { + if(self->big_endian && is_bitfield) { self->size = BUILD_SIZE(NUM_BITS(self->size), 8*info->size - LOW_BIT(self->size) - bitsize); } return 0; diff --git a/Modules/_ctypes/clinic/cfield.c.h b/Modules/_ctypes/clinic/cfield.c.h index f9f930cc47d901..ac7e4786eaef32 100644 --- a/Modules/_ctypes/clinic/cfield.c.h +++ b/Modules/_ctypes/clinic/cfield.c.h @@ -11,7 +11,8 @@ preserve static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, - Py_ssize_t size, Py_ssize_t offset, PyObject *bit_size_obj); + Py_ssize_t size, Py_ssize_t offset, PyObject *bit_size_obj, + int swapped_bytes); static PyObject * PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -19,14 +20,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 5 + #define NUM_KEYWORDS 6 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(bit_size), }, + .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(bit_size), &_Py_ID(swapped_bytes), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -35,14 +36,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "type", "size", "offset", "bit_size", NULL}; + static const char * const _keywords[] = {"name", "type", "size", "offset", "bit_size", "swapped_bytes", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "CField", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[5]; + PyObject *argsbuf[6]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 4; @@ -51,8 +52,9 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t size; Py_ssize_t offset; PyObject *bit_size_obj = Py_None; + int swapped_bytes = 0; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 4, 5, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 4, 6, 0, argsbuf); if (!fastargs) { goto exit; } @@ -89,11 +91,20 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (!noptargs) { goto skip_optional_pos; } - bit_size_obj = fastargs[4]; + if (fastargs[4]) { + bit_size_obj = fastargs[4]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + swapped_bytes = PyObject_IsTrue(fastargs[5]); + if (swapped_bytes < 0) { + goto exit; + } skip_optional_pos: - return_value = PyCField_new_impl(type, name, proto, size, offset, bit_size_obj); + return_value = PyCField_new_impl(type, name, proto, size, offset, bit_size_obj, swapped_bytes); exit: return return_value; } -/*[clinic end generated code: output=ffea985028c6e110 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a6201dc80984b02d input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 299a9e78bbce9b..565dad1bded218 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -227,7 +227,7 @@ PyCField_InitFromDesc(ctypes_state *st, struct CFieldObject* self, Py_ssize_t in Py_ssize_t *pfield_size, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int is_big_endian, LayoutMode layout_mode); + int pack, LayoutMode layout_mode); extern PyObject *PyCData_AtAddress(ctypes_state *st, PyObject *type, void *buf); extern PyObject *PyCData_FromBytes(ctypes_state *st, PyObject *type, char *data, Py_ssize_t length); @@ -273,6 +273,7 @@ typedef struct CFieldObject { Py_ssize_t bit_size; /* -1 if not a bitfield */ PyObject *name; /* exact PyUnicode */ + bool big_endian; /* boolean */ } CFieldObject; /**************************************************************** diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 8b1b8d8d3861ac..8d238091f1e484 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -247,7 +247,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct PyObject *tmp; int pack; Py_ssize_t ffi_ofs; - int big_endian; int arrays_seen = 0; // The following are NULL or hold strong references. @@ -259,17 +258,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct return 0; } - int rc = PyObject_HasAttrWithError(type, &_Py_ID(_swappedbytes_)); - if (rc < 0) { - goto error; - } - if (rc) { - big_endian = !PY_BIG_ENDIAN; - } - else { - big_endian = PY_BIG_ENDIAN; - } - ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); StgInfo *stginfo; if (PyStgInfo_FromType(st, type, &stginfo) < 0) { @@ -525,7 +513,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct int res = PyCField_InitFromDesc(st, prop, i, &field_size, bitsize, &bitofs, &size, &offset, &align, - pack, big_endian, layout_mode); + pack, layout_mode); if (res < 0) { goto error; } @@ -574,7 +562,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct int res = PyCField_InitFromDesc(st, prop, i, &field_size, bitsize, &bitofs, &size, &offset, &align, - pack, big_endian, layout_mode); + pack, layout_mode); if (res < 0) { goto error; } From 0c47fa24983304747c07b14a283d0244c21e8ba9 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 9 Aug 2024 16:17:43 +0200 Subject: [PATCH 19/94] Put layout mode in the field --- Lib/ctypes/_layout.py | 26 +++++++++++++++++++-- Modules/_ctypes/cfield.c | 10 ++++---- Modules/_ctypes/clinic/cfield.c.h | 30 ++++++++++++++++-------- Modules/_ctypes/ctypes.h | 8 ++----- Modules/_ctypes/stgdict.c | 38 ++++--------------------------- 5 files changed, 57 insertions(+), 55 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 48c3fdc126be46..3bbb83e1b7719d 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -48,17 +48,39 @@ def __iter__(self): offset=offset, bit_size=bit_size, swapped_bytes=self.swapped_bytes, + **self._field_args(), ) self.offset += self.size + def _field_args(self): + return {} + class WindowsLayout(_BaseLayout): - pass + def _field_args(self): + return {'_ms': True} class GCCSysVLayout(_BaseLayout): - pass + def __init__(self, cls, *args, **kwargs): + if getattr(cls, '_pack_', None): + raise ValueError('_pack_ is not compatible with gcc-sysv layout') + return super().__init__(cls, *args, **kwargs) if sys.platform == 'win32': NativeLayout = WindowsLayout + DefaultLayout = WindowsLayout else: NativeLayout = GCCSysVLayout + +def default_layout(cls, *args, **kwargs): + layout = getattr(cls, '_layout_', None) + if layout is None: + if sys.platform == 'win32' or getattr(cls, '_pack_', None): + return WindowsLayout(cls, *args, **kwargs) + return GCCSysVLayout(cls, *args, **kwargs) + elif layout == 'ms': + return WindowsLayout(cls, *args, **kwargs) + elif layout == 'gcc-sysv': + return GCCSysVLayout(cls, *args, **kwargs) + else: + raise ValueError(f'unknown _layout_: {layout!r}') diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 5972a17ef39755..cbc446a050251a 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -206,14 +206,15 @@ _ctypes.CField.__new__ as PyCField_new offset: Py_ssize_t bit_size as bit_size_obj: object = None swapped_bytes: bool = False + _ms: bool = False [clinic start generated code]*/ static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, Py_ssize_t size, Py_ssize_t offset, PyObject *bit_size_obj, - int swapped_bytes) -/*[clinic end generated code: output=0eb12310fd6fd816 input=a71fc465f25b600a]*/ + int swapped_bytes, int _ms) +/*[clinic end generated code: output=ce7cebd241952cb1 input=2857465d20fd1002]*/ { PyTypeObject *tp = type; ctypes_state *st = get_module_state_by_class(tp); @@ -292,6 +293,7 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, } else { self->big_endian = PY_BIG_ENDIAN; } + self->_ms_layout = _ms; self->setfunc = NULL; // XXX self->getfunc = NULL; // XXX @@ -308,7 +310,7 @@ int PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, Py_ssize_t index, Py_ssize_t *pfield_size, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, LayoutMode layout_mode) + int pack) { if (self == NULL) { return -1; @@ -376,7 +378,7 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, Py_ssize_t index, assert(bitsize <= info->size * 8); int result; - if (layout_mode == LAYOUT_MODE_MS) { + if (self->_ms_layout) { result = PyCField_FromDesc_msvc( pfield_size, bitsize, pbitofs, psize, poffset, palign, diff --git a/Modules/_ctypes/clinic/cfield.c.h b/Modules/_ctypes/clinic/cfield.c.h index ac7e4786eaef32..e05e3aac642a92 100644 --- a/Modules/_ctypes/clinic/cfield.c.h +++ b/Modules/_ctypes/clinic/cfield.c.h @@ -12,7 +12,7 @@ preserve static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, Py_ssize_t size, Py_ssize_t offset, PyObject *bit_size_obj, - int swapped_bytes); + int swapped_bytes, int _ms); static PyObject * PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -20,14 +20,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 6 + #define NUM_KEYWORDS 7 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(bit_size), &_Py_ID(swapped_bytes), }, + .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(bit_size), &_Py_ID(swapped_bytes), &_Py_ID(_ms), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -36,14 +36,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "type", "size", "offset", "bit_size", "swapped_bytes", NULL}; + static const char * const _keywords[] = {"name", "type", "size", "offset", "bit_size", "swapped_bytes", "_ms", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "CField", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[6]; + PyObject *argsbuf[7]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 4; @@ -53,8 +53,9 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t offset; PyObject *bit_size_obj = Py_None; int swapped_bytes = 0; + int _ms = 0; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 4, 6, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 4, 7, 0, argsbuf); if (!fastargs) { goto exit; } @@ -97,14 +98,23 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_pos; } } - swapped_bytes = PyObject_IsTrue(fastargs[5]); - if (swapped_bytes < 0) { + if (fastargs[5]) { + swapped_bytes = PyObject_IsTrue(fastargs[5]); + if (swapped_bytes < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + _ms = PyObject_IsTrue(fastargs[6]); + if (_ms < 0) { goto exit; } skip_optional_pos: - return_value = PyCField_new_impl(type, name, proto, size, offset, bit_size_obj, swapped_bytes); + return_value = PyCField_new_impl(type, name, proto, size, offset, bit_size_obj, swapped_bytes, _ms); exit: return return_value; } -/*[clinic end generated code: output=a6201dc80984b02d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c391d6260d1a8db0 input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 565dad1bded218..7473bf582effb3 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -216,18 +216,13 @@ extern int PyObject_stginfo(PyObject *self, Py_ssize_t *psize, Py_ssize_t *palig extern struct fielddesc *_ctypes_get_fielddesc(const char *fmt); -typedef enum { - LAYOUT_MODE_MS, - LAYOUT_MODE_GCC_SYSV, -} LayoutMode; - struct CFieldObject; extern int PyCField_InitFromDesc(ctypes_state *st, struct CFieldObject* self, Py_ssize_t index, Py_ssize_t *pfield_size, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, LayoutMode layout_mode); + int pack); extern PyObject *PyCData_AtAddress(ctypes_state *st, PyObject *type, void *buf); extern PyObject *PyCData_FromBytes(ctypes_state *st, PyObject *type, char *data, Py_ssize_t length); @@ -274,6 +269,7 @@ typedef struct CFieldObject { Py_ssize_t bit_size; /* -1 if not a bitfield */ PyObject *name; /* exact PyUnicode */ bool big_endian; /* boolean */ + bool _ms_layout; } CFieldObject; /**************************************************************** diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 8d238091f1e484..1d1533614c8241 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -305,42 +305,14 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct pack = 0; } - #ifdef MS_WIN32 - LayoutMode layout_mode = LAYOUT_MODE_MS; - #else - LayoutMode layout_mode = (pack > 0) ? LAYOUT_MODE_MS : LAYOUT_MODE_GCC_SYSV; - #endif - PyObject *layout_class; if (PyObject_GetOptionalAttr(type, &_Py_ID(_layout_), &tmp) < 0) { goto error; } - if (!tmp) { + if (!tmp || PyUnicode_Check(tmp)) { layout_class = _PyImport_GetModuleAttrString("ctypes._layout", - "NativeLayout"); - } - else if (PyUnicode_Check(tmp)) { - if (PyUnicode_CompareWithASCIIString(tmp, "ms") == 0) { - layout_mode = LAYOUT_MODE_MS; - layout_class = _PyImport_GetModuleAttrString("ctypes._layout", - "WindowsLayout"); - } - else if (PyUnicode_CompareWithASCIIString(tmp, "gcc-sysv") == 0) { - layout_mode = LAYOUT_MODE_GCC_SYSV; - if (pack > 0) { - PyErr_SetString(PyExc_ValueError, - "_pack_ is not compatible with _layout_=\"gcc-sysv\""); - goto error; - } - layout_class = _PyImport_GetModuleAttrString("ctypes._layout", - "GCCSysVLayout"); - } - else { - PyErr_Format(PyExc_ValueError, - "unknown _layout_ %R", tmp); - goto error; - } - Py_DECREF(tmp); + "default_layout"); + Py_XDECREF(tmp); } else { layout_class = tmp; @@ -513,7 +485,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct int res = PyCField_InitFromDesc(st, prop, i, &field_size, bitsize, &bitofs, &size, &offset, &align, - pack, layout_mode); + pack); if (res < 0) { goto error; } @@ -562,7 +534,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct int res = PyCField_InitFromDesc(st, prop, i, &field_size, bitsize, &bitofs, &size, &offset, &align, - pack, layout_mode); + pack); if (res < 0) { goto error; } From 2eefab13a7772d6f7063155c3f1e85d9c5628e81 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 9 Aug 2024 16:25:00 +0200 Subject: [PATCH 20/94] error handling --- Modules/_ctypes/stgdict.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 1d1533614c8241..d49689d9e5a658 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -324,6 +324,9 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct 2, &_Py_ID(is_struct), &_Py_ID(base)); + if (!kwnames) { + goto error; + } PyObject *base_arg = Py_NewRef(baseinfo ? base : Py_None); PyObject *layout = PyObject_Vectorcall( layout_class, From 5b1723c1b4520335147a033f62f58a1ec1581a4f Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 22 Aug 2024 15:49:26 +0200 Subject: [PATCH 21/94] Take out is_bitfield arg --- Modules/_ctypes/cfield.c | 32 +++++++++++++++++--------------- Modules/_ctypes/ctypes.h | 2 +- Modules/_ctypes/stgdict.c | 5 ++--- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index cbc446a050251a..bf21816d7703a5 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -72,6 +72,12 @@ Py_ssize_t LOW_BIT(Py_ssize_t offset); static inline Py_ssize_t BUILD_SIZE(Py_ssize_t bitsize, Py_ssize_t offset); +static bool +_cfield_is_bitfield(CFieldObject* self) +{ + return self->bit_size >= 0; +} + /* PyCField_FromDesc creates and returns a struct/union field descriptor. The function expects to be called repeatedly for all fields in a struct or @@ -104,8 +110,7 @@ PyCField_FromDesc manages: static int PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - CFieldObject* self, StgInfo* info, - int is_bitfield + CFieldObject* self, StgInfo* info ) { // We don't use poffset here, so clear it, if it has been set. @@ -131,7 +136,7 @@ PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, assert(*poffset == 0); self->offset = round_down(*pbitofs, 8*info->align) / 8; - if(is_bitfield) { + if(_cfield_is_bitfield(self)) { Py_ssize_t effective_bitsof = *pbitofs - 8 * self->offset; self->size = BUILD_SIZE(bitsize, effective_bitsof); assert(effective_bitsof <= info->size * 8); @@ -150,8 +155,7 @@ PyCField_FromDesc_msvc( Py_ssize_t *pfield_size, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, - CFieldObject* self, StgInfo* info, - int is_bitfield + CFieldObject* self, StgInfo* info ) { if (pack) { @@ -181,7 +185,7 @@ PyCField_FromDesc_msvc( assert(8 * info->size == *pfield_size); self->offset = *poffset - (*pfield_size) / 8; - if(is_bitfield) { + if(_cfield_is_bitfield(self)) { assert(0 <= (*pfield_size + *pbitofs)); assert((*pfield_size + *pbitofs) < info->size * 8); self->size = BUILD_SIZE(bitsize, *pfield_size + *pbitofs); @@ -308,7 +312,7 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, int PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, Py_ssize_t index, - Py_ssize_t *pfield_size, Py_ssize_t bitsize, + Py_ssize_t *pfield_size, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack) { @@ -366,8 +370,8 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, Py_ssize_t index, self->getfunc = getfunc; self->index = index; - int is_bitfield = bitsize >= 0; - if(!is_bitfield) { + Py_ssize_t bitsize = self->bit_size; + if(!_cfield_is_bitfield(self)) { assert(info->size >= 0); // assert: no overflow; assert((unsigned long long int) info->size @@ -383,23 +387,21 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, Py_ssize_t index, pfield_size, bitsize, pbitofs, psize, poffset, palign, pack, - self, info, - is_bitfield + self, info ); } else { assert(pack == 0); result = PyCField_FromDesc_gcc( bitsize, pbitofs, psize, poffset, palign, - self, info, - is_bitfield + self, info ); } if (result < 0) { return -1; } - assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); - if(self->big_endian && is_bitfield) { + assert(!_cfield_is_bitfield(self) || (LOW_BIT(self->size) <= self->size * 8)); + if(self->big_endian && _cfield_is_bitfield(self)) { self->size = BUILD_SIZE(NUM_BITS(self->size), 8*info->size - LOW_BIT(self->size) - bitsize); } return 0; diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 7473bf582effb3..786f0873bff36d 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -219,7 +219,7 @@ extern struct fielddesc *_ctypes_get_fielddesc(const char *fmt); struct CFieldObject; extern int PyCField_InitFromDesc(ctypes_state *st, struct CFieldObject* self, Py_ssize_t index, - Py_ssize_t *pfield_size, Py_ssize_t bitsize, + Py_ssize_t *pfield_size, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack); diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index d49689d9e5a658..eea5386f0b8858 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -452,7 +452,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } CFieldObject *prop = (CFieldObject *)prop_obj; // borrow from prop_obj - Py_ssize_t bitsize = prop->bit_size; if (PyCArrayTypeObject_Check(st, prop->proto)) { arrays_seen = 1; @@ -486,7 +485,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct /* construct the field now, as `prop->offset` is `offset` with corrected alignment */ int res = PyCField_InitFromDesc(st, prop, i, - &field_size, bitsize, &bitofs, + &field_size, &bitofs, &size, &offset, &align, pack); if (res < 0) { @@ -535,7 +534,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct offset = 0; align = 0; int res = PyCField_InitFromDesc(st, prop, i, - &field_size, bitsize, &bitofs, + &field_size, &bitofs, &size, &offset, &align, pack); if (res < 0) { From 9507752540704990e20837092c320e0000252426 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 22 Aug 2024 16:04:31 +0200 Subject: [PATCH 22/94] Keep _pack_ on the field --- Lib/ctypes/_layout.py | 3 +++ Modules/_ctypes/cfield.c | 35 ++++++++++++++++++++++--------- Modules/_ctypes/clinic/cfield.c.h | 29 +++++++++++++++---------- Modules/_ctypes/ctypes.h | 4 ++-- Modules/_ctypes/stgdict.c | 6 ++---- 5 files changed, 50 insertions(+), 27 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 3bbb83e1b7719d..525e9c59fe37e4 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -31,6 +31,8 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): self.total_align = max(self.align, base_align) + self._pack_ = getattr(cls, '_pack_', None) + def __iter__(self): for field in self.fields: field = tuple(field) @@ -48,6 +50,7 @@ def __iter__(self): offset=offset, bit_size=bit_size, swapped_bytes=self.swapped_bytes, + pack=self._pack_, **self._field_args(), ) self.offset += self.size diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index bf21816d7703a5..f50c55945bf7ce 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -117,6 +117,8 @@ PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, *pbitofs += *poffset * 8; *poffset = 0; + assert(self->pack == 0); // TODO: This shouldn't be a C assertion + *palign = info->align; if (bitsize > 0) { @@ -154,12 +156,12 @@ static int PyCField_FromDesc_msvc( Py_ssize_t *pfield_size, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, - Py_ssize_t *palign, int pack, + Py_ssize_t *palign, CFieldObject* self, StgInfo* info ) { - if (pack) { - *palign = Py_MIN(pack, info->align); + if (self->pack) { + *palign = Py_MIN(self->pack, info->align); } else { *palign = info->align; } @@ -211,14 +213,15 @@ _ctypes.CField.__new__ as PyCField_new bit_size as bit_size_obj: object = None swapped_bytes: bool = False _ms: bool = False + pack as pack_obj: object = None [clinic start generated code]*/ static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, Py_ssize_t size, Py_ssize_t offset, PyObject *bit_size_obj, - int swapped_bytes, int _ms) -/*[clinic end generated code: output=ce7cebd241952cb1 input=2857465d20fd1002]*/ + int swapped_bytes, int _ms, PyObject *pack_obj) +/*[clinic end generated code: output=96c276e7033f8fe2 input=6f5e15c4b2015849]*/ { PyTypeObject *tp = type; ctypes_state *st = get_module_state_by_class(tp); @@ -247,7 +250,8 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, if (bit_size_obj == Py_None) { self->bit_size = -1; - } else { + } + else { self->bit_size = PyLong_AsSsize_t(bit_size_obj); if (self->bit_size < 0) { if (!PyErr_Occurred()) { @@ -288,6 +292,20 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, } } + if (pack_obj == Py_None) { + self->pack = 0; + } + else { + self->pack = PyLong_AsSsize_t(pack_obj); + if (self->pack < 0) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_ValueError, + "_pack_ cannot be negative"); + } + goto error; + } + } + self->proto = Py_NewRef(proto); self->size = size; self->offset = offset; @@ -313,8 +331,7 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, int PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, Py_ssize_t index, Py_ssize_t *pfield_size, - Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack) + Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign) { if (self == NULL) { return -1; @@ -386,11 +403,9 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, Py_ssize_t index, result = PyCField_FromDesc_msvc( pfield_size, bitsize, pbitofs, psize, poffset, palign, - pack, self, info ); } else { - assert(pack == 0); result = PyCField_FromDesc_gcc( bitsize, pbitofs, psize, poffset, palign, diff --git a/Modules/_ctypes/clinic/cfield.c.h b/Modules/_ctypes/clinic/cfield.c.h index e05e3aac642a92..e3eb0c05d70b08 100644 --- a/Modules/_ctypes/clinic/cfield.c.h +++ b/Modules/_ctypes/clinic/cfield.c.h @@ -12,7 +12,7 @@ preserve static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, Py_ssize_t size, Py_ssize_t offset, PyObject *bit_size_obj, - int swapped_bytes, int _ms); + int swapped_bytes, int _ms, PyObject *pack_obj); static PyObject * PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -20,14 +20,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 7 + #define NUM_KEYWORDS 8 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(bit_size), &_Py_ID(swapped_bytes), &_Py_ID(_ms), }, + .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(bit_size), &_Py_ID(swapped_bytes), &_Py_ID(_ms), &_Py_ID(pack), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -36,14 +36,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "type", "size", "offset", "bit_size", "swapped_bytes", "_ms", NULL}; + static const char * const _keywords[] = {"name", "type", "size", "offset", "bit_size", "swapped_bytes", "_ms", "pack", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "CField", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[7]; + PyObject *argsbuf[8]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 4; @@ -54,8 +54,9 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *bit_size_obj = Py_None; int swapped_bytes = 0; int _ms = 0; + PyObject *pack_obj = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 4, 7, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 4, 8, 0, argsbuf); if (!fastargs) { goto exit; } @@ -107,14 +108,20 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_pos; } } - _ms = PyObject_IsTrue(fastargs[6]); - if (_ms < 0) { - goto exit; + if (fastargs[6]) { + _ms = PyObject_IsTrue(fastargs[6]); + if (_ms < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } } + pack_obj = fastargs[7]; skip_optional_pos: - return_value = PyCField_new_impl(type, name, proto, size, offset, bit_size_obj, swapped_bytes, _ms); + return_value = PyCField_new_impl(type, name, proto, size, offset, bit_size_obj, swapped_bytes, _ms, pack_obj); exit: return return_value; } -/*[clinic end generated code: output=c391d6260d1a8db0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1504fff5f57adb10 input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 786f0873bff36d..df0b44e08f69de 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -221,8 +221,7 @@ extern int PyCField_InitFromDesc(ctypes_state *st, struct CFieldObject* self, Py_ssize_t index, Py_ssize_t *pfield_size, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, - Py_ssize_t *palign, - int pack); + Py_ssize_t *palign); extern PyObject *PyCData_AtAddress(ctypes_state *st, PyObject *type, void *buf); extern PyObject *PyCData_FromBytes(ctypes_state *st, PyObject *type, char *data, Py_ssize_t length); @@ -270,6 +269,7 @@ typedef struct CFieldObject { PyObject *name; /* exact PyUnicode */ bool big_endian; /* boolean */ bool _ms_layout; + Py_ssize_t pack; /* 0 if undefined */ } CFieldObject; /**************************************************************** diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index eea5386f0b8858..6afb75178b02f8 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -486,8 +486,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct corrected alignment */ int res = PyCField_InitFromDesc(st, prop, i, &field_size, &bitofs, - &size, &offset, &align, - pack); + &size, &offset, &align); if (res < 0) { goto error; } @@ -535,8 +534,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct align = 0; int res = PyCField_InitFromDesc(st, prop, i, &field_size, &bitofs, - &size, &offset, &align, - pack); + &size, &offset, &align); if (res < 0) { goto error; } From dc859c34784664d6256494edd00ba8aad5afe92f Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 22 Aug 2024 16:16:30 +0200 Subject: [PATCH 23/94] Stick the index in the struct --- Lib/ctypes/_layout.py | 3 +- Modules/_ctypes/cfield.c | 13 +++++---- Modules/_ctypes/clinic/cfield.c.h | 48 ++++++++++++++++++++----------- Modules/_ctypes/ctypes.h | 2 +- Modules/_ctypes/stgdict.c | 11 +++++-- 5 files changed, 50 insertions(+), 27 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 525e9c59fe37e4..223834d7b70606 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -34,7 +34,7 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): self._pack_ = getattr(cls, '_pack_', None) def __iter__(self): - for field in self.fields: + for i, field in enumerate(self.fields): field = tuple(field) try: name, ftype, bit_size = field @@ -51,6 +51,7 @@ def __iter__(self): bit_size=bit_size, swapped_bytes=self.swapped_bytes, pack=self._pack_, + index=i, **self._field_args(), ) self.offset += self.size diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index f50c55945bf7ce..940665a9f64f77 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -210,6 +210,7 @@ _ctypes.CField.__new__ as PyCField_new type as proto: object size: Py_ssize_t offset: Py_ssize_t + index: Py_ssize_t bit_size as bit_size_obj: object = None swapped_bytes: bool = False _ms: bool = False @@ -219,9 +220,10 @@ _ctypes.CField.__new__ as PyCField_new static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, - Py_ssize_t size, Py_ssize_t offset, PyObject *bit_size_obj, - int swapped_bytes, int _ms, PyObject *pack_obj) -/*[clinic end generated code: output=96c276e7033f8fe2 input=6f5e15c4b2015849]*/ + Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, + PyObject *bit_size_obj, int swapped_bytes, int _ms, + PyObject *pack_obj) +/*[clinic end generated code: output=ceac8ecaf7724c6f input=54d25dff55e27e4a]*/ { PyTypeObject *tp = type; ctypes_state *st = get_module_state_by_class(tp); @@ -316,10 +318,10 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, self->big_endian = PY_BIG_ENDIAN; } self->_ms_layout = _ms; + self->index = index; self->setfunc = NULL; // XXX self->getfunc = NULL; // XXX - self->index = 0; // XXX return (PyObject *)self; error: @@ -329,7 +331,7 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, int -PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, Py_ssize_t index, +PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, Py_ssize_t *pfield_size, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign) { @@ -385,7 +387,6 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, Py_ssize_t index, self->setfunc = setfunc; self->getfunc = getfunc; - self->index = index; Py_ssize_t bitsize = self->bit_size; if(!_cfield_is_bitfield(self)) { diff --git a/Modules/_ctypes/clinic/cfield.c.h b/Modules/_ctypes/clinic/cfield.c.h index e3eb0c05d70b08..09bffae2e1f666 100644 --- a/Modules/_ctypes/clinic/cfield.c.h +++ b/Modules/_ctypes/clinic/cfield.c.h @@ -11,8 +11,9 @@ preserve static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, - Py_ssize_t size, Py_ssize_t offset, PyObject *bit_size_obj, - int swapped_bytes, int _ms, PyObject *pack_obj); + Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, + PyObject *bit_size_obj, int swapped_bytes, int _ms, + PyObject *pack_obj); static PyObject * PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -20,14 +21,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 8 + #define NUM_KEYWORDS 9 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(bit_size), &_Py_ID(swapped_bytes), &_Py_ID(_ms), &_Py_ID(pack), }, + .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(index), &_Py_ID(bit_size), &_Py_ID(swapped_bytes), &_Py_ID(_ms), &_Py_ID(pack), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -36,27 +37,28 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "type", "size", "offset", "bit_size", "swapped_bytes", "_ms", "pack", NULL}; + static const char * const _keywords[] = {"name", "type", "size", "offset", "index", "bit_size", "swapped_bytes", "_ms", "pack", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "CField", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[8]; + PyObject *argsbuf[9]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); - Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 4; + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 5; PyObject *name; PyObject *proto; Py_ssize_t size; Py_ssize_t offset; + Py_ssize_t index; PyObject *bit_size_obj = Py_None; int swapped_bytes = 0; int _ms = 0; PyObject *pack_obj = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 4, 8, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 5, 9, 0, argsbuf); if (!fastargs) { goto exit; } @@ -90,17 +92,29 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } offset = ival; } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(fastargs[4]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + index = ival; + } if (!noptargs) { goto skip_optional_pos; } - if (fastargs[4]) { - bit_size_obj = fastargs[4]; + if (fastargs[5]) { + bit_size_obj = fastargs[5]; if (!--noptargs) { goto skip_optional_pos; } } - if (fastargs[5]) { - swapped_bytes = PyObject_IsTrue(fastargs[5]); + if (fastargs[6]) { + swapped_bytes = PyObject_IsTrue(fastargs[6]); if (swapped_bytes < 0) { goto exit; } @@ -108,8 +122,8 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_pos; } } - if (fastargs[6]) { - _ms = PyObject_IsTrue(fastargs[6]); + if (fastargs[7]) { + _ms = PyObject_IsTrue(fastargs[7]); if (_ms < 0) { goto exit; } @@ -117,11 +131,11 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_pos; } } - pack_obj = fastargs[7]; + pack_obj = fastargs[8]; skip_optional_pos: - return_value = PyCField_new_impl(type, name, proto, size, offset, bit_size_obj, swapped_bytes, _ms, pack_obj); + return_value = PyCField_new_impl(type, name, proto, size, offset, index, bit_size_obj, swapped_bytes, _ms, pack_obj); exit: return return_value; } -/*[clinic end generated code: output=1504fff5f57adb10 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9576e54b11a846eb input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index df0b44e08f69de..cdbeddbbbd85a4 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -218,7 +218,7 @@ extern struct fielddesc *_ctypes_get_fielddesc(const char *fmt); struct CFieldObject; extern int -PyCField_InitFromDesc(ctypes_state *st, struct CFieldObject* self, Py_ssize_t index, +PyCField_InitFromDesc(ctypes_state *st, struct CFieldObject* self, Py_ssize_t *pfield_size, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign); diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 6afb75178b02f8..03a84bd0526661 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -453,6 +453,13 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } CFieldObject *prop = (CFieldObject *)prop_obj; // borrow from prop_obj + if (prop->index != i) { + PyErr_Format(PyExc_ValueError, + "field %R index mismatch (expected %d, got %d)", + prop->name, i, prop->index); + goto error; + } + if (PyCArrayTypeObject_Check(st, prop->proto)) { arrays_seen = 1; } @@ -484,7 +491,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct /* construct the field now, as `prop->offset` is `offset` with corrected alignment */ - int res = PyCField_InitFromDesc(st, prop, i, + int res = PyCField_InitFromDesc(st, prop, &field_size, &bitofs, &size, &offset, &align); if (res < 0) { @@ -532,7 +539,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct bitofs = 0; offset = 0; align = 0; - int res = PyCField_InitFromDesc(st, prop, i, + int res = PyCField_InitFromDesc(st, prop, &field_size, &bitofs, &size, &offset, &align); if (res < 0) { From beeab0b0fb45d92539fbadd28fe1b0a52843065e Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 22 Aug 2024 16:26:23 +0200 Subject: [PATCH 24/94] Add packstate struct, with pfield_size --- Modules/_ctypes/cfield.c | 27 ++++++++++++++------------- Modules/_ctypes/ctypes.h | 6 +++++- Modules/_ctypes/stgdict.c | 11 +++++------ 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 940665a9f64f77..63fa9f2ef6538d 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -108,7 +108,7 @@ PyCField_FromDesc manages: */ static int -PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, +PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, CFieldObject* self, StgInfo* info ) @@ -154,7 +154,7 @@ PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, static int PyCField_FromDesc_msvc( - Py_ssize_t *pfield_size, Py_ssize_t bitsize, + _CFieldPackState *packstate, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, CFieldObject* self, StgInfo* info @@ -170,31 +170,31 @@ PyCField_FromDesc_msvc( // *pbitofs is generally non-positive, // and 8 * (*poffset) + *pbitofs points just behind // the end of the last field we placed. - if (0 < *pbitofs + bitsize || 8 * info->size != *pfield_size) { + if (0 < *pbitofs + bitsize || 8 * info->size != packstate->field_size) { // Close the previous bitfield (if any). // and start a new bitfield: *poffset = round_up(*poffset, *palign); *poffset += info->size; - *pfield_size = info->size * 8; + packstate->field_size = info->size * 8; // Reminder: 8 * (*poffset) + *pbitofs points to where we would start a // new field. Ie just behind where we placed the last field plus an // allowance for alignment. - *pbitofs = - *pfield_size; + *pbitofs = - packstate->field_size; } - assert(8 * info->size == *pfield_size); + assert(8 * info->size == packstate->field_size); - self->offset = *poffset - (*pfield_size) / 8; + self->offset = *poffset - (packstate->field_size) / 8; if(_cfield_is_bitfield(self)) { - assert(0 <= (*pfield_size + *pbitofs)); - assert((*pfield_size + *pbitofs) < info->size * 8); - self->size = BUILD_SIZE(bitsize, *pfield_size + *pbitofs); + assert(0 <= (packstate->field_size + *pbitofs)); + assert((packstate->field_size + *pbitofs) < info->size * 8); + self->size = BUILD_SIZE(bitsize, packstate->field_size + *pbitofs); } else { self->size = info->size; } - assert(*pfield_size + *pbitofs <= info->size * 8); + assert(packstate->field_size + *pbitofs <= info->size * 8); *pbitofs += bitsize; *psize = *poffset; @@ -332,7 +332,7 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, int PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, - Py_ssize_t *pfield_size, + _CFieldPackState *packstate, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign) { if (self == NULL) { @@ -402,12 +402,13 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, int result; if (self->_ms_layout) { result = PyCField_FromDesc_msvc( - pfield_size, bitsize, pbitofs, + packstate, bitsize, pbitofs, psize, poffset, palign, self, info ); } else { result = PyCField_FromDesc_gcc( + packstate, bitsize, pbitofs, psize, poffset, palign, self, info diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index cdbeddbbbd85a4..3cb001180c9441 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -217,9 +217,10 @@ extern int PyObject_stginfo(PyObject *self, Py_ssize_t *psize, Py_ssize_t *palig extern struct fielddesc *_ctypes_get_fielddesc(const char *fmt); struct CFieldObject; +struct _CFieldPackState; extern int PyCField_InitFromDesc(ctypes_state *st, struct CFieldObject* self, - Py_ssize_t *pfield_size, + struct _CFieldPackState *pack_state, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign); @@ -271,6 +272,9 @@ typedef struct CFieldObject { bool _ms_layout; Py_ssize_t pack; /* 0 if undefined */ } CFieldObject; +typedef struct _CFieldPackState { + Py_ssize_t field_size; +} _CFieldPackState; // TODO: remove this... /**************************************************************** StgInfo diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 03a84bd0526661..f551d883db9539 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -242,7 +242,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct { Py_ssize_t len, offset, size, align, i; Py_ssize_t union_size, total_align, aligned_size; - Py_ssize_t field_size = 0; + _CFieldPackState packstate = {0}; Py_ssize_t bitofs = 0; PyObject *tmp; int pack; @@ -484,15 +484,14 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct Py_ssize_t last_size = size; Py_ssize_t padding; - if (fieldname == NULL) - { + if (fieldname == NULL) { goto error; } /* construct the field now, as `prop->offset` is `offset` with corrected alignment */ int res = PyCField_InitFromDesc(st, prop, - &field_size, &bitofs, + &packstate, &bitofs, &size, &offset, &align); if (res < 0) { goto error; @@ -534,13 +533,13 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; } } else /* union */ { - field_size = 0; + packstate.field_size = 0; size = 0; bitofs = 0; offset = 0; align = 0; int res = PyCField_InitFromDesc(st, prop, - &field_size, &bitofs, + &packstate, &bitofs, &size, &offset, &align); if (res < 0) { goto error; From accb589cc48988ba1dc7655b083496b36949f076 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 22 Aug 2024 16:31:45 +0200 Subject: [PATCH 25/94] Put pbitoffs in the state --- Modules/_ctypes/cfield.c | 48 +++++++++++++++++++-------------------- Modules/_ctypes/ctypes.h | 5 +++- Modules/_ctypes/stgdict.c | 7 +++--- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 63fa9f2ef6538d..a2d9ea7eca7e57 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -102,19 +102,17 @@ to indicate a bitfield. Here, non-bitfields need bitsize set to size*8. PyCField_FromDesc manages: - *psize: the size of the structure / union so far. -- *poffset, *pbitofs: 8* (*poffset) + *pbitofs points to where the next field - would start. - *palign: the alignment requirements of the last field we placed. */ static int -PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize, Py_ssize_t *pbitofs, +PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, CFieldObject* self, StgInfo* info ) { // We don't use poffset here, so clear it, if it has been set. - *pbitofs += *poffset * 8; + packstate->bitofs += *poffset * 8; *poffset = 0; assert(self->pack == 0); // TODO: This shouldn't be a C assertion @@ -126,28 +124,28 @@ PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize, Py_ssize_ // fits within a single object of its specified type. // That is: determine a "slot", sized & aligned for the specified type, // which contains the bitfield's beginning: - Py_ssize_t slot_start_bit = round_down(*pbitofs, 8 * info->align); + Py_ssize_t slot_start_bit = round_down(packstate->bitofs, 8 * info->align); Py_ssize_t slot_end_bit = slot_start_bit + 8 * info->size; // And see if it also contains the bitfield's last bit: - Py_ssize_t field_end_bit = *pbitofs + bitsize; + Py_ssize_t field_end_bit = packstate->bitofs + bitsize; if (field_end_bit > slot_end_bit) { // It doesn't: add padding (bump up to the next alignment boundary) - *pbitofs = round_up(*pbitofs, 8*info->align); + packstate->bitofs = round_up(packstate->bitofs, 8*info->align); } } assert(*poffset == 0); - self->offset = round_down(*pbitofs, 8*info->align) / 8; + self->offset = round_down(packstate->bitofs, 8*info->align) / 8; if(_cfield_is_bitfield(self)) { - Py_ssize_t effective_bitsof = *pbitofs - 8 * self->offset; + Py_ssize_t effective_bitsof = packstate->bitofs - 8 * self->offset; self->size = BUILD_SIZE(bitsize, effective_bitsof); assert(effective_bitsof <= info->size * 8); } else { self->size = info->size; } - *pbitofs += bitsize; - *psize = round_up(*pbitofs, 8) / 8; + packstate->bitofs += bitsize; + *psize = round_up(packstate->bitofs, 8) / 8; return 0; } @@ -155,7 +153,7 @@ PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize, Py_ssize_ static int PyCField_FromDesc_msvc( _CFieldPackState *packstate, Py_ssize_t bitsize, - Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, + Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, CFieldObject* self, StgInfo* info ) @@ -167,10 +165,10 @@ PyCField_FromDesc_msvc( } // *poffset points to end of current bitfield. - // *pbitofs is generally non-positive, - // and 8 * (*poffset) + *pbitofs points just behind + // bitofs is generally non-positive, + // and 8 * (*poffset) + bitofs points just behind // the end of the last field we placed. - if (0 < *pbitofs + bitsize || 8 * info->size != packstate->field_size) { + if (0 < packstate->bitofs + bitsize || 8 * info->size != packstate->field_size) { // Close the previous bitfield (if any). // and start a new bitfield: *poffset = round_up(*poffset, *palign); @@ -178,25 +176,25 @@ PyCField_FromDesc_msvc( *poffset += info->size; packstate->field_size = info->size * 8; - // Reminder: 8 * (*poffset) + *pbitofs points to where we would start a + // Reminder: 8 * (*poffset) + bitofs points to where we would start a // new field. Ie just behind where we placed the last field plus an // allowance for alignment. - *pbitofs = - packstate->field_size; + packstate->bitofs = - packstate->field_size; } assert(8 * info->size == packstate->field_size); self->offset = *poffset - (packstate->field_size) / 8; if(_cfield_is_bitfield(self)) { - assert(0 <= (packstate->field_size + *pbitofs)); - assert((packstate->field_size + *pbitofs) < info->size * 8); - self->size = BUILD_SIZE(bitsize, packstate->field_size + *pbitofs); + assert(0 <= (packstate->field_size + packstate->bitofs)); + assert((packstate->field_size + packstate->bitofs) < info->size * 8); + self->size = BUILD_SIZE(bitsize, packstate->field_size + packstate->bitofs); } else { self->size = info->size; } - assert(packstate->field_size + *pbitofs <= info->size * 8); + assert(packstate->field_size + packstate->bitofs <= info->size * 8); - *pbitofs += bitsize; + packstate->bitofs += bitsize; *psize = *poffset; return 0; @@ -333,7 +331,7 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, int PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, _CFieldPackState *packstate, - Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign) + Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign) { if (self == NULL) { return -1; @@ -402,14 +400,14 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, int result; if (self->_ms_layout) { result = PyCField_FromDesc_msvc( - packstate, bitsize, pbitofs, + packstate, bitsize, psize, poffset, palign, self, info ); } else { result = PyCField_FromDesc_gcc( packstate, - bitsize, pbitofs, + bitsize, psize, poffset, palign, self, info ); diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 3cb001180c9441..faa968e2e76c28 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -221,7 +221,7 @@ struct _CFieldPackState; extern int PyCField_InitFromDesc(ctypes_state *st, struct CFieldObject* self, struct _CFieldPackState *pack_state, - Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, + Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign); extern PyObject *PyCData_AtAddress(ctypes_state *st, PyObject *type, void *buf); @@ -274,6 +274,9 @@ typedef struct CFieldObject { } CFieldObject; typedef struct _CFieldPackState { Py_ssize_t field_size; + + // `8 * offset + bitofs points` to where the next field would start. + Py_ssize_t bitofs; } _CFieldPackState; // TODO: remove this... /**************************************************************** diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index f551d883db9539..db415d2573dddf 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -243,7 +243,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct Py_ssize_t len, offset, size, align, i; Py_ssize_t union_size, total_align, aligned_size; _CFieldPackState packstate = {0}; - Py_ssize_t bitofs = 0; PyObject *tmp; int pack; Py_ssize_t ffi_ofs; @@ -491,7 +490,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct /* construct the field now, as `prop->offset` is `offset` with corrected alignment */ int res = PyCField_InitFromDesc(st, prop, - &packstate, &bitofs, + &packstate, &size, &offset, &align); if (res < 0) { goto error; @@ -535,11 +534,11 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } else /* union */ { packstate.field_size = 0; size = 0; - bitofs = 0; + packstate.bitofs = 0; offset = 0; align = 0; int res = PyCField_InitFromDesc(st, prop, - &packstate, &bitofs, + &packstate, &size, &offset, &align); if (res < 0) { goto error; From 71fbfb8b3aa8d7001b9b99769064817effbc6d37 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 22 Aug 2024 16:35:36 +0200 Subject: [PATCH 26/94] Put the size in the state --- Modules/_ctypes/cfield.c | 15 +++++++-------- Modules/_ctypes/ctypes.h | 4 +++- Modules/_ctypes/stgdict.c | 24 ++++++++++++------------ 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index a2d9ea7eca7e57..505b6f6f80f30b 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -101,13 +101,12 @@ We do not support zero length bitfields. In fact we use bitsize != 0 elsewhere to indicate a bitfield. Here, non-bitfields need bitsize set to size*8. PyCField_FromDesc manages: -- *psize: the size of the structure / union so far. - *palign: the alignment requirements of the last field we placed. */ static int PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize, - Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, + Py_ssize_t *poffset, Py_ssize_t *palign, CFieldObject* self, StgInfo* info ) { @@ -145,7 +144,7 @@ PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize, } packstate->bitofs += bitsize; - *psize = round_up(packstate->bitofs, 8) / 8; + packstate->size = round_up(packstate->bitofs, 8) / 8; return 0; } @@ -153,7 +152,7 @@ PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize, static int PyCField_FromDesc_msvc( _CFieldPackState *packstate, Py_ssize_t bitsize, - Py_ssize_t *psize, Py_ssize_t *poffset, + Py_ssize_t *poffset, Py_ssize_t *palign, CFieldObject* self, StgInfo* info ) @@ -195,7 +194,7 @@ PyCField_FromDesc_msvc( assert(packstate->field_size + packstate->bitofs <= info->size * 8); packstate->bitofs += bitsize; - *psize = *poffset; + packstate->size = *poffset; return 0; } @@ -331,7 +330,7 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, int PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, _CFieldPackState *packstate, - Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign) + Py_ssize_t *poffset, Py_ssize_t *palign) { if (self == NULL) { return -1; @@ -401,14 +400,14 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, if (self->_ms_layout) { result = PyCField_FromDesc_msvc( packstate, bitsize, - psize, poffset, palign, + poffset, palign, self, info ); } else { result = PyCField_FromDesc_gcc( packstate, bitsize, - psize, poffset, palign, + poffset, palign, self, info ); } diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index faa968e2e76c28..5535e09842635e 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -221,7 +221,7 @@ struct _CFieldPackState; extern int PyCField_InitFromDesc(ctypes_state *st, struct CFieldObject* self, struct _CFieldPackState *pack_state, - Py_ssize_t *psize, Py_ssize_t *poffset, + Py_ssize_t *poffset, Py_ssize_t *palign); extern PyObject *PyCData_AtAddress(ctypes_state *st, PyObject *type, void *buf); @@ -277,6 +277,8 @@ typedef struct _CFieldPackState { // `8 * offset + bitofs points` to where the next field would start. Py_ssize_t bitofs; + + Py_ssize_t size; // the size of the structure / union so far } _CFieldPackState; // TODO: remove this... /**************************************************************** diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index db415d2573dddf..5cd5e70f8bbe75 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -240,7 +240,7 @@ _ctypes_alloc_format_padding(const char *prefix, Py_ssize_t padding) int PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct) { - Py_ssize_t len, offset, size, align, i; + Py_ssize_t len, offset, align, i; Py_ssize_t union_size, total_align, aligned_size; _CFieldPackState packstate = {0}; PyObject *tmp; @@ -392,7 +392,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } if (baseinfo) { - size = offset = baseinfo->size; + packstate.size = offset = baseinfo->size; align = baseinfo->align; union_size = 0; total_align = align ? align : 1; @@ -413,7 +413,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct ffi_ofs = baseinfo->length; } else { offset = 0; - size = 0; + packstate.size = 0; align = 0; union_size = 0; total_align = forced_alignment; @@ -480,7 +480,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct char *ptr; Py_ssize_t len; char *buf; - Py_ssize_t last_size = size; + Py_ssize_t last_size = packstate.size; Py_ssize_t padding; if (fieldname == NULL) { @@ -491,7 +491,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct corrected alignment */ int res = PyCField_InitFromDesc(st, prop, &packstate, - &size, &offset, &align); + &offset, &align); if (res < 0) { goto error; } @@ -533,17 +533,17 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } } else /* union */ { packstate.field_size = 0; - size = 0; + packstate.size = 0; packstate.bitofs = 0; offset = 0; align = 0; int res = PyCField_InitFromDesc(st, prop, &packstate, - &size, &offset, &align); + &offset, &align); if (res < 0) { goto error; } - union_size = max(size, union_size); + union_size = max(packstate.size, union_size); } total_align = max(align, total_align); @@ -555,18 +555,18 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct Py_CLEAR(layout_fields); if (!isStruct) { - size = union_size; + packstate.size = union_size; } /* Adjust the size according to the alignment requirements */ - aligned_size = ((size + total_align - 1) / total_align) * total_align; + aligned_size = ((packstate.size + total_align - 1) / total_align) * total_align; if (isStruct) { char *ptr; Py_ssize_t padding; /* Pad up to the full size of the struct */ - padding = aligned_size - size; + padding = aligned_size - packstate.size; if (padding > 0) { ptr = stginfo->format; stginfo->format = _ctypes_alloc_format_padding(ptr, padding); @@ -603,7 +603,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct # define MAX_STRUCT_SIZE 16 #endif - if (arrays_seen && (size <= MAX_STRUCT_SIZE)) { + if (arrays_seen && (packstate.size <= MAX_STRUCT_SIZE)) { /* * See bpo-22273 and gh-110190. Arrays are normally treated as * pointers, which is fine when an array name is being passed as From 9c6e3b0f82510ba17f78f81ed36fe04135a8a883 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 22 Aug 2024 16:40:20 +0200 Subject: [PATCH 27/94] Add offset to the state --- Modules/_ctypes/cfield.c | 33 ++++++++++++++++----------------- Modules/_ctypes/ctypes.h | 4 ++-- Modules/_ctypes/stgdict.c | 12 ++++++------ 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 505b6f6f80f30b..b221be2d143557 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -106,13 +106,13 @@ PyCField_FromDesc manages: static int PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize, - Py_ssize_t *poffset, Py_ssize_t *palign, + Py_ssize_t *palign, CFieldObject* self, StgInfo* info ) { - // We don't use poffset here, so clear it, if it has been set. - packstate->bitofs += *poffset * 8; - *poffset = 0; + // We don't use packstate->offset here, so clear it, if it has been set. + packstate->bitofs += packstate->offset * 8; + packstate->offset = 0; assert(self->pack == 0); // TODO: This shouldn't be a C assertion @@ -132,7 +132,7 @@ PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize, packstate->bitofs = round_up(packstate->bitofs, 8*info->align); } } - assert(*poffset == 0); + assert(packstate->offset == 0); self->offset = round_down(packstate->bitofs, 8*info->align) / 8; if(_cfield_is_bitfield(self)) { @@ -152,7 +152,6 @@ PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize, static int PyCField_FromDesc_msvc( _CFieldPackState *packstate, Py_ssize_t bitsize, - Py_ssize_t *poffset, Py_ssize_t *palign, CFieldObject* self, StgInfo* info ) @@ -163,19 +162,19 @@ PyCField_FromDesc_msvc( *palign = info->align; } - // *poffset points to end of current bitfield. - // bitofs is generally non-positive, - // and 8 * (*poffset) + bitofs points just behind + // packstate->offset points to end of current bitfield. + // packstate->bitofs is generally non-positive, + // and 8 * packstate->offset + packstate->bitofs points just behind // the end of the last field we placed. if (0 < packstate->bitofs + bitsize || 8 * info->size != packstate->field_size) { // Close the previous bitfield (if any). // and start a new bitfield: - *poffset = round_up(*poffset, *palign); + packstate->offset = round_up(packstate->offset, *palign); - *poffset += info->size; + packstate->offset += info->size; packstate->field_size = info->size * 8; - // Reminder: 8 * (*poffset) + bitofs points to where we would start a + // Reminder: 8 * (packstate->offset) + bitofs points to where we would start a // new field. Ie just behind where we placed the last field plus an // allowance for alignment. packstate->bitofs = - packstate->field_size; @@ -183,7 +182,7 @@ PyCField_FromDesc_msvc( assert(8 * info->size == packstate->field_size); - self->offset = *poffset - (packstate->field_size) / 8; + self->offset = packstate->offset - (packstate->field_size) / 8; if(_cfield_is_bitfield(self)) { assert(0 <= (packstate->field_size + packstate->bitofs)); assert((packstate->field_size + packstate->bitofs) < info->size * 8); @@ -194,7 +193,7 @@ PyCField_FromDesc_msvc( assert(packstate->field_size + packstate->bitofs <= info->size * 8); packstate->bitofs += bitsize; - packstate->size = *poffset; + packstate->size = packstate->offset; return 0; } @@ -330,7 +329,7 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, int PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, _CFieldPackState *packstate, - Py_ssize_t *poffset, Py_ssize_t *palign) + Py_ssize_t *palign) { if (self == NULL) { return -1; @@ -400,14 +399,14 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, if (self->_ms_layout) { result = PyCField_FromDesc_msvc( packstate, bitsize, - poffset, palign, + palign, self, info ); } else { result = PyCField_FromDesc_gcc( packstate, bitsize, - poffset, palign, + palign, self, info ); } diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 5535e09842635e..7caf17c75ea083 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -221,7 +221,6 @@ struct _CFieldPackState; extern int PyCField_InitFromDesc(ctypes_state *st, struct CFieldObject* self, struct _CFieldPackState *pack_state, - Py_ssize_t *poffset, Py_ssize_t *palign); extern PyObject *PyCData_AtAddress(ctypes_state *st, PyObject *type, void *buf); @@ -275,8 +274,9 @@ typedef struct CFieldObject { typedef struct _CFieldPackState { Py_ssize_t field_size; - // `8 * offset + bitofs points` to where the next field would start. + // `8 * offset + bitofs` points to where the next field would start. Py_ssize_t bitofs; + Py_ssize_t offset; Py_ssize_t size; // the size of the structure / union so far } _CFieldPackState; // TODO: remove this... diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 5cd5e70f8bbe75..a21a8d0fb7b17b 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -240,7 +240,7 @@ _ctypes_alloc_format_padding(const char *prefix, Py_ssize_t padding) int PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct) { - Py_ssize_t len, offset, align, i; + Py_ssize_t len, align, i; Py_ssize_t union_size, total_align, aligned_size; _CFieldPackState packstate = {0}; PyObject *tmp; @@ -392,7 +392,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } if (baseinfo) { - packstate.size = offset = baseinfo->size; + packstate.size = packstate.offset = baseinfo->size; align = baseinfo->align; union_size = 0; total_align = align ? align : 1; @@ -412,7 +412,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } ffi_ofs = baseinfo->length; } else { - offset = 0; + packstate.offset = 0; packstate.size = 0; align = 0; union_size = 0; @@ -491,7 +491,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct corrected alignment */ int res = PyCField_InitFromDesc(st, prop, &packstate, - &offset, &align); + &align); if (res < 0) { goto error; } @@ -535,11 +535,11 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct packstate.field_size = 0; packstate.size = 0; packstate.bitofs = 0; - offset = 0; + packstate.offset = 0; align = 0; int res = PyCField_InitFromDesc(st, prop, &packstate, - &offset, &align); + &align); if (res < 0) { goto error; } From a6f8fdcae806b004703a08b3d3948b1f155b844a Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 22 Aug 2024 16:43:18 +0200 Subject: [PATCH 28/94] Put align in the state --- Modules/_ctypes/cfield.c | 18 ++++++------------ Modules/_ctypes/ctypes.h | 4 ++-- Modules/_ctypes/stgdict.c | 20 ++++++++------------ 3 files changed, 16 insertions(+), 26 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index b221be2d143557..0ccdd1d2fe7f67 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -100,13 +100,11 @@ See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html#index-mms-bitfields for We do not support zero length bitfields. In fact we use bitsize != 0 elsewhere to indicate a bitfield. Here, non-bitfields need bitsize set to size*8. -PyCField_FromDesc manages: -- *palign: the alignment requirements of the last field we placed. +PyCField_FromDesc manages the pack state struct. */ static int PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize, - Py_ssize_t *palign, CFieldObject* self, StgInfo* info ) { @@ -116,7 +114,7 @@ PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize, assert(self->pack == 0); // TODO: This shouldn't be a C assertion - *palign = info->align; + packstate->align = info->align; if (bitsize > 0) { // Determine whether the bit field, if placed at the next free bit, @@ -152,14 +150,13 @@ PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize, static int PyCField_FromDesc_msvc( _CFieldPackState *packstate, Py_ssize_t bitsize, - Py_ssize_t *palign, CFieldObject* self, StgInfo* info ) { if (self->pack) { - *palign = Py_MIN(self->pack, info->align); + packstate->align = Py_MIN(self->pack, info->align); } else { - *palign = info->align; + packstate->align = info->align; } // packstate->offset points to end of current bitfield. @@ -169,7 +166,7 @@ PyCField_FromDesc_msvc( if (0 < packstate->bitofs + bitsize || 8 * info->size != packstate->field_size) { // Close the previous bitfield (if any). // and start a new bitfield: - packstate->offset = round_up(packstate->offset, *palign); + packstate->offset = round_up(packstate->offset, packstate->align); packstate->offset += info->size; @@ -328,8 +325,7 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, int PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, - _CFieldPackState *packstate, - Py_ssize_t *palign) + _CFieldPackState *packstate) { if (self == NULL) { return -1; @@ -399,14 +395,12 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, if (self->_ms_layout) { result = PyCField_FromDesc_msvc( packstate, bitsize, - palign, self, info ); } else { result = PyCField_FromDesc_gcc( packstate, bitsize, - palign, self, info ); } diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 7caf17c75ea083..77c1fddc6dc691 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -220,8 +220,7 @@ struct CFieldObject; struct _CFieldPackState; extern int PyCField_InitFromDesc(ctypes_state *st, struct CFieldObject* self, - struct _CFieldPackState *pack_state, - Py_ssize_t *palign); + struct _CFieldPackState *pack_state); extern PyObject *PyCData_AtAddress(ctypes_state *st, PyObject *type, void *buf); extern PyObject *PyCData_FromBytes(ctypes_state *st, PyObject *type, char *data, Py_ssize_t length); @@ -279,6 +278,7 @@ typedef struct _CFieldPackState { Py_ssize_t offset; Py_ssize_t size; // the size of the structure / union so far + Py_ssize_t align; // the alignment requirements of the last field placed } _CFieldPackState; // TODO: remove this... /**************************************************************** diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index a21a8d0fb7b17b..2a8d40a6258c39 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -240,7 +240,7 @@ _ctypes_alloc_format_padding(const char *prefix, Py_ssize_t padding) int PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct) { - Py_ssize_t len, align, i; + Py_ssize_t len, i; Py_ssize_t union_size, total_align, aligned_size; _CFieldPackState packstate = {0}; PyObject *tmp; @@ -393,9 +393,9 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (baseinfo) { packstate.size = packstate.offset = baseinfo->size; - align = baseinfo->align; + packstate.align = baseinfo->align; union_size = 0; - total_align = align ? align : 1; + total_align = packstate.align ? packstate.align : 1; total_align = max(total_align, forced_alignment); stginfo->ffi_type_pointer.type = FFI_TYPE_STRUCT; stginfo->ffi_type_pointer.elements = PyMem_New(ffi_type *, baseinfo->length + len + 1); @@ -414,7 +414,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } else { packstate.offset = 0; packstate.size = 0; - align = 0; + packstate.align = 0; union_size = 0; total_align = forced_alignment; stginfo->ffi_type_pointer.type = FFI_TYPE_STRUCT; @@ -489,9 +489,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct /* construct the field now, as `prop->offset` is `offset` with corrected alignment */ - int res = PyCField_InitFromDesc(st, prop, - &packstate, - &align); + int res = PyCField_InitFromDesc(st, prop, &packstate); if (res < 0) { goto error; } @@ -536,16 +534,14 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct packstate.size = 0; packstate.bitofs = 0; packstate.offset = 0; - align = 0; - int res = PyCField_InitFromDesc(st, prop, - &packstate, - &align); + packstate.align = 0; + int res = PyCField_InitFromDesc(st, prop, &packstate); if (res < 0) { goto error; } union_size = max(packstate.size, union_size); } - total_align = max(align, total_align); + total_align = max(packstate.align, total_align); if (-1 == PyObject_SetAttr(type, prop->name, prop_obj)) { goto error; From 6803815c2db66c5541e60c7630bfdd528846dfae Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 22 Aug 2024 16:54:54 +0200 Subject: [PATCH 29/94] Move getfunc/setfunc to __new__ --- Modules/_ctypes/cfield.c | 74 ++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 41 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 0ccdd1d2fe7f67..be482d2d743322 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -313,72 +313,64 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, self->_ms_layout = _ms; self->index = index; - self->setfunc = NULL; // XXX - self->getfunc = NULL; // XXX - - return (PyObject *)self; -error: - Py_DECREF(self); - return NULL; -} - - -int -PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, - _CFieldPackState *packstate) -{ - if (self == NULL) { - return -1; - } - assert(self->proto); - StgInfo *info; - if (PyStgInfo_FromType(st, self->proto, &info) < 0) { - return -1; - } - if (!info) { - PyErr_SetString(PyExc_TypeError, - "has no _stginfo_"); - return -1; - } - - PyObject* proto = self->proto; - /* Field descriptors for 'c_char * n' are be scpecial cased to return a Python string instead of an Array object instance... */ - SETFUNC setfunc = NULL; - GETFUNC getfunc = NULL; + self->setfunc = NULL; + self->getfunc = NULL; if (PyCArrayTypeObject_Check(st, proto)) { StgInfo *ainfo; if (PyStgInfo_FromType(st, proto, &ainfo) < 0) { - return -1; + goto error; } if (ainfo && ainfo->proto) { StgInfo *iinfo; if (PyStgInfo_FromType(st, ainfo->proto, &iinfo) < 0) { - return -1; + goto error; } if (!iinfo) { PyErr_SetString(PyExc_TypeError, "has no _stginfo_"); - return -1; + goto error; } if (iinfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { struct fielddesc *fd = _ctypes_get_fielddesc("s"); - getfunc = fd->getfunc; - setfunc = fd->setfunc; + self->getfunc = fd->getfunc; + self->setfunc = fd->setfunc; } if (iinfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { struct fielddesc *fd = _ctypes_get_fielddesc("U"); - getfunc = fd->getfunc; - setfunc = fd->setfunc; + self->getfunc = fd->getfunc; + self->setfunc = fd->setfunc; } } } - self->setfunc = setfunc; - self->getfunc = getfunc; + return (PyObject *)self; +error: + Py_DECREF(self); + return NULL; +} + + +int +PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, + _CFieldPackState *packstate) +{ + if (self == NULL) { + return -1; + } + assert(self->proto); + StgInfo *info; + if (PyStgInfo_FromType(st, self->proto, &info) < 0) { + return -1; + } + if (!info) { + PyErr_SetString(PyExc_TypeError, + "has no _stginfo_"); + return -1; + } Py_ssize_t bitsize = self->bit_size; if(!_cfield_is_bitfield(self)) { From e0fc0afaa3d6d731d63014dc953ce02b21343a84 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 16:14:43 +0200 Subject: [PATCH 30/94] Start verifying the internal pack state --- Lib/ctypes/_layout.py | 14 ++++++++++ Modules/_ctypes/cfield.c | 46 +++++++++++++++++++++++++++++-- Modules/_ctypes/clinic/cfield.c.h | 25 +++++++++++------ Modules/_ctypes/ctypes.h | 22 ++++++++------- 4 files changed, 86 insertions(+), 21 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 223834d7b70606..9c0abdccb0c0d4 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -1,5 +1,6 @@ import sys import warnings +import struct from _ctypes import CField import ctypes @@ -43,6 +44,18 @@ def __iter__(self): bit_size = None size = ctypes.sizeof(ftype) offset = self.offset + + ################################## State check (remove this) + if isinstance(self, WindowsLayout): + state_field_size = size * 8 + else: + state_field_size = 0 + state_to_check = struct.pack( + "nnnnn", + state_field_size, -1, -1, -1, -1 + ) + ################################## + yield CField( name=name, type=ftype, @@ -52,6 +65,7 @@ def __iter__(self): swapped_bytes=self.swapped_bytes, pack=self._pack_, index=i, + state_to_check=state_to_check, **self._field_args(), ) self.offset += self.size diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index be482d2d743322..22cda9435a83d9 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -208,6 +208,7 @@ _ctypes.CField.__new__ as PyCField_new swapped_bytes: bool = False _ms: bool = False pack as pack_obj: object = None + state_to_check: object = NULL [clinic start generated code]*/ @@ -215,8 +216,8 @@ static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, PyObject *bit_size_obj, int swapped_bytes, int _ms, - PyObject *pack_obj) -/*[clinic end generated code: output=ceac8ecaf7724c6f input=54d25dff55e27e4a]*/ + PyObject *pack_obj, PyObject *state_to_check) +/*[clinic end generated code: output=cacdc95aba80f787 input=55968097dd1d41f3]*/ { PyTypeObject *tp = type; ctypes_state *st = get_module_state_by_class(tp); @@ -347,6 +348,23 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, } } + if (state_to_check) { + char *buf; + Py_ssize_t len; + if (PyBytes_AsStringAndSize(state_to_check, &buf, &len) < 0) { + goto error; + } + if (len != sizeof(_CFieldPackState)) { + PyErr_Format(PyExc_ValueError, + "state_to_check size invalid, want %zd, got %zd", + sizeof(_CFieldPackState), len); + } + memcpy(&self->state_to_check, buf, sizeof(_CFieldPackState)); + } + else { + memset(&self->state_to_check, 0xff, sizeof(_CFieldPackState)); + } + return (PyObject *)self; error: Py_DECREF(self); @@ -403,6 +421,30 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, if(self->big_endian && _cfield_is_bitfield(self)) { self->size = BUILD_SIZE(NUM_BITS(self->size), 8*info->size - LOW_BIT(self->size) - bitsize); } + +#define CHECK_FIELD(FIELD, CAN_SKIP) \ + { \ + Py_ssize_t got = self->state_to_check.FIELD; \ + if (got >= 0 || !CAN_SKIP) { \ + Py_ssize_t expected = packstate->FIELD; \ + if (got != expected) { \ + PyErr_Format( \ + PyExc_AssertionError, \ + "state_to_check." #FIELD \ + " invalid, want %zd, got %zd", \ + expected, got); \ + return -1; \ + } \ + } \ + } + + CHECK_FIELD(field_size, false); + CHECK_FIELD(bitofs, true); + CHECK_FIELD(offset, true); + CHECK_FIELD(size, true); + CHECK_FIELD(align, true); +#undef CHECK_FIELD + return 0; } diff --git a/Modules/_ctypes/clinic/cfield.c.h b/Modules/_ctypes/clinic/cfield.c.h index 09bffae2e1f666..8a0fdf32639066 100644 --- a/Modules/_ctypes/clinic/cfield.c.h +++ b/Modules/_ctypes/clinic/cfield.c.h @@ -13,7 +13,7 @@ static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, PyObject *bit_size_obj, int swapped_bytes, int _ms, - PyObject *pack_obj); + PyObject *pack_obj, PyObject *state_to_check); static PyObject * PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -21,14 +21,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 9 + #define NUM_KEYWORDS 10 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(index), &_Py_ID(bit_size), &_Py_ID(swapped_bytes), &_Py_ID(_ms), &_Py_ID(pack), }, + .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(index), &_Py_ID(bit_size), &_Py_ID(swapped_bytes), &_Py_ID(_ms), &_Py_ID(pack), &_Py_ID(state_to_check), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -37,14 +37,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "type", "size", "offset", "index", "bit_size", "swapped_bytes", "_ms", "pack", NULL}; + static const char * const _keywords[] = {"name", "type", "size", "offset", "index", "bit_size", "swapped_bytes", "_ms", "pack", "state_to_check", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "CField", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[9]; + PyObject *argsbuf[10]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 5; @@ -57,8 +57,9 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int swapped_bytes = 0; int _ms = 0; PyObject *pack_obj = Py_None; + PyObject *state_to_check = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 5, 9, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 5, 10, 0, argsbuf); if (!fastargs) { goto exit; } @@ -131,11 +132,17 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_pos; } } - pack_obj = fastargs[8]; + if (fastargs[8]) { + pack_obj = fastargs[8]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + state_to_check = fastargs[9]; skip_optional_pos: - return_value = PyCField_new_impl(type, name, proto, size, offset, index, bit_size_obj, swapped_bytes, _ms, pack_obj); + return_value = PyCField_new_impl(type, name, proto, size, offset, index, bit_size_obj, swapped_bytes, _ms, pack_obj, state_to_check); exit: return return_value; } -/*[clinic end generated code: output=9576e54b11a846eb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=acb9bbd196bf5e44 input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 77c1fddc6dc691..19a3bd9e32361f 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -253,6 +253,16 @@ struct fielddesc { GETFUNC getfunc_swapped; }; +typedef struct _CFieldPackState { + Py_ssize_t field_size; + + // `8 * offset + bitofs` points to where the next field would start. + Py_ssize_t bitofs; + Py_ssize_t offset; + + Py_ssize_t size; // the size of the structure / union so far + Py_ssize_t align; // the alignment requirements of the last field placed +} _CFieldPackState; // TODO: remove this... typedef struct CFieldObject { PyObject_HEAD Py_ssize_t offset; @@ -269,17 +279,9 @@ typedef struct CFieldObject { bool big_endian; /* boolean */ bool _ms_layout; Py_ssize_t pack; /* 0 if undefined */ -} CFieldObject; -typedef struct _CFieldPackState { - Py_ssize_t field_size; - - // `8 * offset + bitofs` points to where the next field would start. - Py_ssize_t bitofs; - Py_ssize_t offset; - Py_ssize_t size; // the size of the structure / union so far - Py_ssize_t align; // the alignment requirements of the last field placed -} _CFieldPackState; // TODO: remove this... + _CFieldPackState state_to_check; // TODO: remove this. +} CFieldObject; /**************************************************************** StgInfo From a6814b7936ecaf8a5e570a8788f6689a5b3620fc Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 16:27:46 +0200 Subject: [PATCH 31/94] Put offset & align in for the simple cases --- Lib/ctypes/_layout.py | 32 ++++++++++++++++++++++++++++---- Modules/_ctypes/cfield.c | 8 ++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 9c0abdccb0c0d4..e70e48001df83b 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -13,11 +13,13 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): self.fields = fields self.is_struct = is_struct + self.size = 0 + self.offset = 0 if base: - self.size = self.offset = ctypes.sizeof(base) + if is_struct: + self.size = self.offset = ctypes.sizeof(base) base_align = ctypes.alignment(base) else: - self.size = 0 self.offset = 0 base_align = 1 @@ -35,6 +37,7 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): self._pack_ = getattr(cls, '_pack_', None) def __iter__(self): + self.gave_up = False for i, field in enumerate(self.fields): field = tuple(field) try: @@ -45,14 +48,34 @@ def __iter__(self): size = ctypes.sizeof(ftype) offset = self.offset + if bit_size: + self.gave_up = True + if size != ctypes.alignment(ftype): + self.gave_up = True + if self._pack_: + self.gave_up = True + ################################## State check (remove this) if isinstance(self, WindowsLayout): state_field_size = size * 8 else: state_field_size = 0 + state_bitofs = -1 + state_offset = -1 + state_size = -1 + state_align = -1 + + if not self.gave_up: + if isinstance(self, GCCSysVLayout): + # We don't use packstate->offset here, so clear it, if it has been set. + state_offset = 0 + else: + state_offset = self.offset + state_align = ctypes.alignment(ftype) + state_to_check = struct.pack( "nnnnn", - state_field_size, -1, -1, -1, -1 + state_field_size, state_bitofs, state_offset, state_size, state_align ) ################################## @@ -68,7 +91,8 @@ def __iter__(self): state_to_check=state_to_check, **self._field_args(), ) - self.offset += self.size + if self.is_struct: + self.offset += self.size def _field_args(self): return {} diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 22cda9435a83d9..77fa7f861e5645 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -356,8 +356,8 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, } if (len != sizeof(_CFieldPackState)) { PyErr_Format(PyExc_ValueError, - "state_to_check size invalid, want %zd, got %zd", - sizeof(_CFieldPackState), len); + "state_to_check size invalid for %R, want %zd, got %zd", + self->name, sizeof(_CFieldPackState), len); } memcpy(&self->state_to_check, buf, sizeof(_CFieldPackState)); } @@ -431,8 +431,8 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, PyErr_Format( \ PyExc_AssertionError, \ "state_to_check." #FIELD \ - " invalid, want %zd, got %zd", \ - expected, got); \ + " invalid for %R, want %zd, got %zd", \ + self->name, expected, got); \ return -1; \ } \ } \ From 377172199c727bd31b6da4c591be1671db0d7686 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 17:16:50 +0200 Subject: [PATCH 32/94] Remove unnecessary condition --- Modules/_ctypes/cfield.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 77fa7f861e5645..4f860c818259b5 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -116,20 +116,19 @@ PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize, packstate->align = info->align; - if (bitsize > 0) { - // Determine whether the bit field, if placed at the next free bit, - // fits within a single object of its specified type. - // That is: determine a "slot", sized & aligned for the specified type, - // which contains the bitfield's beginning: - Py_ssize_t slot_start_bit = round_down(packstate->bitofs, 8 * info->align); - Py_ssize_t slot_end_bit = slot_start_bit + 8 * info->size; - // And see if it also contains the bitfield's last bit: - Py_ssize_t field_end_bit = packstate->bitofs + bitsize; - if (field_end_bit > slot_end_bit) { - // It doesn't: add padding (bump up to the next alignment boundary) - packstate->bitofs = round_up(packstate->bitofs, 8*info->align); - } + // Determine whether the bit field, if placed at the next free bit, + // fits within a single object of its specified type. + // That is: determine a "slot", sized & aligned for the specified type, + // which contains the bitfield's beginning: + Py_ssize_t slot_start_bit = round_down(packstate->bitofs, 8 * info->align); + Py_ssize_t slot_end_bit = slot_start_bit + 8 * info->size; + // And see if it also contains the bitfield's last bit: + Py_ssize_t field_end_bit = packstate->bitofs + bitsize; + if (field_end_bit > slot_end_bit) { + // It doesn't: add padding (bump up to the next alignment boundary) + packstate->bitofs = round_up(packstate->bitofs, 8*info->align); } + assert(packstate->offset == 0); self->offset = round_down(packstate->bitofs, 8*info->align) / 8; From ca4d40f1ad226f20a9b9451066a2e9b08bd4e806 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 17:20:22 +0200 Subject: [PATCH 33/94] Move bitsize calculation to the algorithm --- Modules/_ctypes/cfield.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 4f860c818259b5..3daa5e270315d9 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -104,10 +104,13 @@ PyCField_FromDesc manages the pack state struct. */ static int -PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize, +PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize_in, CFieldObject* self, StgInfo* info ) { + Py_ssize_t bitsize = _cfield_is_bitfield(self) ? self->bit_size : 8 * info->size; + assert(bitsize == bitsize_in); + // We don't use packstate->offset here, so clear it, if it has been set. packstate->bitofs += packstate->offset * 8; packstate->offset = 0; @@ -148,10 +151,13 @@ PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize, static int PyCField_FromDesc_msvc( - _CFieldPackState *packstate, Py_ssize_t bitsize, + _CFieldPackState *packstate, Py_ssize_t bitsize_in, CFieldObject* self, StgInfo* info ) { + Py_ssize_t bitsize = _cfield_is_bitfield(self) ? self->bit_size : 8 * info->size; + assert(bitsize == bitsize_in); + if (self->pack) { packstate->align = Py_MIN(self->pack, info->align); } else { From 3e3eec8a7d9d95e36dcc9255753159295835d66e Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 17:21:28 +0200 Subject: [PATCH 34/94] finish --- Modules/_ctypes/cfield.c | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 3daa5e270315d9..6a6fa37a431a91 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -104,12 +104,11 @@ PyCField_FromDesc manages the pack state struct. */ static int -PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize_in, +PyCField_FromDesc_gcc(_CFieldPackState *packstate, CFieldObject* self, StgInfo* info ) { Py_ssize_t bitsize = _cfield_is_bitfield(self) ? self->bit_size : 8 * info->size; - assert(bitsize == bitsize_in); // We don't use packstate->offset here, so clear it, if it has been set. packstate->bitofs += packstate->offset * 8; @@ -151,12 +150,11 @@ PyCField_FromDesc_gcc(_CFieldPackState *packstate, Py_ssize_t bitsize_in, static int PyCField_FromDesc_msvc( - _CFieldPackState *packstate, Py_ssize_t bitsize_in, + _CFieldPackState *packstate, CFieldObject* self, StgInfo* info ) { Py_ssize_t bitsize = _cfield_is_bitfield(self) ? self->bit_size : 8 * info->size; - assert(bitsize == bitsize_in); if (self->pack) { packstate->align = Py_MIN(self->pack, info->align); @@ -408,16 +406,9 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, int result; if (self->_ms_layout) { - result = PyCField_FromDesc_msvc( - packstate, bitsize, - self, info - ); + result = PyCField_FromDesc_msvc(packstate, self, info); } else { - result = PyCField_FromDesc_gcc( - packstate, - bitsize, - self, info - ); + result = PyCField_FromDesc_gcc(packstate, self, info); } if (result < 0) { return -1; From 3853c38e5b8110db2c4c77239b727ac86a3ae475 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 18:22:36 +0200 Subject: [PATCH 35/94] Copy the GCC/SysV algorithm to (ugly) Python code --- Lib/ctypes/_layout.py | 142 ++++++++++++++++++++----- Lib/test/test_ctypes/test_bitfields.py | 10 +- Modules/_ctypes/cfield.c | 13 +++ 3 files changed, 134 insertions(+), 31 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index e70e48001df83b..ccdb5bb022bf66 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -5,6 +5,38 @@ from _ctypes import CField import ctypes +def round_down(n, multiple): + assert(n >= 0); + assert(multiple >= 0); + if (multiple == 0): + return n; + return int(n / multiple) * multiple; + +def round_up(n, multiple): + assert(n >= 0); + assert(multiple >= 0); + if (multiple == 0): + return n; + return int((n + multiple - 1) / multiple) * multiple; + +def LOW_BIT(offset): + return offset & 0xFFFF; + +def NUM_BITS(bitsize): + return bitsize >> 16; + +def BUILD_SIZE(bitsize, offset): + assert(0 <= offset); + assert(offset <= 0xFFFF); + ## We don't support zero length bitfields. + ## And GET_BITFIELD uses NUM_BITS(size)==0, + ## to figure out whether we are handling a bitfield. + assert(0 < bitsize); + result = (bitsize << 16) + offset; + assert(bitsize == NUM_BITS(result)); + assert(offset == LOW_BIT(result)); + return result; + class _BaseLayout: def __init__(self, cls, fields, is_struct, base, **kwargs): if kwargs: @@ -15,13 +47,13 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): self.size = 0 self.offset = 0 + base_align = 1 if base: if is_struct: self.size = self.offset = ctypes.sizeof(base) base_align = ctypes.alignment(base) - else: - self.offset = 0 - base_align = 1 + #print(base, is_struct, self.size) + self.base = base self.align = getattr(cls, '_align_', 1) if self.align < 0: @@ -38,40 +70,96 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): def __iter__(self): self.gave_up = False + if isinstance(self, GCCSysVLayout): + state_field_size = 0 + state_bitofs = 0 + state_offset = 0 + state_size = 0 + state_align = 0 + #print(self.base, self.is_struct, self.size, self.size * 8) + if self.base and self.is_struct: + state_size = state_offset = ctypes.sizeof(self.base) + state_align = ctypes.alignment(self.base) + else: + state_field_size = 0 + state_bitofs = -1 + state_offset = -1 + state_size = -1 + state_align = -1 + + for i, field in enumerate(self.fields): + if not self.is_struct: + if isinstance(self, GCCSysVLayout): + state_field_size = 0 + state_bitofs = 0 + state_offset = 0 + state_size = 0 + state_align = 0 + field = tuple(field) try: - name, ftype, bit_size = field - except ValueError: name, ftype = field - bit_size = None + is_bitfield = False + bit_size = ctypes.sizeof(ftype) + except ValueError: + name, ftype, bit_size = field + is_bitfield = True + if bit_size <= 0: + raise ValueError(f'number of bits invalid for bit field {name!r}') size = ctypes.sizeof(ftype) offset = self.offset - if bit_size: - self.gave_up = True - if size != ctypes.alignment(ftype): - self.gave_up = True - if self._pack_: - self.gave_up = True + info_size = ctypes.sizeof(ftype) + info_align = ctypes.alignment(ftype) + + if isinstance(self, GCCSysVLayout): + # We don't use packstate->offset here, so clear it, if it has been set. + state_bitofs += state_offset * 8; + state_offset = 0 + + if is_bitfield: + bitsize = bit_size + else: + bitsize = 8 * info_size; + + ## We don't use packstate->offset here, so clear it, if it has been set. + state_bitofs += state_offset * 8; + state_offset = 0; + + assert(self._pack_ in (0, None)); ## TODO: This shouldn't be a C assertion + + state_align = info_align; + + ## Determine whether the bit field, if placed at the next free bit, + ## fits within a single object of its specified type. + ## That is: determine a "slot", sized & aligned for the specified type, + ## which contains the bitfield's beginning: + slot_start_bit = round_down(state_bitofs, 8 * info_align); + slot_end_bit = slot_start_bit + 8 * info_size; + ## And see if it also contains the bitfield's last bit: + field_end_bit = state_bitofs + bitsize; + if field_end_bit > slot_end_bit: + ## It doesn't: add padding (bump up to the next alignment boundary) + state_bitofs = round_up(state_bitofs, 8 * info_align); + + assert(state_offset == 0); + + offset = int(round_down(state_bitofs, 8 * info_align) / 8); + if is_bitfield: + effective_bitsof = state_bitofs - 8 * offset; + size = BUILD_SIZE(bitsize, effective_bitsof); + assert(effective_bitsof <= info_size * 8); + else: + size = info_size; + + state_bitofs += bitsize; + state_size = int(round_up(state_bitofs, 8) / 8); + ################################## State check (remove this) if isinstance(self, WindowsLayout): state_field_size = size * 8 - else: - state_field_size = 0 - state_bitofs = -1 - state_offset = -1 - state_size = -1 - state_align = -1 - - if not self.gave_up: - if isinstance(self, GCCSysVLayout): - # We don't use packstate->offset here, so clear it, if it has been set. - state_offset = 0 - else: - state_offset = self.offset - state_align = ctypes.alignment(ftype) state_to_check = struct.pack( "nnnnn", @@ -84,7 +172,7 @@ def __iter__(self): type=ftype, size=size, offset=offset, - bit_size=bit_size, + bit_size=bit_size if is_bitfield else None, swapped_bytes=self.swapped_bytes, pack=self._pack_, index=i, diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index e6509e6bf89e1d..86d6a7021459d1 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -185,9 +185,9 @@ class X(Structure): x.a, x.b = 0, -1 self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) - def fail_fields(self, *fields): + def fail_fields(self, *fields, expected=None): return self.get_except(type(Structure), "X", (), - {"_fields_": fields}) + {"_fields_": fields}, expected=expected) def test_nonint_types(self): # bit fields are not allowed on non-integer types. @@ -220,7 +220,7 @@ def test_single_bitfield_size(self): with self.subTest(c_typ): if sizeof(c_typ) != alignment(c_typ): self.skipTest('assumes size=alignment') - result = self.fail_fields(("a", c_typ, -1)) + result = self.fail_fields(("a", c_typ, -1), expected=ValueError) self.assertEqual(result, (ValueError, "number of bits invalid for bit field 'a'")) @@ -267,10 +267,12 @@ class X(Structure): self.assertEqual(X.b.offset, sizeof(c_short)*1) self.assertEqual(X.c.offset, sizeof(c_short)*2) - def get_except(self, func, *args, **kw): + def get_except(self, func, *args, expected=None, **kw): try: func(*args, **kw) except Exception as detail: + if expected and not isinstance(detail, expected): + raise return detail.__class__, str(detail) def test_mixed_1(self): diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 6a6fa37a431a91..7b529de4158da5 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -418,6 +418,19 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, self->size = BUILD_SIZE(NUM_BITS(self->size), 8*info->size - LOW_BIT(self->size) - bitsize); } +#define CHECK_FIELD(FIELD, CAN_SKIP) \ + { \ + printf("want%4zd -- got%4zd in " #FIELD "\n", packstate->FIELD, self->state_to_check.FIELD); \ + } + if (memcmp(&self->state_to_check, packstate, sizeof(*packstate)) && self->state_to_check.align >= 0) { + CHECK_FIELD(field_size, false); + CHECK_FIELD(bitofs, true); + CHECK_FIELD(offset, true); + CHECK_FIELD(size, true); + CHECK_FIELD(align, true); + } +#undef CHECK_FIELD + #define CHECK_FIELD(FIELD, CAN_SKIP) \ { \ Py_ssize_t got = self->state_to_check.FIELD; \ From a8ad714f3550d869644a905279b814bb2c4a5000 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 18:28:47 +0200 Subject: [PATCH 36/94] Copy the MS algorithm to (ugly) Python code --- Lib/ctypes/_layout.py | 69 +++++++++++++++++++++++++++++----------- Modules/_ctypes/cfield.c | 8 ++--- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index ccdb5bb022bf66..fdd20c5aadc022 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -70,22 +70,15 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): def __iter__(self): self.gave_up = False - if isinstance(self, GCCSysVLayout): - state_field_size = 0 - state_bitofs = 0 - state_offset = 0 - state_size = 0 - state_align = 0 - #print(self.base, self.is_struct, self.size, self.size * 8) - if self.base and self.is_struct: - state_size = state_offset = ctypes.sizeof(self.base) - state_align = ctypes.alignment(self.base) - else: - state_field_size = 0 - state_bitofs = -1 - state_offset = -1 - state_size = -1 - state_align = -1 + state_field_size = 0 + state_bitofs = 0 + state_offset = 0 + state_size = 0 + state_align = 0 + #print(self.base, self.is_struct, self.size, self.size * 8) + if self.base and self.is_struct: + state_size = state_offset = ctypes.sizeof(self.base) + state_align = ctypes.alignment(self.base) for i, field in enumerate(self.fields): @@ -155,12 +148,50 @@ def __iter__(self): state_bitofs += bitsize; state_size = int(round_up(state_bitofs, 8) / 8); + else: + if is_bitfield: + bitsize = bit_size + else: + bitsize = 8 * info_size; + if self._pack_: + state_align = min(self._pack_, info_align); + else: + state_align = info_align; - ################################## State check (remove this) - if isinstance(self, WindowsLayout): - state_field_size = size * 8 + ## packstate->offset points to end of current bitfield. + ## packstate->bitofs is generally non-positive, + ## and 8 * packstate->offset + packstate->bitofs points just behind + ## the end of the last field we placed. + if ((0 < state_bitofs + bitsize) or (8 * info_size != state_field_size)): + ## Close the previous bitfield (if any). + ## and start a new bitfield: + state_offset = round_up(state_offset, state_align); + + state_offset += info_size; + + state_field_size = info_size * 8; + ## Reminder: 8 * (packstate->offset) + bitofs points to where we would start a + ## new field. Ie just behind where we placed the last field plus an + ## allowance for alignment. + state_bitofs = - state_field_size; + + assert(8 * info_size == state_field_size); + offset = state_offset - int((state_field_size) // 8); + if is_bitfield: + assert(0 <= (state_field_size + state_bitofs)); + assert((state_field_size + state_bitofs) < info_size * 8); + size = BUILD_SIZE(bitsize, state_field_size + state_bitofs); + else: + size = info_size; + assert(state_field_size + state_bitofs <= info_size * 8); + + state_bitofs += bitsize; + state_size = state_offset; + + + ################################## State check (remove this) state_to_check = struct.pack( "nnnnn", state_field_size, state_bitofs, state_offset, state_size, state_align diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 7b529de4158da5..a34931fbbd333c 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -448,10 +448,10 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, } CHECK_FIELD(field_size, false); - CHECK_FIELD(bitofs, true); - CHECK_FIELD(offset, true); - CHECK_FIELD(size, true); - CHECK_FIELD(align, true); + CHECK_FIELD(bitofs, false); + CHECK_FIELD(offset, false); + CHECK_FIELD(size, false); + CHECK_FIELD(align, false); #undef CHECK_FIELD return 0; From 5be7edf754115f3af8dbd8a608bf236010f9f730 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 18:34:52 +0200 Subject: [PATCH 37/94] Remove the C code --- Modules/_ctypes/cfield.c | 142 +-------------------------------------- 1 file changed, 2 insertions(+), 140 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index a34931fbbd333c..b9764a400bc984 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -103,101 +103,6 @@ to indicate a bitfield. Here, non-bitfields need bitsize set to size*8. PyCField_FromDesc manages the pack state struct. */ -static int -PyCField_FromDesc_gcc(_CFieldPackState *packstate, - CFieldObject* self, StgInfo* info - ) -{ - Py_ssize_t bitsize = _cfield_is_bitfield(self) ? self->bit_size : 8 * info->size; - - // We don't use packstate->offset here, so clear it, if it has been set. - packstate->bitofs += packstate->offset * 8; - packstate->offset = 0; - - assert(self->pack == 0); // TODO: This shouldn't be a C assertion - - packstate->align = info->align; - - // Determine whether the bit field, if placed at the next free bit, - // fits within a single object of its specified type. - // That is: determine a "slot", sized & aligned for the specified type, - // which contains the bitfield's beginning: - Py_ssize_t slot_start_bit = round_down(packstate->bitofs, 8 * info->align); - Py_ssize_t slot_end_bit = slot_start_bit + 8 * info->size; - // And see if it also contains the bitfield's last bit: - Py_ssize_t field_end_bit = packstate->bitofs + bitsize; - if (field_end_bit > slot_end_bit) { - // It doesn't: add padding (bump up to the next alignment boundary) - packstate->bitofs = round_up(packstate->bitofs, 8*info->align); - } - - assert(packstate->offset == 0); - - self->offset = round_down(packstate->bitofs, 8*info->align) / 8; - if(_cfield_is_bitfield(self)) { - Py_ssize_t effective_bitsof = packstate->bitofs - 8 * self->offset; - self->size = BUILD_SIZE(bitsize, effective_bitsof); - assert(effective_bitsof <= info->size * 8); - } else { - self->size = info->size; - } - - packstate->bitofs += bitsize; - packstate->size = round_up(packstate->bitofs, 8) / 8; - - return 0; -} - -static int -PyCField_FromDesc_msvc( - _CFieldPackState *packstate, - CFieldObject* self, StgInfo* info - ) -{ - Py_ssize_t bitsize = _cfield_is_bitfield(self) ? self->bit_size : 8 * info->size; - - if (self->pack) { - packstate->align = Py_MIN(self->pack, info->align); - } else { - packstate->align = info->align; - } - - // packstate->offset points to end of current bitfield. - // packstate->bitofs is generally non-positive, - // and 8 * packstate->offset + packstate->bitofs points just behind - // the end of the last field we placed. - if (0 < packstate->bitofs + bitsize || 8 * info->size != packstate->field_size) { - // Close the previous bitfield (if any). - // and start a new bitfield: - packstate->offset = round_up(packstate->offset, packstate->align); - - packstate->offset += info->size; - - packstate->field_size = info->size * 8; - // Reminder: 8 * (packstate->offset) + bitofs points to where we would start a - // new field. Ie just behind where we placed the last field plus an - // allowance for alignment. - packstate->bitofs = - packstate->field_size; - } - - assert(8 * info->size == packstate->field_size); - - self->offset = packstate->offset - (packstate->field_size) / 8; - if(_cfield_is_bitfield(self)) { - assert(0 <= (packstate->field_size + packstate->bitofs)); - assert((packstate->field_size + packstate->bitofs) < info->size * 8); - self->size = BUILD_SIZE(bitsize, packstate->field_size + packstate->bitofs); - } else { - self->size = info->size; - } - assert(packstate->field_size + packstate->bitofs <= info->size * 8); - - packstate->bitofs += bitsize; - packstate->size = packstate->offset; - - return 0; -} - /*[clinic input] @classmethod _ctypes.CField.__new__ as PyCField_new @@ -404,56 +309,13 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, } assert(bitsize <= info->size * 8); - int result; - if (self->_ms_layout) { - result = PyCField_FromDesc_msvc(packstate, self, info); - } else { - result = PyCField_FromDesc_gcc(packstate, self, info); - } - if (result < 0) { - return -1; - } + memcpy(packstate, &self->state_to_check, sizeof(_CFieldPackState)); + assert(!_cfield_is_bitfield(self) || (LOW_BIT(self->size) <= self->size * 8)); if(self->big_endian && _cfield_is_bitfield(self)) { self->size = BUILD_SIZE(NUM_BITS(self->size), 8*info->size - LOW_BIT(self->size) - bitsize); } -#define CHECK_FIELD(FIELD, CAN_SKIP) \ - { \ - printf("want%4zd -- got%4zd in " #FIELD "\n", packstate->FIELD, self->state_to_check.FIELD); \ - } - if (memcmp(&self->state_to_check, packstate, sizeof(*packstate)) && self->state_to_check.align >= 0) { - CHECK_FIELD(field_size, false); - CHECK_FIELD(bitofs, true); - CHECK_FIELD(offset, true); - CHECK_FIELD(size, true); - CHECK_FIELD(align, true); - } -#undef CHECK_FIELD - -#define CHECK_FIELD(FIELD, CAN_SKIP) \ - { \ - Py_ssize_t got = self->state_to_check.FIELD; \ - if (got >= 0 || !CAN_SKIP) { \ - Py_ssize_t expected = packstate->FIELD; \ - if (got != expected) { \ - PyErr_Format( \ - PyExc_AssertionError, \ - "state_to_check." #FIELD \ - " invalid for %R, want %zd, got %zd", \ - self->name, expected, got); \ - return -1; \ - } \ - } \ - } - - CHECK_FIELD(field_size, false); - CHECK_FIELD(bitofs, false); - CHECK_FIELD(offset, false); - CHECK_FIELD(size, false); - CHECK_FIELD(align, false); -#undef CHECK_FIELD - return 0; } From 7218287663274218f3f8cfd23ec23e2ae63dbb96 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 18:41:54 +0200 Subject: [PATCH 38/94] Move assertion & big-endian adjustment --- Lib/ctypes/_layout.py | 8 ++++++++ Modules/_ctypes/cfield.c | 5 ----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index fdd20c5aadc022..0ef9113986563a 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -63,6 +63,10 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): self.align == 1 self.swapped_bytes = hasattr(cls, '_swappedbytes_') + if self.swapped_bytes: + self.big_endian = sys.byteorder == 'little' + else: + self.big_endian = sys.byteorder == 'big' self.total_align = max(self.align, base_align) @@ -198,6 +202,10 @@ def __iter__(self): ) ################################## + assert((not is_bitfield) or (LOW_BIT(size) <= size * 8)); + if self.big_endian and is_bitfield: + size = BUILD_SIZE(NUM_BITS(size), 8*info_size - LOW_BIT(size) - bit_size); + yield CField( name=name, type=ftype, diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index b9764a400bc984..a5b6c325c22a33 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -311,11 +311,6 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, memcpy(packstate, &self->state_to_check, sizeof(_CFieldPackState)); - assert(!_cfield_is_bitfield(self) || (LOW_BIT(self->size) <= self->size * 8)); - if(self->big_endian && _cfield_is_bitfield(self)) { - self->size = BUILD_SIZE(NUM_BITS(self->size), 8*info->size - LOW_BIT(self->size) - bitsize); - } - return 0; } From addf44c4cae448255601d05f15c0b2a35dc7b01f Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 18:51:17 +0200 Subject: [PATCH 39/94] Move assertions out --- Modules/_ctypes/cfield.c | 45 ++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index a5b6c325c22a33..2f08d449bfc389 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -127,9 +127,23 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, PyObject *pack_obj, PyObject *state_to_check) /*[clinic end generated code: output=cacdc95aba80f787 input=55968097dd1d41f3]*/ { + CFieldObject* self = NULL; + if (size < 0) { + PyErr_Format(PyExc_ValueError, + "size of field %R must not be negative, got %zd", name, size); + goto error; + } + // assert: no overflow; + if ((unsigned long long int) size + >= (1ULL << (8*sizeof(Py_ssize_t)-1)) / 8) { + PyErr_Format(PyExc_ValueError, + "size of field %R is too big: %zd", name, size); + goto error; + } + PyTypeObject *tp = type; ctypes_state *st = get_module_state_by_class(tp); - CFieldObject* self = (CFieldObject *)tp->tp_alloc(tp, 0); + self = (CFieldObject *)tp->tp_alloc(tp, 0); if (!self) { return NULL; } @@ -165,6 +179,12 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, } goto error; } + if (self->bit_size > size * 8) { + PyErr_Format(PyExc_ValueError, + "number of bits too large for bit field %R", + self->name); + goto error; + } switch(info->ffi_type_pointer.type) { case FFI_TYPE_UINT8: case FFI_TYPE_UINT16: @@ -275,7 +295,7 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, return (PyObject *)self; error: - Py_DECREF(self); + Py_XDECREF(self); return NULL; } @@ -287,27 +307,6 @@ PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, if (self == NULL) { return -1; } - assert(self->proto); - StgInfo *info; - if (PyStgInfo_FromType(st, self->proto, &info) < 0) { - return -1; - } - if (!info) { - PyErr_SetString(PyExc_TypeError, - "has no _stginfo_"); - return -1; - } - - Py_ssize_t bitsize = self->bit_size; - if(!_cfield_is_bitfield(self)) { - assert(info->size >= 0); - // assert: no overflow; - assert((unsigned long long int) info->size - < (1ULL << (8*sizeof(Py_ssize_t)-1)) / 8); - bitsize = 8 * info->size; - // Caution: bitsize might still be 0 now. - } - assert(bitsize <= info->size * 8); memcpy(packstate, &self->state_to_check, sizeof(_CFieldPackState)); From d92831a8a7f37c20392ac43f6f19dd4bf2518cf0 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 18:53:37 +0200 Subject: [PATCH 40/94] Remove PyCField_InitFromDesc --- Modules/_ctypes/cfield.c | 13 ------------- Modules/_ctypes/ctypes.h | 6 ------ Modules/_ctypes/stgdict.c | 17 ++++------------- 3 files changed, 4 insertions(+), 32 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 2f08d449bfc389..1b8affd5627344 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -300,19 +300,6 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, } -int -PyCField_InitFromDesc(ctypes_state *st, CFieldObject* self, - _CFieldPackState *packstate) -{ - if (self == NULL) { - return -1; - } - - memcpy(packstate, &self->state_to_check, sizeof(_CFieldPackState)); - - return 0; -} - static int PyCField_set(CFieldObject *self, PyObject *inst, PyObject *value) { diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 19a3bd9e32361f..8a9c9c55b881a3 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -216,12 +216,6 @@ extern int PyObject_stginfo(PyObject *self, Py_ssize_t *psize, Py_ssize_t *palig extern struct fielddesc *_ctypes_get_fielddesc(const char *fmt); -struct CFieldObject; -struct _CFieldPackState; -extern int -PyCField_InitFromDesc(ctypes_state *st, struct CFieldObject* self, - struct _CFieldPackState *pack_state); - extern PyObject *PyCData_AtAddress(ctypes_state *st, PyObject *type, void *buf); extern PyObject *PyCData_FromBytes(ctypes_state *st, PyObject *type, char *data, Py_ssize_t length); diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 2a8d40a6258c39..1db6d75f10a825 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -489,10 +489,8 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct /* construct the field now, as `prop->offset` is `offset` with corrected alignment */ - int res = PyCField_InitFromDesc(st, prop, &packstate); - if (res < 0) { - goto error; - } + assert(prop); + memcpy(&packstate, &prop->state_to_check, sizeof(_CFieldPackState)); /* number of bytes between the end of the last field and the start of this one */ @@ -530,15 +528,8 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; } } else /* union */ { - packstate.field_size = 0; - packstate.size = 0; - packstate.bitofs = 0; - packstate.offset = 0; - packstate.align = 0; - int res = PyCField_InitFromDesc(st, prop, &packstate); - if (res < 0) { - goto error; - } + assert(prop); + memcpy(&packstate, &prop->state_to_check, sizeof(_CFieldPackState)); union_size = max(packstate.size, union_size); } total_align = max(packstate.align, total_align); From 466d82f2680a745c84704655ec716a2c7cdff019 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 18:55:46 +0200 Subject: [PATCH 41/94] Remove field_size, bitofs, offset from the pack state --- Lib/ctypes/_layout.py | 5 +++-- Modules/_ctypes/ctypes.h | 6 ------ Modules/_ctypes/stgdict.c | 3 +-- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 0ef9113986563a..10d98a30412749 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -75,6 +75,7 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): def __iter__(self): self.gave_up = False state_field_size = 0 + # `8 * offset + bitofs` points to where the next field would start. state_bitofs = 0 state_offset = 0 state_size = 0 @@ -197,8 +198,8 @@ def __iter__(self): ################################## State check (remove this) state_to_check = struct.pack( - "nnnnn", - state_field_size, state_bitofs, state_offset, state_size, state_align + "nn", + state_size, state_align ) ################################## diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 8a9c9c55b881a3..81c059a0bebfd1 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -248,12 +248,6 @@ struct fielddesc { }; typedef struct _CFieldPackState { - Py_ssize_t field_size; - - // `8 * offset + bitofs` points to where the next field would start. - Py_ssize_t bitofs; - Py_ssize_t offset; - Py_ssize_t size; // the size of the structure / union so far Py_ssize_t align; // the alignment requirements of the last field placed } _CFieldPackState; // TODO: remove this... diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 1db6d75f10a825..7ac1bdf25e239e 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -392,7 +392,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } if (baseinfo) { - packstate.size = packstate.offset = baseinfo->size; + packstate.size = baseinfo->size; packstate.align = baseinfo->align; union_size = 0; total_align = packstate.align ? packstate.align : 1; @@ -412,7 +412,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } ffi_ofs = baseinfo->length; } else { - packstate.offset = 0; packstate.size = 0; packstate.align = 0; union_size = 0; From 1af5db38a6e3e3d8e1570ac8c30289a0bd972933 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 19:17:44 +0200 Subject: [PATCH 42/94] Remove pack from C code --- Lib/ctypes/_layout.py | 15 ++++++++++++++- Modules/_ctypes/stgdict.c | 23 ----------------------- 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 10d98a30412749..0409158e5683e7 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -37,6 +37,8 @@ def BUILD_SIZE(bitsize, offset): assert(offset == LOW_BIT(result)); return result; +_INT_MAX = (1 << (ctypes.sizeof(ctypes.c_int) * 8) - 1) - 1 + class _BaseLayout: def __init__(self, cls, fields, is_struct, base, **kwargs): if kwargs: @@ -70,7 +72,18 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): self.total_align = max(self.align, base_align) - self._pack_ = getattr(cls, '_pack_', None) + _pack_ = getattr(cls, '_pack_', None) + if _pack_ is not None: + try: + self._pack_ = int(_pack_) + except (TypeError, ValueError): + raise ValueError("_pack_ must be an integer") + if self._pack_ < 0: + raise ValueError("_pack_ must be a non-negative integer") + if self._pack_ > _INT_MAX: + raise ValueError("_pack_ too big") + else: + self._pack_ = None def __iter__(self): self.gave_up = False diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 7ac1bdf25e239e..9f1fc04a1250cd 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -244,7 +244,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct Py_ssize_t union_size, total_align, aligned_size; _CFieldPackState packstate = {0}; PyObject *tmp; - int pack; Py_ssize_t ffi_ofs; int arrays_seen = 0; @@ -282,28 +281,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; } - if (PyObject_GetOptionalAttr(type, &_Py_ID(_pack_), &tmp) < 0) { - goto error; - } - if (tmp) { - pack = PyLong_AsInt(tmp); - Py_DECREF(tmp); - if (pack < 0) { - if (!PyErr_Occurred() || - PyErr_ExceptionMatches(PyExc_TypeError) || - PyErr_ExceptionMatches(PyExc_OverflowError)) - { - PyErr_SetString(PyExc_ValueError, - "_pack_ must be a non-negative integer"); - } - goto error; - } - } - else { - /* Setting `_pack_ = 0` amounts to using the default alignment */ - pack = 0; - } - PyObject *layout_class; if (PyObject_GetOptionalAttr(type, &_Py_ID(_layout_), &tmp) < 0) { goto error; From df3143d3d8d84a08197d7490745ba1c7cfd383ed Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 19:41:17 +0200 Subject: [PATCH 43/94] Make _layout_ return an object with fields --- Lib/ctypes/_layout.py | 12 +++++------- Modules/_ctypes/stgdict.c | 13 ++++++++++--- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 0409158e5683e7..f2da01df0fcdb1 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -44,7 +44,6 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): if kwargs: warnings.warn(f'Unknown keyword arguments: {list(kwargs.keys())}') self.cls = cls - self.fields = fields self.is_struct = is_struct self.size = 0 @@ -85,8 +84,8 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): else: self._pack_ = None - def __iter__(self): - self.gave_up = False + self.fields = [] + state_field_size = 0 # `8 * offset + bitofs` points to where the next field would start. state_bitofs = 0 @@ -98,8 +97,7 @@ def __iter__(self): state_size = state_offset = ctypes.sizeof(self.base) state_align = ctypes.alignment(self.base) - - for i, field in enumerate(self.fields): + for i, field in enumerate(fields): if not self.is_struct: if isinstance(self, GCCSysVLayout): state_field_size = 0 @@ -220,7 +218,7 @@ def __iter__(self): if self.big_endian and is_bitfield: size = BUILD_SIZE(NUM_BITS(size), 8*info_size - LOW_BIT(size) - bit_size); - yield CField( + self.fields.append(CField( name=name, type=ftype, size=size, @@ -231,7 +229,7 @@ def __iter__(self): index=i, state_to_check=state_to_check, **self._field_args(), - ) + )) if self.is_struct: self.offset += self.size diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 9f1fc04a1250cd..b005b66e376163 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -339,7 +339,12 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; } - layout_fields = PySequence_Fast(layout, "layout must return a sequence"); + PyObject *layout_fields_obj = PyObject_GetAttrString(layout, "fields"); + if (!layout_fields_obj) { + goto error; + } + layout_fields = PySequence_Tuple(layout_fields_obj); + Py_DECREF(layout_fields_obj); Py_DECREF(layout); if (!layout_fields) { goto error; @@ -355,8 +360,10 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } if (len != PySequence_Fast_GET_SIZE(layout_fields)) { - PyErr_SetString(PyExc_ValueError, - "number of '_fields_' must match result of layout... for now."); + PyErr_Format(PyExc_ValueError, + "number of '_fields_' must match result of layout... for now. want %zd, got %zd", + len, PySequence_Fast_GET_SIZE(layout_fields)); + goto error; } if (stginfo->format) { From a22c557c2a693bf2d2f43624cbab2a973fdb149b Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 19:52:09 +0200 Subject: [PATCH 44/94] Avoid self.* assignments --- Lib/ctypes/_layout.py | 65 ++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 41 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index f2da01df0fcdb1..6fab95e70ba07e 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -43,46 +43,30 @@ class _BaseLayout: def __init__(self, cls, fields, is_struct, base, **kwargs): if kwargs: warnings.warn(f'Unknown keyword arguments: {list(kwargs.keys())}') - self.cls = cls - self.is_struct = is_struct - - self.size = 0 - self.offset = 0 - base_align = 1 - if base: - if is_struct: - self.size = self.offset = ctypes.sizeof(base) - base_align = ctypes.alignment(base) - #print(base, is_struct, self.size) - self.base = base - - self.align = getattr(cls, '_align_', 1) - if self.align < 0: + + align = getattr(cls, '_align_', 1) + if align < 0: raise ValueError('_align_ must be a non-negative integer') - elif self.align == 0: + elif align == 0: # Setting `_align_ = 0` amounts to using the default alignment - self.align == 1 + align == 1 - self.swapped_bytes = hasattr(cls, '_swappedbytes_') - if self.swapped_bytes: - self.big_endian = sys.byteorder == 'little' + swapped_bytes = hasattr(cls, '_swappedbytes_') + if swapped_bytes: + big_endian = sys.byteorder == 'little' else: - self.big_endian = sys.byteorder == 'big' - - self.total_align = max(self.align, base_align) + big_endian = sys.byteorder == 'big' _pack_ = getattr(cls, '_pack_', None) if _pack_ is not None: try: - self._pack_ = int(_pack_) + _pack_ = int(_pack_) except (TypeError, ValueError): raise ValueError("_pack_ must be an integer") - if self._pack_ < 0: + if _pack_ < 0: raise ValueError("_pack_ must be a non-negative integer") - if self._pack_ > _INT_MAX: + if _pack_ > _INT_MAX: raise ValueError("_pack_ too big") - else: - self._pack_ = None self.fields = [] @@ -93,12 +77,12 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): state_size = 0 state_align = 0 #print(self.base, self.is_struct, self.size, self.size * 8) - if self.base and self.is_struct: - state_size = state_offset = ctypes.sizeof(self.base) - state_align = ctypes.alignment(self.base) + if base and is_struct: + state_size = state_offset = ctypes.sizeof(base) + state_align = ctypes.alignment(base) for i, field in enumerate(fields): - if not self.is_struct: + if not is_struct: if isinstance(self, GCCSysVLayout): state_field_size = 0 state_bitofs = 0 @@ -117,7 +101,6 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): if bit_size <= 0: raise ValueError(f'number of bits invalid for bit field {name!r}') size = ctypes.sizeof(ftype) - offset = self.offset info_size = ctypes.sizeof(ftype) info_align = ctypes.alignment(ftype) @@ -136,7 +119,7 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): state_bitofs += state_offset * 8; state_offset = 0; - assert(self._pack_ in (0, None)); ## TODO: This shouldn't be a C assertion + assert(_pack_ in (0, None)); ## TODO: This shouldn't be a C assertion state_align = info_align; @@ -170,8 +153,8 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): else: bitsize = 8 * info_size; - if self._pack_: - state_align = min(self._pack_, info_align); + if _pack_: + state_align = min(_pack_, info_align); else: state_align = info_align; @@ -215,7 +198,7 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): ################################## assert((not is_bitfield) or (LOW_BIT(size) <= size * 8)); - if self.big_endian and is_bitfield: + if big_endian and is_bitfield: size = BUILD_SIZE(NUM_BITS(size), 8*info_size - LOW_BIT(size) - bit_size); self.fields.append(CField( @@ -224,14 +207,14 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): size=size, offset=offset, bit_size=bit_size if is_bitfield else None, - swapped_bytes=self.swapped_bytes, - pack=self._pack_, + swapped_bytes=swapped_bytes, + pack=_pack_, index=i, state_to_check=state_to_check, **self._field_args(), )) - if self.is_struct: - self.offset += self.size + + self.align = align def _field_args(self): return {} From 47150d2ef5646581ac4caf9a0228474e185526f7 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 19:58:27 +0200 Subject: [PATCH 45/94] Pass size and align from Python --- Lib/ctypes/_layout.py | 4 +++- Modules/_ctypes/stgdict.c | 44 ++++++++++++++++++++++++++++++++------- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 6fab95e70ba07e..8ec8ccd6707997 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -214,7 +214,9 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): **self._field_args(), )) - self.align = align + self.forced_align = align + self.size = state_size + self.align = state_align def _field_args(self): return {} diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index b005b66e376163..e8846472bcb2b1 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -241,7 +241,7 @@ int PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct) { Py_ssize_t len, i; - Py_ssize_t union_size, total_align, aligned_size; + Py_ssize_t union_size, aligned_size; _CFieldPackState packstate = {0}; PyObject *tmp; Py_ssize_t ffi_ofs; @@ -251,6 +251,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct // They're cleared on error. PyObject *prop_obj = NULL; PyObject *layout_fields = NULL; + PyObject *layout = NULL; if (fields == NULL) { return 0; @@ -304,7 +305,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; } PyObject *base_arg = Py_NewRef(baseinfo ? base : Py_None); - PyObject *layout = PyObject_Vectorcall( + layout = PyObject_Vectorcall( layout_class, 1 + (PyObject*[]){ NULL, @@ -323,9 +324,9 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (!layout) { goto error; } - tmp = PyObject_GetAttr(layout, &_Py_ID(align)); + + tmp = PyObject_GetAttrString(layout, "forced_align"); if (!tmp) { - Py_DECREF(layout); goto error; } int forced_alignment = PyLong_AsInt(tmp); @@ -335,7 +336,34 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct PyErr_SetString(PyExc_ValueError, "_align_ must be a non-negative integer"); } - Py_DECREF(layout); + goto error; + } + + tmp = PyObject_GetAttrString(layout, "align"); + if (!tmp) { + goto error; + } + Py_ssize_t total_align = PyLong_AsInt(tmp); + Py_DECREF(tmp); + if (total_align < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "total_align must be a non-negative integer"); + } + goto error; + } + + tmp = PyObject_GetAttrString(layout, "size"); + if (!tmp) { + goto error; + } + Py_ssize_t total_size = PyLong_AsInt(tmp); + Py_DECREF(tmp); + if (total_size < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "size must be a non-negative integer"); + } goto error; } @@ -345,10 +373,10 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } layout_fields = PySequence_Tuple(layout_fields_obj); Py_DECREF(layout_fields_obj); - Py_DECREF(layout); if (!layout_fields) { goto error; } + Py_CLEAR(layout); len = PySequence_Size(fields); if (len == -1) { @@ -379,7 +407,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct packstate.size = baseinfo->size; packstate.align = baseinfo->align; union_size = 0; - total_align = packstate.align ? packstate.align : 1; + total_align = baseinfo->align ? baseinfo->align : 1; total_align = max(total_align, forced_alignment); stginfo->ffi_type_pointer.type = FFI_TYPE_STRUCT; stginfo->ffi_type_pointer.elements = PyMem_New(ffi_type *, baseinfo->length + len + 1); @@ -410,6 +438,8 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct sizeof(ffi_type *) * (len + 1)); ffi_ofs = 0; } + packstate.size = total_size; + packstate.align = total_align; assert(stginfo->format == NULL); if (isStruct) { From c2440455186a0e0f5b0c0c3b45eaea0758df23a6 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 20:03:56 +0200 Subject: [PATCH 46/94] Remove some more of the packstate --- Modules/_ctypes/stgdict.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index e8846472bcb2b1..20131bfff660b1 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -404,8 +404,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } if (baseinfo) { - packstate.size = baseinfo->size; - packstate.align = baseinfo->align; union_size = 0; total_align = baseinfo->align ? baseinfo->align : 1; total_align = max(total_align, forced_alignment); @@ -424,8 +422,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } ffi_ofs = baseinfo->length; } else { - packstate.size = 0; - packstate.align = 0; union_size = 0; total_align = forced_alignment; stginfo->ffi_type_pointer.type = FFI_TYPE_STRUCT; @@ -439,7 +435,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct ffi_ofs = 0; } packstate.size = total_size; - packstate.align = total_align; assert(stginfo->format == NULL); if (isStruct) { @@ -555,18 +550,18 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct Py_CLEAR(layout_fields); if (!isStruct) { - packstate.size = union_size; + total_size = union_size; } /* Adjust the size according to the alignment requirements */ - aligned_size = ((packstate.size + total_align - 1) / total_align) * total_align; + aligned_size = ((total_size + total_align - 1) / total_align) * total_align; if (isStruct) { char *ptr; Py_ssize_t padding; /* Pad up to the full size of the struct */ - padding = aligned_size - packstate.size; + padding = aligned_size - total_size; if (padding > 0) { ptr = stginfo->format; stginfo->format = _ctypes_alloc_format_padding(ptr, padding); @@ -603,7 +598,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct # define MAX_STRUCT_SIZE 16 #endif - if (arrays_seen && (packstate.size <= MAX_STRUCT_SIZE)) { + if (arrays_seen && (total_size <= MAX_STRUCT_SIZE)) { /* * See bpo-22273 and gh-110190. Arrays are normally treated as * pointers, which is fine when an array name is being passed as From 903c0fa43a1b58ff599619db097ba7fcbc4befd1 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 20:15:04 +0200 Subject: [PATCH 47/94] Remove align from the state --- Lib/ctypes/_layout.py | 14 ++++++++++---- Modules/_ctypes/ctypes.h | 1 - Modules/_ctypes/stgdict.c | 4 ---- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 8ec8ccd6707997..203ae2a3697f7f 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -50,6 +50,12 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): elif align == 0: # Setting `_align_ = 0` amounts to using the default alignment align == 1 + self.forced_align = align + + if base: + total_align = max(1, ctypes.alignment(base), align) + else: + total_align = align swapped_bytes = hasattr(cls, '_swappedbytes_') if swapped_bytes: @@ -192,8 +198,8 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): ################################## State check (remove this) state_to_check = struct.pack( - "nn", - state_size, state_align + "n", + state_size ) ################################## @@ -213,10 +219,10 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): state_to_check=state_to_check, **self._field_args(), )) + total_align = max(total_align, state_align) - self.forced_align = align self.size = state_size - self.align = state_align + self.align = total_align def _field_args(self): return {} diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 81c059a0bebfd1..bcc38c14404583 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -249,7 +249,6 @@ struct fielddesc { typedef struct _CFieldPackState { Py_ssize_t size; // the size of the structure / union so far - Py_ssize_t align; // the alignment requirements of the last field placed } _CFieldPackState; // TODO: remove this... typedef struct CFieldObject { PyObject_HEAD diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 20131bfff660b1..fc1e6e2393e286 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -405,8 +405,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (baseinfo) { union_size = 0; - total_align = baseinfo->align ? baseinfo->align : 1; - total_align = max(total_align, forced_alignment); stginfo->ffi_type_pointer.type = FFI_TYPE_STRUCT; stginfo->ffi_type_pointer.elements = PyMem_New(ffi_type *, baseinfo->length + len + 1); if (stginfo->ffi_type_pointer.elements == NULL) { @@ -423,7 +421,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct ffi_ofs = baseinfo->length; } else { union_size = 0; - total_align = forced_alignment; stginfo->ffi_type_pointer.type = FFI_TYPE_STRUCT; stginfo->ffi_type_pointer.elements = PyMem_New(ffi_type *, len + 1); if (stginfo->ffi_type_pointer.elements == NULL) { @@ -540,7 +537,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct memcpy(&packstate, &prop->state_to_check, sizeof(_CFieldPackState)); union_size = max(packstate.size, union_size); } - total_align = max(packstate.align, total_align); if (-1 == PyObject_SetAttr(type, prop->name, prop_obj)) { goto error; From 551a6ed7f0ff0a12d95a698f0ed96a080b226c20 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 20:19:23 +0200 Subject: [PATCH 48/94] Compute padding in Python --- Lib/ctypes/_layout.py | 3 +++ Modules/_ctypes/cfield.c | 7 ++++-- Modules/_ctypes/clinic/cfield.c.h | 39 +++++++++++++++++++++++-------- Modules/_ctypes/ctypes.h | 1 + Modules/_ctypes/stgdict.c | 10 ++------ 5 files changed, 40 insertions(+), 20 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 203ae2a3697f7f..75ff1c9e34fbd5 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -87,6 +87,7 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): state_size = state_offset = ctypes.sizeof(base) state_align = ctypes.alignment(base) + last_size = state_size for i, field in enumerate(fields): if not is_struct: if isinstance(self, GCCSysVLayout): @@ -217,9 +218,11 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): pack=_pack_, index=i, state_to_check=state_to_check, + padding=offset - last_size, **self._field_args(), )) total_align = max(total_align, state_align) + last_size = state_size self.size = state_size self.align = total_align diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 1b8affd5627344..2e5f554c073b1a 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -117,6 +117,7 @@ _ctypes.CField.__new__ as PyCField_new _ms: bool = False pack as pack_obj: object = None state_to_check: object = NULL + padding: Py_ssize_t = 0 [clinic start generated code]*/ @@ -124,8 +125,9 @@ static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, PyObject *bit_size_obj, int swapped_bytes, int _ms, - PyObject *pack_obj, PyObject *state_to_check) -/*[clinic end generated code: output=cacdc95aba80f787 input=55968097dd1d41f3]*/ + PyObject *pack_obj, PyObject *state_to_check, + Py_ssize_t padding) +/*[clinic end generated code: output=4682681495942d3c input=2bc0747656719770]*/ { CFieldObject* self = NULL; if (size < 0) { @@ -233,6 +235,7 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, self->proto = Py_NewRef(proto); self->size = size; self->offset = offset; + self->padding = padding; if (swapped_bytes) { self->big_endian = !PY_BIG_ENDIAN; diff --git a/Modules/_ctypes/clinic/cfield.c.h b/Modules/_ctypes/clinic/cfield.c.h index 8a0fdf32639066..a61403be7da21d 100644 --- a/Modules/_ctypes/clinic/cfield.c.h +++ b/Modules/_ctypes/clinic/cfield.c.h @@ -13,7 +13,8 @@ static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, PyObject *bit_size_obj, int swapped_bytes, int _ms, - PyObject *pack_obj, PyObject *state_to_check); + PyObject *pack_obj, PyObject *state_to_check, + Py_ssize_t padding); static PyObject * PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -21,14 +22,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 10 + #define NUM_KEYWORDS 11 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(index), &_Py_ID(bit_size), &_Py_ID(swapped_bytes), &_Py_ID(_ms), &_Py_ID(pack), &_Py_ID(state_to_check), }, + .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(index), &_Py_ID(bit_size), &_Py_ID(swapped_bytes), &_Py_ID(_ms), &_Py_ID(pack), &_Py_ID(state_to_check), &_Py_ID(padding), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -37,14 +38,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "type", "size", "offset", "index", "bit_size", "swapped_bytes", "_ms", "pack", "state_to_check", NULL}; + static const char * const _keywords[] = {"name", "type", "size", "offset", "index", "bit_size", "swapped_bytes", "_ms", "pack", "state_to_check", "padding", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "CField", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[10]; + PyObject *argsbuf[11]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 5; @@ -57,9 +58,10 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int swapped_bytes = 0; int _ms = 0; PyObject *pack_obj = Py_None; - PyObject *state_to_check = Py_None; + PyObject *state_to_check = NULL; + Py_ssize_t padding = 0; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 5, 10, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 5, 11, 0, argsbuf); if (!fastargs) { goto exit; } @@ -138,11 +140,28 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_pos; } } - state_to_check = fastargs[9]; + if (fastargs[9]) { + state_to_check = fastargs[9]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(fastargs[10]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + padding = ival; + } skip_optional_pos: - return_value = PyCField_new_impl(type, name, proto, size, offset, index, bit_size_obj, swapped_bytes, _ms, pack_obj, state_to_check); + return_value = PyCField_new_impl(type, name, proto, size, offset, index, bit_size_obj, swapped_bytes, _ms, pack_obj, state_to_check, padding); exit: return return_value; } -/*[clinic end generated code: output=acb9bbd196bf5e44 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=07d746db3e0847a2 input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index bcc38c14404583..e02e4dbce5fae5 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -266,6 +266,7 @@ typedef struct CFieldObject { bool big_endian; /* boolean */ bool _ms_layout; Py_ssize_t pack; /* 0 if undefined */ + Py_ssize_t padding; _CFieldPackState state_to_check; // TODO: remove this. } CFieldObject; diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index fc1e6e2393e286..5993ea2b0d53b4 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -485,8 +485,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct char *ptr; Py_ssize_t len; char *buf; - Py_ssize_t last_size = packstate.size; - Py_ssize_t padding; if (fieldname == NULL) { goto error; @@ -497,13 +495,9 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct assert(prop); memcpy(&packstate, &prop->state_to_check, sizeof(_CFieldPackState)); - /* number of bytes between the end of the last field and the start - of this one */ - padding = prop->offset - last_size; - - if (padding > 0) { + if (prop->padding > 0) { ptr = stginfo->format; - stginfo->format = _ctypes_alloc_format_padding(ptr, padding); + stginfo->format = _ctypes_alloc_format_padding(ptr, prop->padding); PyMem_Free(ptr); if (stginfo->format == NULL) { goto error; From 0bb7797a57aacee01662a8335980fff7a20ddbaf Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 20:22:45 +0200 Subject: [PATCH 49/94] Move union_size to Python --- Modules/_ctypes/stgdict.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 5993ea2b0d53b4..6fd712d8a82a7c 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -241,7 +241,7 @@ int PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct) { Py_ssize_t len, i; - Py_ssize_t union_size, aligned_size; + Py_ssize_t aligned_size; _CFieldPackState packstate = {0}; PyObject *tmp; Py_ssize_t ffi_ofs; @@ -404,7 +404,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } if (baseinfo) { - union_size = 0; stginfo->ffi_type_pointer.type = FFI_TYPE_STRUCT; stginfo->ffi_type_pointer.elements = PyMem_New(ffi_type *, baseinfo->length + len + 1); if (stginfo->ffi_type_pointer.elements == NULL) { @@ -420,7 +419,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } ffi_ofs = baseinfo->length; } else { - union_size = 0; stginfo->ffi_type_pointer.type = FFI_TYPE_STRUCT; stginfo->ffi_type_pointer.elements = PyMem_New(ffi_type *, len + 1); if (stginfo->ffi_type_pointer.elements == NULL) { @@ -431,7 +429,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct sizeof(ffi_type *) * (len + 1)); ffi_ofs = 0; } - packstate.size = total_size; assert(stginfo->format == NULL); if (isStruct) { @@ -529,7 +526,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } else /* union */ { assert(prop); memcpy(&packstate, &prop->state_to_check, sizeof(_CFieldPackState)); - union_size = max(packstate.size, union_size); } if (-1 == PyObject_SetAttr(type, prop->name, prop_obj)) { @@ -539,10 +535,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } Py_CLEAR(layout_fields); - if (!isStruct) { - total_size = union_size; - } - /* Adjust the size according to the alignment requirements */ aligned_size = ((total_size + total_align - 1) / total_align) * total_align; From 9bc45c595b6b9c9cc3b20170616d2f413ab9b372 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 20:25:35 +0200 Subject: [PATCH 50/94] Remove the C state --- Lib/ctypes/_layout.py | 15 ++++++--------- Modules/_ctypes/cfield.c | 23 ++--------------------- Modules/_ctypes/clinic/cfield.c.h | 26 +++++++++----------------- Modules/_ctypes/ctypes.h | 7 +------ Modules/_ctypes/stgdict.c | 10 +--------- 5 files changed, 19 insertions(+), 62 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 75ff1c9e34fbd5..616639a32554c9 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -87,6 +87,7 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): state_size = state_offset = ctypes.sizeof(base) state_align = ctypes.alignment(base) + union_size = 0 last_size = state_size for i, field in enumerate(fields): if not is_struct: @@ -197,13 +198,6 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): state_size = state_offset; - ################################## State check (remove this) - state_to_check = struct.pack( - "n", - state_size - ) - ################################## - assert((not is_bitfield) or (LOW_BIT(size) <= size * 8)); if big_endian and is_bitfield: size = BUILD_SIZE(NUM_BITS(size), 8*info_size - LOW_BIT(size) - bit_size); @@ -217,14 +211,17 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): swapped_bytes=swapped_bytes, pack=_pack_, index=i, - state_to_check=state_to_check, padding=offset - last_size, **self._field_args(), )) total_align = max(total_align, state_align) last_size = state_size + union_size = max(state_size, union_size); - self.size = state_size + if is_struct: + self.size = state_size + else: + self.size = union_size self.align = total_align def _field_args(self): diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 2e5f554c073b1a..b95a1403453967 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -116,7 +116,6 @@ _ctypes.CField.__new__ as PyCField_new swapped_bytes: bool = False _ms: bool = False pack as pack_obj: object = None - state_to_check: object = NULL padding: Py_ssize_t = 0 [clinic start generated code]*/ @@ -125,9 +124,8 @@ static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, PyObject *bit_size_obj, int swapped_bytes, int _ms, - PyObject *pack_obj, PyObject *state_to_check, - Py_ssize_t padding) -/*[clinic end generated code: output=4682681495942d3c input=2bc0747656719770]*/ + PyObject *pack_obj, Py_ssize_t padding) +/*[clinic end generated code: output=fd3f7577cdc18215 input=797bc6b6969d0986]*/ { CFieldObject* self = NULL; if (size < 0) { @@ -279,23 +277,6 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, } } - if (state_to_check) { - char *buf; - Py_ssize_t len; - if (PyBytes_AsStringAndSize(state_to_check, &buf, &len) < 0) { - goto error; - } - if (len != sizeof(_CFieldPackState)) { - PyErr_Format(PyExc_ValueError, - "state_to_check size invalid for %R, want %zd, got %zd", - self->name, sizeof(_CFieldPackState), len); - } - memcpy(&self->state_to_check, buf, sizeof(_CFieldPackState)); - } - else { - memset(&self->state_to_check, 0xff, sizeof(_CFieldPackState)); - } - return (PyObject *)self; error: Py_XDECREF(self); diff --git a/Modules/_ctypes/clinic/cfield.c.h b/Modules/_ctypes/clinic/cfield.c.h index a61403be7da21d..d0b0efbf97d448 100644 --- a/Modules/_ctypes/clinic/cfield.c.h +++ b/Modules/_ctypes/clinic/cfield.c.h @@ -13,8 +13,7 @@ static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, PyObject *bit_size_obj, int swapped_bytes, int _ms, - PyObject *pack_obj, PyObject *state_to_check, - Py_ssize_t padding); + PyObject *pack_obj, Py_ssize_t padding); static PyObject * PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -22,14 +21,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 11 + #define NUM_KEYWORDS 10 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(index), &_Py_ID(bit_size), &_Py_ID(swapped_bytes), &_Py_ID(_ms), &_Py_ID(pack), &_Py_ID(state_to_check), &_Py_ID(padding), }, + .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(index), &_Py_ID(bit_size), &_Py_ID(swapped_bytes), &_Py_ID(_ms), &_Py_ID(pack), &_Py_ID(padding), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -38,14 +37,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "type", "size", "offset", "index", "bit_size", "swapped_bytes", "_ms", "pack", "state_to_check", "padding", NULL}; + static const char * const _keywords[] = {"name", "type", "size", "offset", "index", "bit_size", "swapped_bytes", "_ms", "pack", "padding", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "CField", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[11]; + PyObject *argsbuf[10]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 5; @@ -58,10 +57,9 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int swapped_bytes = 0; int _ms = 0; PyObject *pack_obj = Py_None; - PyObject *state_to_check = NULL; Py_ssize_t padding = 0; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 5, 11, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 5, 10, 0, argsbuf); if (!fastargs) { goto exit; } @@ -140,15 +138,9 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_pos; } } - if (fastargs[9]) { - state_to_check = fastargs[9]; - if (!--noptargs) { - goto skip_optional_pos; - } - } { Py_ssize_t ival = -1; - PyObject *iobj = _PyNumber_Index(fastargs[10]); + PyObject *iobj = _PyNumber_Index(fastargs[9]); if (iobj != NULL) { ival = PyLong_AsSsize_t(iobj); Py_DECREF(iobj); @@ -159,9 +151,9 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) padding = ival; } skip_optional_pos: - return_value = PyCField_new_impl(type, name, proto, size, offset, index, bit_size_obj, swapped_bytes, _ms, pack_obj, state_to_check, padding); + return_value = PyCField_new_impl(type, name, proto, size, offset, index, bit_size_obj, swapped_bytes, _ms, pack_obj, padding); exit: return return_value; } -/*[clinic end generated code: output=07d746db3e0847a2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d9326c74d88bf23f input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index e02e4dbce5fae5..1b5da63036391a 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -247,9 +247,6 @@ struct fielddesc { GETFUNC getfunc_swapped; }; -typedef struct _CFieldPackState { - Py_ssize_t size; // the size of the structure / union so far -} _CFieldPackState; // TODO: remove this... typedef struct CFieldObject { PyObject_HEAD Py_ssize_t offset; @@ -266,9 +263,7 @@ typedef struct CFieldObject { bool big_endian; /* boolean */ bool _ms_layout; Py_ssize_t pack; /* 0 if undefined */ - Py_ssize_t padding; - - _CFieldPackState state_to_check; // TODO: remove this. + Py_ssize_t padding; /* number of bytes between the end of the last field and the start of this one */ } CFieldObject; /**************************************************************** diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 6fd712d8a82a7c..4f0620daee136c 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -242,7 +242,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct { Py_ssize_t len, i; Py_ssize_t aligned_size; - _CFieldPackState packstate = {0}; PyObject *tmp; Py_ssize_t ffi_ofs; int arrays_seen = 0; @@ -476,6 +475,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct stginfo->flags |= TYPEFLAG_HASPOINTER; info->flags |= DICTFLAG_FINAL; /* mark field type final */ + assert(prop); if (isStruct) { const char *fieldfmt = info->format ? info->format : "B"; const char *fieldname = PyUnicode_AsUTF8(prop->name); @@ -487,11 +487,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; } - /* construct the field now, as `prop->offset` is `offset` with - corrected alignment */ - assert(prop); - memcpy(&packstate, &prop->state_to_check, sizeof(_CFieldPackState)); - if (prop->padding > 0) { ptr = stginfo->format; stginfo->format = _ctypes_alloc_format_padding(ptr, prop->padding); @@ -523,9 +518,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (stginfo->format == NULL) { goto error; } - } else /* union */ { - assert(prop); - memcpy(&packstate, &prop->state_to_check, sizeof(_CFieldPackState)); } if (-1 == PyObject_SetAttr(type, prop->name, prop_obj)) { From 7a76a8e6072a2467ab829aad6305688fe6039d80 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 20:36:17 +0200 Subject: [PATCH 51/94] Set up for checking format --- Lib/ctypes/_layout.py | 1 + Modules/_ctypes/cfield.c | 8 ++++-- Modules/_ctypes/clinic/cfield.c.h | 41 ++++++++++++++++++------------- Modules/_ctypes/ctypes.h | 1 + Modules/_ctypes/stgdict.c | 9 +++++++ 5 files changed, 41 insertions(+), 19 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 616639a32554c9..3ef2134d6c1173 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -212,6 +212,7 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): pack=_pack_, index=i, padding=offset - last_size, + #format="", **self._field_args(), )) total_align = max(total_align, state_align) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index b95a1403453967..669a8802fc7222 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -117,6 +117,7 @@ _ctypes.CField.__new__ as PyCField_new _ms: bool = False pack as pack_obj: object = None padding: Py_ssize_t = 0 + format: object = NULL [clinic start generated code]*/ @@ -124,8 +125,8 @@ static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, PyObject *bit_size_obj, int swapped_bytes, int _ms, - PyObject *pack_obj, Py_ssize_t padding) -/*[clinic end generated code: output=fd3f7577cdc18215 input=797bc6b6969d0986]*/ + PyObject *pack_obj, Py_ssize_t padding, PyObject *format) +/*[clinic end generated code: output=f28af1d73f425818 input=42f2966d4050f232]*/ { CFieldObject* self = NULL; if (size < 0) { @@ -234,6 +235,7 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, self->size = size; self->offset = offset; self->padding = padding; + self->format = Py_XNewRef(format); if (swapped_bytes) { self->big_endian = !PY_BIG_ENDIAN; @@ -347,6 +349,7 @@ PyCField_traverse(CFieldObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); Py_VISIT(self->proto); + Py_VISIT(self->format); return 0; } @@ -354,6 +357,7 @@ static int PyCField_clear(CFieldObject *self) { Py_CLEAR(self->proto); + Py_CLEAR(self->format); return 0; } diff --git a/Modules/_ctypes/clinic/cfield.c.h b/Modules/_ctypes/clinic/cfield.c.h index d0b0efbf97d448..e9e3e774b5385c 100644 --- a/Modules/_ctypes/clinic/cfield.c.h +++ b/Modules/_ctypes/clinic/cfield.c.h @@ -13,7 +13,7 @@ static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, PyObject *bit_size_obj, int swapped_bytes, int _ms, - PyObject *pack_obj, Py_ssize_t padding); + PyObject *pack_obj, Py_ssize_t padding, PyObject *format); static PyObject * PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -21,14 +21,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 10 + #define NUM_KEYWORDS 11 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(index), &_Py_ID(bit_size), &_Py_ID(swapped_bytes), &_Py_ID(_ms), &_Py_ID(pack), &_Py_ID(padding), }, + .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(index), &_Py_ID(bit_size), &_Py_ID(swapped_bytes), &_Py_ID(_ms), &_Py_ID(pack), &_Py_ID(padding), &_Py_ID(format), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -37,14 +37,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "type", "size", "offset", "index", "bit_size", "swapped_bytes", "_ms", "pack", "padding", NULL}; + static const char * const _keywords[] = {"name", "type", "size", "offset", "index", "bit_size", "swapped_bytes", "_ms", "pack", "padding", "format", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "CField", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[10]; + PyObject *argsbuf[11]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 5; @@ -58,8 +58,9 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int _ms = 0; PyObject *pack_obj = Py_None; Py_ssize_t padding = 0; + PyObject *format = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 5, 10, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 5, 11, 0, argsbuf); if (!fastargs) { goto exit; } @@ -138,22 +139,28 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_pos; } } - { - Py_ssize_t ival = -1; - PyObject *iobj = _PyNumber_Index(fastargs[9]); - if (iobj != NULL) { - ival = PyLong_AsSsize_t(iobj); - Py_DECREF(iobj); + if (fastargs[9]) { + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(fastargs[9]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + padding = ival; } - if (ival == -1 && PyErr_Occurred()) { - goto exit; + if (!--noptargs) { + goto skip_optional_pos; } - padding = ival; } + format = fastargs[10]; skip_optional_pos: - return_value = PyCField_new_impl(type, name, proto, size, offset, index, bit_size_obj, swapped_bytes, _ms, pack_obj, padding); + return_value = PyCField_new_impl(type, name, proto, size, offset, index, bit_size_obj, swapped_bytes, _ms, pack_obj, padding, format); exit: return return_value; } -/*[clinic end generated code: output=d9326c74d88bf23f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e851010bf9297fdc input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 1b5da63036391a..229e9404708375 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -264,6 +264,7 @@ typedef struct CFieldObject { bool _ms_layout; Py_ssize_t pack; /* 0 if undefined */ Py_ssize_t padding; /* number of bytes between the end of the last field and the start of this one */ + PyObject *format; /* unicode */ } CFieldObject; /**************************************************************** diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 4f0620daee136c..dce54981dd3215 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -518,6 +518,15 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (stginfo->format == NULL) { goto error; } + if (prop->format && prop->format != Py_None) { + const char *buf = PyUnicode_AsUTF8(prop->format); + if (!buf) goto error; + if (strcmp(stginfo->format, buf)) { + PyErr_Format(PyExc_AssertionError, + "formats don't match after field %R:\nexp: \"%s\"\ngot: \"%s\"", prop->name, stginfo->format, buf); + goto error; + } + } } if (-1 == PyObject_SetAttr(type, prop->name, prop_obj)) { From cdd028c0f13a466a180582c59163f216dbdf8d95 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 21:06:12 +0200 Subject: [PATCH 52/94] Compute the format spec in Python --- Lib/ctypes/_layout.py | 48 +++++++++++++++++++++++++++++++++++---- Modules/_ctypes/stgdict.c | 18 +++++++++++++++ 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 3ef2134d6c1173..36b18443e2a61b 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -2,7 +2,7 @@ import warnings import struct -from _ctypes import CField +from _ctypes import CField, buffer_info import ctypes def round_down(n, multiple): @@ -76,6 +76,11 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): self.fields = [] + if is_struct: + format_spec = "T{" + else: + format_spec = "B" + state_field_size = 0 # `8 * offset + bitofs` points to where the next field would start. state_bitofs = 0 @@ -202,6 +207,23 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): if big_endian and is_bitfield: size = BUILD_SIZE(NUM_BITS(size), 8*info_size - LOW_BIT(size) - bit_size); + padding = offset - last_size + if is_struct: + format_spec += padding_spec(padding) + + fieldfmt, bf_ndim, bf_shape = buffer_info(ftype) + if fieldfmt is None: + fieldfmt = "B" + if ftype is cls: + # This is wrong, but it matches behavior of the previous + # C implementation. We'll error out later, anyway. + fieldfmt = format_spec + buf = f"{fieldfmt}:{name}:" + if bf_shape: + shape_numbers = ",".join(str(n) for n in bf_shape) + buf = f"({shape_numbers}){buf}" + format_spec += buf + self.fields.append(CField( name=name, type=ftype, @@ -211,8 +233,8 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): swapped_bytes=swapped_bytes, pack=_pack_, index=i, - padding=offset - last_size, - #format="", + padding=padding, + format=format_spec, **self._field_args(), )) total_align = max(total_align, state_align) @@ -220,14 +242,30 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): union_size = max(state_size, union_size); if is_struct: - self.size = state_size + total_size = state_size else: - self.size = union_size + total_size = union_size + + aligned_size = int((total_size + total_align - 1) / total_align) * total_align + + if is_struct: + padding = aligned_size - total_size + format_spec += f"{padding_spec(padding)}}}" + + self.size = total_size self.align = total_align + self.format_spec = format_spec def _field_args(self): return {} +def padding_spec(padding): + if padding <= 0: + return "" + if padding == 1: + return "x" + return f"{padding}x" + class WindowsLayout(_BaseLayout): def _field_args(self): diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index dce54981dd3215..f1f8f700a105de 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -251,6 +251,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct PyObject *prop_obj = NULL; PyObject *layout_fields = NULL; PyObject *layout = NULL; + PyObject *format_spec_obj = NULL; if (fields == NULL) { return 0; @@ -366,6 +367,15 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; } + format_spec_obj = PyObject_GetAttrString(layout, "format_spec"); + if (!format_spec_obj) { + goto error; + } + const char *format_spec = PyUnicode_AsUTF8(format_spec_obj); + if (!format_spec) { + goto error; + } + PyObject *layout_fields_obj = PyObject_GetAttrString(layout, "fields"); if (!layout_fields_obj) { goto error; @@ -561,6 +571,12 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; } + if (strcmp(stginfo->format, format_spec)) { + PyErr_Format(PyExc_AssertionError, + "formats don't match at end:\nexp: \"%s\"\ngot: \"%s\"", stginfo->format, format_spec); + goto error; + } + stginfo->ffi_type_pointer.alignment = Py_SAFE_DOWNCAST(total_align, Py_ssize_t, unsigned short); @@ -855,5 +871,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct error: Py_XDECREF(prop_obj); Py_XDECREF(layout_fields); + Py_XDECREF(layout); + Py_XDECREF(format_spec_obj); return -1; } From 4de515f521afc85b9601154da35199dd2431bea9 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 21:07:48 +0200 Subject: [PATCH 53/94] Fix refcounting/error handling --- Modules/_ctypes/stgdict.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index f1f8f700a105de..e7b12764dc90d2 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -246,6 +246,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct Py_ssize_t ffi_ofs; int arrays_seen = 0; + int retval = -1; // The following are NULL or hold strong references. // They're cleared on error. PyObject *prop_obj = NULL; @@ -867,11 +868,11 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } stginfo->flags |= DICTFLAG_FINAL; - return MakeAnonFields(type); + retval = MakeAnonFields(type); error: Py_XDECREF(prop_obj); Py_XDECREF(layout_fields); Py_XDECREF(layout); Py_XDECREF(format_spec_obj); - return -1; + return retval; } From adbe0194cffe59c4be4ce5bf8b126039fbb53e4a Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 21:16:25 +0200 Subject: [PATCH 54/94] Remove the format calculations from C --- Lib/ctypes/_layout.py | 4 -- Modules/_ctypes/_ctypes.c | 4 +- Modules/_ctypes/ctypes.h | 4 -- Modules/_ctypes/stgdict.c | 123 +++----------------------------------- 4 files changed, 11 insertions(+), 124 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 36b18443e2a61b..4a545146728f19 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -214,10 +214,6 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): fieldfmt, bf_ndim, bf_shape = buffer_info(ftype) if fieldfmt is None: fieldfmt = "B" - if ftype is cls: - # This is wrong, but it matches behavior of the previous - # C implementation. We'll error out later, anyway. - fieldfmt = format_spec buf = f"{fieldfmt}:{name}:" if bf_shape: shape_numbers = ",".join(str(n) for n in bf_shape) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 078e756da51f91..2b23be7b753e34 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -320,7 +320,7 @@ _ctypes_alloc_format_string_for_type(char code, int big_endian) indicator set. If called with a suffix of NULL the error indicator must already be set. */ -char * +static char * _ctypes_alloc_format_string(const char *prefix, const char *suffix) { size_t len; @@ -352,7 +352,7 @@ _ctypes_alloc_format_string(const char *prefix, const char *suffix) Returns NULL on failure, with the error indicator set. If called with a suffix of NULL the error indicator must already be set. */ -char * +static char * _ctypes_alloc_format_string_with_shape(int ndim, const Py_ssize_t *shape, const char *prefix, const char *suffix) { diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 229e9404708375..5e489332b1a362 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -430,10 +430,6 @@ extern void *_ctypes_alloc_closure(void); extern PyObject *PyCData_FromBaseObj(ctypes_state *st, PyObject *type, PyObject *base, Py_ssize_t index, char *adr); -extern char *_ctypes_alloc_format_string(const char *prefix, const char *suffix); -extern char *_ctypes_alloc_format_string_with_shape(int ndim, - const Py_ssize_t *shape, - const char *prefix, const char *suffix); extern int _ctypes_simple_instance(ctypes_state *st, PyObject *obj); diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index e7b12764dc90d2..66ab2e7028bc8a 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -210,29 +210,6 @@ MakeAnonFields(PyObject *type) return 0; } -/* - Allocate a memory block for a pep3118 format string, copy prefix (if - non-null) into it and append `{padding}x` to the end. - Returns NULL on failure, with the error indicator set. -*/ -char * -_ctypes_alloc_format_padding(const char *prefix, Py_ssize_t padding) -{ - /* int64 decimal characters + x + null */ - char buf[19 + 1 + 1]; - - assert(padding > 0); - - if (padding == 1) { - /* Use x instead of 1x, for brevity */ - return _ctypes_alloc_format_string(prefix, "x"); - } - - int ret = PyOS_snprintf(buf, sizeof(buf), "%zdx", padding); (void)ret; - assert(0 <= ret && ret < (Py_ssize_t)sizeof(buf)); - return _ctypes_alloc_format_string(prefix, buf); -} - /* Retrieve the (optional) _pack_ attribute from a type, the _fields_ attribute, and initialize StgInfo. Used for Structure and Union subclasses. @@ -372,7 +349,9 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (!format_spec_obj) { goto error; } - const char *format_spec = PyUnicode_AsUTF8(format_spec_obj); + Py_ssize_t format_spec_size; + const char *format_spec = PyUnicode_AsUTF8AndSize(format_spec_obj, + &format_spec_size); if (!format_spec) { goto error; } @@ -408,6 +387,12 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct PyMem_Free(stginfo->format); stginfo->format = NULL; } + stginfo->format = PyMem_Malloc(format_spec_size + 1); + if (!stginfo->format) { + PyErr_NoMemory(); + goto error; + } + memcpy(stginfo->format, format_spec, format_spec_size + 1); if (stginfo->ffi_type_pointer.elements) { PyMem_Free(stginfo->ffi_type_pointer.elements); @@ -440,16 +425,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct ffi_ofs = 0; } - assert(stginfo->format == NULL); - if (isStruct) { - stginfo->format = _ctypes_alloc_format_string(NULL, "T{"); - } else { - /* PEP3118 doesn't support union. Use 'B' for bytes. */ - stginfo->format = _ctypes_alloc_format_string(NULL, "B"); - } - if (stginfo->format == NULL) - goto error; - for (i = 0; i < len; ++i) { prop_obj = PySequence_Fast_GET_ITEM(layout_fields, i); if (!prop_obj) { @@ -487,58 +462,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct info->flags |= DICTFLAG_FINAL; /* mark field type final */ assert(prop); - if (isStruct) { - const char *fieldfmt = info->format ? info->format : "B"; - const char *fieldname = PyUnicode_AsUTF8(prop->name); - char *ptr; - Py_ssize_t len; - char *buf; - - if (fieldname == NULL) { - goto error; - } - - if (prop->padding > 0) { - ptr = stginfo->format; - stginfo->format = _ctypes_alloc_format_padding(ptr, prop->padding); - PyMem_Free(ptr); - if (stginfo->format == NULL) { - goto error; - } - } - - len = strlen(fieldname) + strlen(fieldfmt); - - buf = PyMem_Malloc(len + 2 + 1); - if (buf == NULL) { - PyErr_NoMemory(); - goto error; - } - sprintf(buf, "%s:%s:", fieldfmt, fieldname); - - ptr = stginfo->format; - if (info->shape != NULL) { - stginfo->format = _ctypes_alloc_format_string_with_shape( - info->ndim, info->shape, stginfo->format, buf); - } else { - stginfo->format = _ctypes_alloc_format_string(stginfo->format, buf); - } - PyMem_Free(ptr); - PyMem_Free(buf); - - if (stginfo->format == NULL) { - goto error; - } - if (prop->format && prop->format != Py_None) { - const char *buf = PyUnicode_AsUTF8(prop->format); - if (!buf) goto error; - if (strcmp(stginfo->format, buf)) { - PyErr_Format(PyExc_AssertionError, - "formats don't match after field %R:\nexp: \"%s\"\ngot: \"%s\"", prop->name, stginfo->format, buf); - goto error; - } - } - } if (-1 == PyObject_SetAttr(type, prop->name, prop_obj)) { goto error; @@ -550,34 +473,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct /* Adjust the size according to the alignment requirements */ aligned_size = ((total_size + total_align - 1) / total_align) * total_align; - if (isStruct) { - char *ptr; - Py_ssize_t padding; - - /* Pad up to the full size of the struct */ - padding = aligned_size - total_size; - if (padding > 0) { - ptr = stginfo->format; - stginfo->format = _ctypes_alloc_format_padding(ptr, padding); - PyMem_Free(ptr); - if (stginfo->format == NULL) { - goto error; - } - } - - ptr = stginfo->format; - stginfo->format = _ctypes_alloc_format_string(stginfo->format, "}"); - PyMem_Free(ptr); - if (stginfo->format == NULL) - goto error; - } - - if (strcmp(stginfo->format, format_spec)) { - PyErr_Format(PyExc_AssertionError, - "formats don't match at end:\nexp: \"%s\"\ngot: \"%s\"", stginfo->format, format_spec); - goto error; - } - stginfo->ffi_type_pointer.alignment = Py_SAFE_DOWNCAST(total_align, Py_ssize_t, unsigned short); From c1644b6458c1724b21341f060f73b6a9a765a555 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 21:20:22 +0200 Subject: [PATCH 55/94] Return the aligned size from Python --- Lib/ctypes/_layout.py | 3 ++- Modules/_ctypes/stgdict.c | 8 ++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 4a545146728f19..ecb96fe8959ca6 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -242,13 +242,14 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): else: total_size = union_size + # Adjust the size according to the alignment requirements aligned_size = int((total_size + total_align - 1) / total_align) * total_align if is_struct: padding = aligned_size - total_size format_spec += f"{padding_spec(padding)}}}" - self.size = total_size + self.size = aligned_size self.align = total_align self.format_spec = format_spec diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 66ab2e7028bc8a..469a5762552a5e 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -218,7 +218,6 @@ int PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct) { Py_ssize_t len, i; - Py_ssize_t aligned_size; PyObject *tmp; Py_ssize_t ffi_ofs; int arrays_seen = 0; @@ -470,15 +469,12 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } Py_CLEAR(layout_fields); - /* Adjust the size according to the alignment requirements */ - aligned_size = ((total_size + total_align - 1) / total_align) * total_align; - stginfo->ffi_type_pointer.alignment = Py_SAFE_DOWNCAST(total_align, Py_ssize_t, unsigned short); - stginfo->ffi_type_pointer.size = aligned_size; + stginfo->ffi_type_pointer.size = total_size; - stginfo->size = aligned_size; + stginfo->size = total_size; stginfo->align = total_align; stginfo->length = ffi_ofs + len; From 4c293798cf7f3f03b121518091db6b6f8e16ee93 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 21:27:48 +0200 Subject: [PATCH 56/94] layout_fields is a tuple --- Modules/_ctypes/stgdict.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 469a5762552a5e..bb7acd637b3d7c 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -375,10 +375,10 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; } - if (len != PySequence_Fast_GET_SIZE(layout_fields)) { + if (len != PyTuple_GET_SIZE(layout_fields)) { PyErr_Format(PyExc_ValueError, "number of '_fields_' must match result of layout... for now. want %zd, got %zd", - len, PySequence_Fast_GET_SIZE(layout_fields)); + len, PyTuple_GET_SIZE(layout_fields)); goto error; } @@ -425,7 +425,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } for (i = 0; i < len; ++i) { - prop_obj = PySequence_Fast_GET_ITEM(layout_fields, i); + prop_obj = PyTuple_GET_ITEM(layout_fields, i); if (!prop_obj) { goto error; } From c2e54a8aa9df819d2b778dea0aa3e2f400043c27 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 21:32:14 +0200 Subject: [PATCH 57/94] Borrow from the layout_fields tuple --- Modules/_ctypes/stgdict.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index bb7acd637b3d7c..768e1f4104e983 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -225,7 +225,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct int retval = -1; // The following are NULL or hold strong references. // They're cleared on error. - PyObject *prop_obj = NULL; PyObject *layout_fields = NULL; PyObject *layout = NULL; PyObject *format_spec_obj = NULL; @@ -425,11 +424,8 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } for (i = 0; i < len; ++i) { - prop_obj = PyTuple_GET_ITEM(layout_fields, i); - if (!prop_obj) { - goto error; - } - Py_INCREF(prop_obj); + PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); + assert(prop_obj); if (!PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type)) { PyErr_Format(PyExc_TypeError, "fields must be of type CField, got %T", prop_obj); @@ -465,9 +461,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (-1 == PyObject_SetAttr(type, prop->name, prop_obj)) { goto error; } - Py_CLEAR(prop_obj); } - Py_CLEAR(layout_fields); stginfo->ffi_type_pointer.alignment = Py_SAFE_DOWNCAST(total_align, Py_ssize_t, @@ -569,6 +563,11 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct PyObject *pair = PySequence_GetItem(fields, i); int bitsize = 0; + PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); // borrowed + assert(prop_obj); + assert(PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type)); + CFieldObject *prop = (CFieldObject *)prop_obj; // borrowed + if (pair == NULL) { goto error; } @@ -666,6 +665,11 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct PyObject *pair = PySequence_GetItem(fields, i); int bitsize = 0; + PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); // borrowed + assert(prop_obj); + assert(PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type)); + CFieldObject *prop = (CFieldObject *)prop_obj; // borrowed + if (pair == NULL) { PyMem_Free(type_block); goto error; @@ -761,7 +765,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct retval = MakeAnonFields(type); error: - Py_XDECREF(prop_obj); Py_XDECREF(layout_fields); Py_XDECREF(layout); Py_XDECREF(format_spec_obj); From 61a33d3e5067e3fcd51d56373d011cd4e5efe7d1 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 21:35:30 +0200 Subject: [PATCH 58/94] Use stuff from prop, not pair --- Modules/_ctypes/stgdict.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 768e1f4104e983..90a6797cdd4730 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -579,7 +579,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } StgInfo *info; - if (PyStgInfo_FromType(st, desc, &info) < 0) { + if (PyStgInfo_FromType(st, prop->proto, &info) < 0) { Py_DECREF(pair); goto error; } @@ -591,7 +591,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; } - if (!PyCArrayTypeObject_Check(st, desc)) { + if (!PyCArrayTypeObject_Check(st, prop->proto)) { /* Not an array. Just need an ffi_type pointer. */ num_ffi_type_pointers++; } @@ -689,7 +689,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } StgInfo *info; - if (PyStgInfo_FromType(st, desc, &info) < 0) { + if (PyStgInfo_FromType(st, prop->proto, &info) < 0) { Py_DECREF(pair); PyMem_Free(type_block); goto error; @@ -706,7 +706,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } assert(element_index < (ffi_ofs + len)); /* will be used below */ - if (!PyCArrayTypeObject_Check(st, desc)) { + if (!PyCArrayTypeObject_Check(st, prop->proto)) { /* Not an array. Just copy over the element ffi_type. */ element_types[element_index++] = &info->ffi_type_pointer; } From f3f3622a85509ab64969c71d24d6841a1f0c61db Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 21:39:50 +0200 Subject: [PATCH 59/94] Remove use of pair --- Modules/_ctypes/stgdict.c | 63 ++------------------------------------- 1 file changed, 2 insertions(+), 61 deletions(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 90a6797cdd4730..ba2774b498945e 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -559,37 +559,16 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct /* first pass to see how much memory to allocate */ for (i = 0; i < len; ++i) { - PyObject *name, *desc; - PyObject *pair = PySequence_GetItem(fields, i); - int bitsize = 0; - PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); // borrowed assert(prop_obj); assert(PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type)); CFieldObject *prop = (CFieldObject *)prop_obj; // borrowed - if (pair == NULL) { - goto error; - } - if (!PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) { - PyErr_SetString(PyExc_TypeError, - "'_fields_' must be a sequence of (name, C type) pairs"); - Py_DECREF(pair); - goto error; - } - StgInfo *info; if (PyStgInfo_FromType(st, prop->proto, &info) < 0) { - Py_DECREF(pair); - goto error; - } - if (info == NULL) { - Py_DECREF(pair); - PyErr_Format(PyExc_TypeError, - "second item in _fields_ tuple (index %zd) must be a C type", - i); goto error; } + assert(info); if (!PyCArrayTypeObject_Check(st, prop->proto)) { /* Not an array. Just need an ffi_type pointer. */ @@ -601,11 +580,9 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct StgInfo *einfo; if (PyStgInfo_FromType(st, info->proto, &einfo) < 0) { - Py_DECREF(pair); goto error; } if (einfo == NULL) { - Py_DECREF(pair); PyErr_Format(PyExc_TypeError, "second item in _fields_ tuple (index %zd) must be a C type", i); @@ -619,7 +596,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct num_ffi_types++; num_ffi_type_pointers += length + 1; } - Py_DECREF(pair); } /* @@ -661,49 +637,17 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct /* second pass to actually set the type pointers */ for (i = 0; i < len; ++i) { - PyObject *name, *desc; - PyObject *pair = PySequence_GetItem(fields, i); - int bitsize = 0; - PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); // borrowed assert(prop_obj); assert(PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type)); CFieldObject *prop = (CFieldObject *)prop_obj; // borrowed - if (pair == NULL) { - PyMem_Free(type_block); - goto error; - } - /* In theory, we made this call in the first pass, so it *shouldn't* - * fail. However, you never know, and the code above might change - * later - keeping the check in here is a tad defensive but it - * will affect program size only slightly and performance hardly at - * all. - */ - if (!PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) { - PyErr_SetString(PyExc_TypeError, - "'_fields_' must be a sequence of (name, C type) pairs"); - Py_DECREF(pair); - PyMem_Free(type_block); - goto error; - } - StgInfo *info; if (PyStgInfo_FromType(st, prop->proto, &info) < 0) { - Py_DECREF(pair); - PyMem_Free(type_block); - goto error; - } - - /* Possibly this check could be avoided, but see above comment. */ - if (info == NULL) { - Py_DECREF(pair); PyMem_Free(type_block); - PyErr_Format(PyExc_TypeError, - "second item in _fields_ tuple (index %zd) must be a C type", - i); goto error; } + assert(info); assert(element_index < (ffi_ofs + len)); /* will be used below */ if (!PyCArrayTypeObject_Check(st, prop->proto)) { @@ -714,12 +658,10 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct Py_ssize_t length = info->length; StgInfo *einfo; if (PyStgInfo_FromType(st, info->proto, &einfo) < 0) { - Py_DECREF(pair); PyMem_Free(type_block); goto error; } if (einfo == NULL) { - Py_DECREF(pair); PyMem_Free(type_block); PyErr_Format(PyExc_TypeError, "second item in _fields_ tuple (index %zd) must be a C type", @@ -741,7 +683,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct assert(dummy_index < (num_ffi_type_pointers)); dummy_types[dummy_index++] = NULL; } - Py_DECREF(pair); } element_types[element_index] = NULL; From 34edfd8671f67e97bb4070c385c0da3751134d6f Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 21:43:10 +0200 Subject: [PATCH 60/94] Only pass fields to Python, do nothing else with it --- Modules/_ctypes/stgdict.c | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index ba2774b498945e..93aba941b0377d 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -217,7 +217,7 @@ MakeAnonFields(PyObject *type) int PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct) { - Py_ssize_t len, i; + Py_ssize_t i; PyObject *tmp; Py_ssize_t ffi_ofs; int arrays_seen = 0; @@ -297,6 +297,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct Py_DECREF(kwnames); Py_DECREF(base_arg); Py_DECREF(layout_class); + fields = NULL; // a borrowed reference we won't be using again if (!layout) { goto error; } @@ -365,21 +366,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } Py_CLEAR(layout); - len = PySequence_Size(fields); - if (len == -1) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_SetString(PyExc_TypeError, - "'_fields_' must be a sequence of pairs"); - } - goto error; - } - - if (len != PyTuple_GET_SIZE(layout_fields)) { - PyErr_Format(PyExc_ValueError, - "number of '_fields_' must match result of layout... for now. want %zd, got %zd", - len, PyTuple_GET_SIZE(layout_fields)); - goto error; - } + Py_ssize_t len = PyTuple_GET_SIZE(layout_fields); if (stginfo->format) { PyMem_Free(stginfo->format); From 833edf480e66e2d0a8ada8003639ec2c06911920 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 21:49:22 +0200 Subject: [PATCH 61/94] Don't pass forced_align --- Lib/ctypes/_layout.py | 1 - Modules/_ctypes/stgdict.c | 14 -------------- 2 files changed, 15 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index ecb96fe8959ca6..053f65de3ff4f6 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -50,7 +50,6 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): elif align == 0: # Setting `_align_ = 0` amounts to using the default alignment align == 1 - self.forced_align = align if base: total_align = max(1, ctypes.alignment(base), align) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 93aba941b0377d..35ce23ff38b2f4 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -302,20 +302,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; } - tmp = PyObject_GetAttrString(layout, "forced_align"); - if (!tmp) { - goto error; - } - int forced_alignment = PyLong_AsInt(tmp); - Py_DECREF(tmp); - if (forced_alignment < 0) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_ValueError, - "_align_ must be a non-negative integer"); - } - goto error; - } - tmp = PyObject_GetAttrString(layout, "align"); if (!tmp) { goto error; From 779fa8164251b64e74d487cd9b81115bf87f1aa3 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 21:54:06 +0200 Subject: [PATCH 62/94] Remove scaffolding members from CFieldObject --- Lib/ctypes/_layout.py | 11 +---- Modules/_ctypes/cfield.c | 34 +-------------- Modules/_ctypes/clinic/cfield.c.h | 71 ++++--------------------------- Modules/_ctypes/ctypes.h | 5 --- 4 files changed, 12 insertions(+), 109 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 053f65de3ff4f6..414c3bc329c037 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -225,12 +225,7 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): size=size, offset=offset, bit_size=bit_size if is_bitfield else None, - swapped_bytes=swapped_bytes, - pack=_pack_, index=i, - padding=padding, - format=format_spec, - **self._field_args(), )) total_align = max(total_align, state_align) last_size = state_size @@ -252,9 +247,6 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): self.align = total_align self.format_spec = format_spec - def _field_args(self): - return {} - def padding_spec(padding): if padding <= 0: return "" @@ -264,8 +256,7 @@ def padding_spec(padding): class WindowsLayout(_BaseLayout): - def _field_args(self): - return {'_ms': True} + pass class GCCSysVLayout(_BaseLayout): def __init__(self, cls, *args, **kwargs): diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 669a8802fc7222..a3f887c1916e78 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -113,20 +113,14 @@ _ctypes.CField.__new__ as PyCField_new offset: Py_ssize_t index: Py_ssize_t bit_size as bit_size_obj: object = None - swapped_bytes: bool = False - _ms: bool = False - pack as pack_obj: object = None - padding: Py_ssize_t = 0 - format: object = NULL [clinic start generated code]*/ static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, - PyObject *bit_size_obj, int swapped_bytes, int _ms, - PyObject *pack_obj, Py_ssize_t padding, PyObject *format) -/*[clinic end generated code: output=f28af1d73f425818 input=42f2966d4050f232]*/ + PyObject *bit_size_obj) +/*[clinic end generated code: output=43649ef9157c5f58 input=3d813f56373c4caa]*/ { CFieldObject* self = NULL; if (size < 0) { @@ -217,32 +211,10 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, } } - if (pack_obj == Py_None) { - self->pack = 0; - } - else { - self->pack = PyLong_AsSsize_t(pack_obj); - if (self->pack < 0) { - if (!PyErr_Occurred()) { - PyErr_Format(PyExc_ValueError, - "_pack_ cannot be negative"); - } - goto error; - } - } - self->proto = Py_NewRef(proto); self->size = size; self->offset = offset; - self->padding = padding; - self->format = Py_XNewRef(format); - if (swapped_bytes) { - self->big_endian = !PY_BIG_ENDIAN; - } else { - self->big_endian = PY_BIG_ENDIAN; - } - self->_ms_layout = _ms; self->index = index; /* Field descriptors for 'c_char * n' are be scpecial cased to @@ -349,7 +321,6 @@ PyCField_traverse(CFieldObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); Py_VISIT(self->proto); - Py_VISIT(self->format); return 0; } @@ -357,7 +328,6 @@ static int PyCField_clear(CFieldObject *self) { Py_CLEAR(self->proto); - Py_CLEAR(self->format); return 0; } diff --git a/Modules/_ctypes/clinic/cfield.c.h b/Modules/_ctypes/clinic/cfield.c.h index e9e3e774b5385c..df5da783e050ed 100644 --- a/Modules/_ctypes/clinic/cfield.c.h +++ b/Modules/_ctypes/clinic/cfield.c.h @@ -12,8 +12,7 @@ preserve static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, - PyObject *bit_size_obj, int swapped_bytes, int _ms, - PyObject *pack_obj, Py_ssize_t padding, PyObject *format); + PyObject *bit_size_obj); static PyObject * PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -21,14 +20,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 11 + #define NUM_KEYWORDS 6 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(index), &_Py_ID(bit_size), &_Py_ID(swapped_bytes), &_Py_ID(_ms), &_Py_ID(pack), &_Py_ID(padding), &_Py_ID(format), }, + .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(index), &_Py_ID(bit_size), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -37,14 +36,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "type", "size", "offset", "index", "bit_size", "swapped_bytes", "_ms", "pack", "padding", "format", NULL}; + static const char * const _keywords[] = {"name", "type", "size", "offset", "index", "bit_size", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "CField", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[11]; + PyObject *argsbuf[6]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 5; @@ -54,13 +53,8 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t offset; Py_ssize_t index; PyObject *bit_size_obj = Py_None; - int swapped_bytes = 0; - int _ms = 0; - PyObject *pack_obj = Py_None; - Py_ssize_t padding = 0; - PyObject *format = Py_None; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 5, 11, 0, argsbuf); + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 5, 6, 0, argsbuf); if (!fastargs) { goto exit; } @@ -109,58 +103,11 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (!noptargs) { goto skip_optional_pos; } - if (fastargs[5]) { - bit_size_obj = fastargs[5]; - if (!--noptargs) { - goto skip_optional_pos; - } - } - if (fastargs[6]) { - swapped_bytes = PyObject_IsTrue(fastargs[6]); - if (swapped_bytes < 0) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_pos; - } - } - if (fastargs[7]) { - _ms = PyObject_IsTrue(fastargs[7]); - if (_ms < 0) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_pos; - } - } - if (fastargs[8]) { - pack_obj = fastargs[8]; - if (!--noptargs) { - goto skip_optional_pos; - } - } - if (fastargs[9]) { - { - Py_ssize_t ival = -1; - PyObject *iobj = _PyNumber_Index(fastargs[9]); - if (iobj != NULL) { - ival = PyLong_AsSsize_t(iobj); - Py_DECREF(iobj); - } - if (ival == -1 && PyErr_Occurred()) { - goto exit; - } - padding = ival; - } - if (!--noptargs) { - goto skip_optional_pos; - } - } - format = fastargs[10]; + bit_size_obj = fastargs[5]; skip_optional_pos: - return_value = PyCField_new_impl(type, name, proto, size, offset, index, bit_size_obj, swapped_bytes, _ms, pack_obj, padding, format); + return_value = PyCField_new_impl(type, name, proto, size, offset, index, bit_size_obj); exit: return return_value; } -/*[clinic end generated code: output=e851010bf9297fdc input=a9049054013a1b77]*/ +/*[clinic end generated code: output=27c010bae9be7213 input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 5e489332b1a362..dbe51c23176885 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -260,11 +260,6 @@ typedef struct CFieldObject { Py_ssize_t bit_size; /* -1 if not a bitfield */ PyObject *name; /* exact PyUnicode */ - bool big_endian; /* boolean */ - bool _ms_layout; - Py_ssize_t pack; /* 0 if undefined */ - Py_ssize_t padding; /* number of bytes between the end of the last field and the start of this one */ - PyObject *format; /* unicode */ } CFieldObject; /**************************************************************** From 7b3926ed2bfdf14a057466b4599ec6bf94f2e101 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 22:04:17 +0200 Subject: [PATCH 63/94] Unify the bit_size variable --- Lib/ctypes/_layout.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 414c3bc329c037..618dde89a17ebc 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -106,7 +106,7 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): try: name, ftype = field is_bitfield = False - bit_size = ctypes.sizeof(ftype) + bit_size = ctypes.sizeof(ftype) * 8 except ValueError: name, ftype, bit_size = field is_bitfield = True @@ -122,11 +122,6 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): state_bitofs += state_offset * 8; state_offset = 0 - if is_bitfield: - bitsize = bit_size - else: - bitsize = 8 * info_size; - ## We don't use packstate->offset here, so clear it, if it has been set. state_bitofs += state_offset * 8; state_offset = 0; @@ -142,7 +137,7 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): slot_start_bit = round_down(state_bitofs, 8 * info_align); slot_end_bit = slot_start_bit + 8 * info_size; ## And see if it also contains the bitfield's last bit: - field_end_bit = state_bitofs + bitsize; + field_end_bit = state_bitofs + bit_size; if field_end_bit > slot_end_bit: ## It doesn't: add padding (bump up to the next alignment boundary) state_bitofs = round_up(state_bitofs, 8 * info_align); @@ -152,19 +147,14 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): offset = int(round_down(state_bitofs, 8 * info_align) / 8); if is_bitfield: effective_bitsof = state_bitofs - 8 * offset; - size = BUILD_SIZE(bitsize, effective_bitsof); + size = BUILD_SIZE(bit_size, effective_bitsof); assert(effective_bitsof <= info_size * 8); else: size = info_size; - state_bitofs += bitsize; + state_bitofs += bit_size; state_size = int(round_up(state_bitofs, 8) / 8); else: - if is_bitfield: - bitsize = bit_size - else: - bitsize = 8 * info_size; - if _pack_: state_align = min(_pack_, info_align); else: @@ -174,7 +164,7 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): ## packstate->bitofs is generally non-positive, ## and 8 * packstate->offset + packstate->bitofs points just behind ## the end of the last field we placed. - if ((0 < state_bitofs + bitsize) or (8 * info_size != state_field_size)): + if ((0 < state_bitofs + bit_size) or (8 * info_size != state_field_size)): ## Close the previous bitfield (if any). ## and start a new bitfield: state_offset = round_up(state_offset, state_align); @@ -193,12 +183,12 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): if is_bitfield: assert(0 <= (state_field_size + state_bitofs)); assert((state_field_size + state_bitofs) < info_size * 8); - size = BUILD_SIZE(bitsize, state_field_size + state_bitofs); + size = BUILD_SIZE(bit_size, state_field_size + state_bitofs); else: size = info_size; assert(state_field_size + state_bitofs <= info_size * 8); - state_bitofs += bitsize; + state_bitofs += bit_size; state_size = state_offset; From 5f351e700cf2df2c0ec8d668579bd9b39e24ca69 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 22:33:26 +0200 Subject: [PATCH 64/94] Start simplifying the Python code --- Lib/ctypes/_layout.py | 105 +++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 53 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 618dde89a17ebc..ccd5668e46c98c 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -37,13 +37,26 @@ def BUILD_SIZE(bitsize, offset): assert(offset == LOW_BIT(result)); return result; +def build_size(bit_size, effective_bitsof, big_endian, type_size): + if big_endian: + return BUILD_SIZE(bit_size, 8 * type_size - effective_bitsof - bit_size) + return BUILD_SIZE(bit_size, effective_bitsof) + _INT_MAX = (1 << (ctypes.sizeof(ctypes.c_int) * 8) - 1) - 1 -class _BaseLayout: - def __init__(self, cls, fields, is_struct, base, **kwargs): - if kwargs: - warnings.warn(f'Unknown keyword arguments: {list(kwargs.keys())}') +class _structunion_layout: + """Compute the layout of a struct or union + + This is a callable that returns an object with attributes: + - fields: sequence of CField objects + - size: total size of the aggregate + - align: total alignment requirement of the aggregate + - format_spec: buffer format specification (as a string, UTF-8 but + best kept ASCII-only) + Technically this is a class, that might change. + """ + def __init__(self, cls, fields, is_struct, base, _ms): align = getattr(cls, '_align_', 1) if align < 0: raise ValueError('_align_ must be a non-negative integer') @@ -52,9 +65,7 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): align == 1 if base: - total_align = max(1, ctypes.alignment(base), align) - else: - total_align = align + align = max(ctypes.alignment(base), align) swapped_bytes = hasattr(cls, '_swappedbytes_') if swapped_bytes: @@ -72,6 +83,8 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): raise ValueError("_pack_ must be a non-negative integer") if _pack_ > _INT_MAX: raise ValueError("_pack_ too big") + if not _ms: + raise ValueError('_pack_ is not compatible with gcc-sysv layout') self.fields = [] @@ -93,14 +106,14 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): union_size = 0 last_size = state_size + last_field = None for i, field in enumerate(fields): if not is_struct: - if isinstance(self, GCCSysVLayout): - state_field_size = 0 - state_bitofs = 0 - state_offset = 0 - state_size = 0 - state_align = 0 + state_field_size = 0 + state_bitofs = 0 + state_offset = 0 + state_size = 0 + state_align = 0 field = tuple(field) try: @@ -114,10 +127,10 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): raise ValueError(f'number of bits invalid for bit field {name!r}') size = ctypes.sizeof(ftype) - info_size = ctypes.sizeof(ftype) + type_size = ctypes.sizeof(ftype) info_align = ctypes.alignment(ftype) - if isinstance(self, GCCSysVLayout): + if not _ms: # We don't use packstate->offset here, so clear it, if it has been set. state_bitofs += state_offset * 8; state_offset = 0 @@ -135,7 +148,7 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): ## That is: determine a "slot", sized & aligned for the specified type, ## which contains the bitfield's beginning: slot_start_bit = round_down(state_bitofs, 8 * info_align); - slot_end_bit = slot_start_bit + 8 * info_size; + slot_end_bit = slot_start_bit + 8 * type_size; ## And see if it also contains the bitfield's last bit: field_end_bit = state_bitofs + bit_size; if field_end_bit > slot_end_bit: @@ -147,10 +160,11 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): offset = int(round_down(state_bitofs, 8 * info_align) / 8); if is_bitfield: effective_bitsof = state_bitofs - 8 * offset; - size = BUILD_SIZE(bit_size, effective_bitsof); - assert(effective_bitsof <= info_size * 8); + size = build_size(bit_size, effective_bitsof, + big_endian, type_size) + assert(effective_bitsof <= type_size * 8); else: - size = info_size; + size = type_size; state_bitofs += bit_size; state_size = int(round_up(state_bitofs, 8) / 8); @@ -164,37 +178,36 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): ## packstate->bitofs is generally non-positive, ## and 8 * packstate->offset + packstate->bitofs points just behind ## the end of the last field we placed. - if ((0 < state_bitofs + bit_size) or (8 * info_size != state_field_size)): + if ((0 < state_bitofs + bit_size) or (8 * type_size != state_field_size)): ## Close the previous bitfield (if any). ## and start a new bitfield: state_offset = round_up(state_offset, state_align); - state_offset += info_size; + state_offset += type_size; - state_field_size = info_size * 8; + state_field_size = type_size * 8; ## Reminder: 8 * (packstate->offset) + bitofs points to where we would start a ## new field. Ie just behind where we placed the last field plus an ## allowance for alignment. state_bitofs = - state_field_size; - assert(8 * info_size == state_field_size); + assert(8 * type_size == state_field_size); offset = state_offset - int((state_field_size) // 8); if is_bitfield: assert(0 <= (state_field_size + state_bitofs)); - assert((state_field_size + state_bitofs) < info_size * 8); - size = BUILD_SIZE(bit_size, state_field_size + state_bitofs); + assert((state_field_size + state_bitofs) < type_size * 8); + size = build_size(bit_size, state_field_size + state_bitofs, + big_endian, type_size) else: - size = info_size; - assert(state_field_size + state_bitofs <= info_size * 8); + size = type_size; + assert(state_field_size + state_bitofs <= type_size * 8); state_bitofs += bit_size; state_size = state_offset; assert((not is_bitfield) or (LOW_BIT(size) <= size * 8)); - if big_endian and is_bitfield: - size = BUILD_SIZE(NUM_BITS(size), 8*info_size - LOW_BIT(size) - bit_size); padding = offset - last_size if is_struct: @@ -209,15 +222,16 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): buf = f"({shape_numbers}){buf}" format_spec += buf - self.fields.append(CField( + last_field = CField( name=name, type=ftype, size=size, offset=offset, bit_size=bit_size if is_bitfield else None, index=i, - )) - total_align = max(total_align, state_align) + ) + self.fields.append(last_field) + align = max(align, state_align) last_size = state_size union_size = max(state_size, union_size); @@ -227,14 +241,14 @@ def __init__(self, cls, fields, is_struct, base, **kwargs): total_size = union_size # Adjust the size according to the alignment requirements - aligned_size = int((total_size + total_align - 1) / total_align) * total_align + aligned_size = int((total_size + align - 1) / align) * align if is_struct: padding = aligned_size - total_size format_spec += f"{padding_spec(padding)}}}" self.size = aligned_size - self.align = total_align + self.align = align self.format_spec = format_spec def padding_spec(padding): @@ -245,30 +259,15 @@ def padding_spec(padding): return f"{padding}x" -class WindowsLayout(_BaseLayout): - pass - -class GCCSysVLayout(_BaseLayout): - def __init__(self, cls, *args, **kwargs): - if getattr(cls, '_pack_', None): - raise ValueError('_pack_ is not compatible with gcc-sysv layout') - return super().__init__(cls, *args, **kwargs) - -if sys.platform == 'win32': - NativeLayout = WindowsLayout - DefaultLayout = WindowsLayout -else: - NativeLayout = GCCSysVLayout - def default_layout(cls, *args, **kwargs): layout = getattr(cls, '_layout_', None) if layout is None: if sys.platform == 'win32' or getattr(cls, '_pack_', None): - return WindowsLayout(cls, *args, **kwargs) - return GCCSysVLayout(cls, *args, **kwargs) + return _structunion_layout(cls, *args, **kwargs, _ms=True) + return _structunion_layout(cls, *args, **kwargs, _ms=False) elif layout == 'ms': - return WindowsLayout(cls, *args, **kwargs) + return _structunion_layout(cls, *args, **kwargs, _ms=True) elif layout == 'gcc-sysv': - return GCCSysVLayout(cls, *args, **kwargs) + return _structunion_layout(cls, *args, **kwargs, _ms=False) else: raise ValueError(f'unknown _layout_: {layout!r}') From d2f0da31da06935d7c2de0e710cc06b4542ba53e Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 22:39:42 +0200 Subject: [PATCH 65/94] Remove state_align --- Lib/ctypes/_layout.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index ccd5668e46c98c..915b70b4ee4902 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -98,22 +98,19 @@ def __init__(self, cls, fields, is_struct, base, _ms): state_bitofs = 0 state_offset = 0 state_size = 0 - state_align = 0 - #print(self.base, self.is_struct, self.size, self.size * 8) - if base and is_struct: + if base: state_size = state_offset = ctypes.sizeof(base) - state_align = ctypes.alignment(base) union_size = 0 last_size = state_size last_field = None for i, field in enumerate(fields): if not is_struct: + # Unions start fresh each time state_field_size = 0 state_bitofs = 0 state_offset = 0 state_size = 0 - state_align = 0 field = tuple(field) try: @@ -125,10 +122,9 @@ def __init__(self, cls, fields, is_struct, base, _ms): is_bitfield = True if bit_size <= 0: raise ValueError(f'number of bits invalid for bit field {name!r}') - size = ctypes.sizeof(ftype) type_size = ctypes.sizeof(ftype) - info_align = ctypes.alignment(ftype) + type_align = ctypes.alignment(ftype) if not _ms: # We don't use packstate->offset here, so clear it, if it has been set. @@ -141,23 +137,21 @@ def __init__(self, cls, fields, is_struct, base, _ms): assert(_pack_ in (0, None)); ## TODO: This shouldn't be a C assertion - state_align = info_align; - ## Determine whether the bit field, if placed at the next free bit, ## fits within a single object of its specified type. ## That is: determine a "slot", sized & aligned for the specified type, ## which contains the bitfield's beginning: - slot_start_bit = round_down(state_bitofs, 8 * info_align); + slot_start_bit = round_down(state_bitofs, 8 * type_align); slot_end_bit = slot_start_bit + 8 * type_size; ## And see if it also contains the bitfield's last bit: field_end_bit = state_bitofs + bit_size; if field_end_bit > slot_end_bit: ## It doesn't: add padding (bump up to the next alignment boundary) - state_bitofs = round_up(state_bitofs, 8 * info_align); + state_bitofs = round_up(state_bitofs, 8 * type_align); assert(state_offset == 0); - offset = int(round_down(state_bitofs, 8 * info_align) / 8); + offset = int(round_down(state_bitofs, 8 * type_align) / 8); if is_bitfield: effective_bitsof = state_bitofs - 8 * offset; size = build_size(bit_size, effective_bitsof, @@ -170,9 +164,7 @@ def __init__(self, cls, fields, is_struct, base, _ms): state_size = int(round_up(state_bitofs, 8) / 8); else: if _pack_: - state_align = min(_pack_, info_align); - else: - state_align = info_align; + type_align = min(_pack_, type_align); ## packstate->offset points to end of current bitfield. ## packstate->bitofs is generally non-positive, @@ -181,7 +173,7 @@ def __init__(self, cls, fields, is_struct, base, _ms): if ((0 < state_bitofs + bit_size) or (8 * type_size != state_field_size)): ## Close the previous bitfield (if any). ## and start a new bitfield: - state_offset = round_up(state_offset, state_align); + state_offset = round_up(state_offset, type_align); state_offset += type_size; @@ -231,7 +223,7 @@ def __init__(self, cls, fields, is_struct, base, _ms): index=i, ) self.fields.append(last_field) - align = max(align, state_align) + align = max(align, type_align) last_size = state_size union_size = max(state_size, union_size); From 4b346d4a31b90b0f37b22db135bf12ebb44886d7 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 22:51:14 +0200 Subject: [PATCH 66/94] Make it a bit clearer yet --- Lib/ctypes/_layout.py | 46 +++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 915b70b4ee4902..0fe08583f7f11f 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -89,20 +89,20 @@ def __init__(self, cls, fields, is_struct, base, _ms): self.fields = [] if is_struct: - format_spec = "T{" + format_spec_parts = ["T{"] else: - format_spec = "B" + format_spec_parts = ["B"] state_field_size = 0 # `8 * offset + bitofs` points to where the next field would start. state_bitofs = 0 state_offset = 0 - state_size = 0 + struct_size = 0 # size if this was a struct if base: - state_size = state_offset = ctypes.sizeof(base) + struct_size = state_offset = ctypes.sizeof(base) union_size = 0 - last_size = state_size + last_size = struct_size last_field = None for i, field in enumerate(fields): if not is_struct: @@ -110,7 +110,6 @@ def __init__(self, cls, fields, is_struct, base, _ms): state_field_size = 0 state_bitofs = 0 state_offset = 0 - state_size = 0 field = tuple(field) try: @@ -161,7 +160,7 @@ def __init__(self, cls, fields, is_struct, base, _ms): size = type_size; state_bitofs += bit_size; - state_size = int(round_up(state_bitofs, 8) / 8); + struct_size = int(round_up(state_bitofs, 8) / 8); else: if _pack_: type_align = min(_pack_, type_align); @@ -196,23 +195,27 @@ def __init__(self, cls, fields, is_struct, base, _ms): assert(state_field_size + state_bitofs <= type_size * 8); state_bitofs += bit_size; - state_size = state_offset; + struct_size = state_offset; assert((not is_bitfield) or (LOW_BIT(size) <= size * 8)); - padding = offset - last_size if is_struct: - format_spec += padding_spec(padding) + padding = offset - last_size + format_spec_parts.append(padding_spec(padding)) fieldfmt, bf_ndim, bf_shape = buffer_info(ftype) + + if bf_shape: + format_spec_parts.extend(( + "(", + ','.join(str(n) for n in bf_shape), + ")", + )) + if fieldfmt is None: fieldfmt = "B" - buf = f"{fieldfmt}:{name}:" - if bf_shape: - shape_numbers = ",".join(str(n) for n in bf_shape) - buf = f"({shape_numbers}){buf}" - format_spec += buf + format_spec_parts.append(f"{fieldfmt}:{name}:") last_field = CField( name=name, @@ -224,24 +227,25 @@ def __init__(self, cls, fields, is_struct, base, _ms): ) self.fields.append(last_field) align = max(align, type_align) - last_size = state_size - union_size = max(state_size, union_size); + last_size = struct_size + union_size = max(struct_size, union_size); if is_struct: - total_size = state_size + total_size = struct_size else: total_size = union_size # Adjust the size according to the alignment requirements - aligned_size = int((total_size + align - 1) / align) * align + aligned_size = round_up(total_size, align) if is_struct: padding = aligned_size - total_size - format_spec += f"{padding_spec(padding)}}}" + format_spec_parts.append(padding_spec(padding)) + format_spec_parts.append("}") self.size = aligned_size self.align = align - self.format_spec = format_spec + self.format_spec = "".join(format_spec_parts) def padding_spec(padding): if padding <= 0: From 95d9c54e3b40a4296b254105643258e16902819e Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 23:21:22 +0200 Subject: [PATCH 67/94] More cleanup --- Lib/ctypes/_layout.py | 224 ++++++++++++++++++++++-------------------- 1 file changed, 118 insertions(+), 106 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 0fe08583f7f11f..15f3d1ac34bad4 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -6,41 +6,37 @@ import ctypes def round_down(n, multiple): - assert(n >= 0); - assert(multiple >= 0); - if (multiple == 0): - return n; - return int(n / multiple) * multiple; + assert n >= 0 + assert multiple > 0 + return (n // multiple) * multiple def round_up(n, multiple): - assert(n >= 0); - assert(multiple >= 0); - if (multiple == 0): - return n; - return int((n + multiple - 1) / multiple) * multiple; + assert n >= 0 + assert multiple > 0 + return ((n + multiple - 1) // multiple) * multiple def LOW_BIT(offset): - return offset & 0xFFFF; + return offset & 0xFFFF def NUM_BITS(bitsize): - return bitsize >> 16; + return bitsize >> 16 def BUILD_SIZE(bitsize, offset): - assert(0 <= offset); - assert(offset <= 0xFFFF); + assert(0 <= offset) + assert(offset <= 0xFFFF) ## We don't support zero length bitfields. ## And GET_BITFIELD uses NUM_BITS(size)==0, ## to figure out whether we are handling a bitfield. - assert(0 < bitsize); - result = (bitsize << 16) + offset; - assert(bitsize == NUM_BITS(result)); - assert(offset == LOW_BIT(result)); - return result; + assert(0 < bitsize) + result = (bitsize << 16) + offset + assert(bitsize == NUM_BITS(result)) + assert(offset == LOW_BIT(result)) + return result -def build_size(bit_size, effective_bitsof, big_endian, type_size): +def build_size(bit_size, bit_offset, big_endian, type_size): if big_endian: - return BUILD_SIZE(bit_size, 8 * type_size - effective_bitsof - bit_size) - return BUILD_SIZE(bit_size, effective_bitsof) + return BUILD_SIZE(bit_size, 8 * type_size - bit_offset - bit_size) + return BUILD_SIZE(bit_size, bit_offset) _INT_MAX = (1 << (ctypes.sizeof(ctypes.c_int) * 8) - 1) - 1 @@ -73,15 +69,15 @@ def __init__(self, cls, fields, is_struct, base, _ms): else: big_endian = sys.byteorder == 'big' - _pack_ = getattr(cls, '_pack_', None) - if _pack_ is not None: + pack = getattr(cls, '_pack_', None) + if pack is not None: try: - _pack_ = int(_pack_) + pack = int(pack) except (TypeError, ValueError): raise ValueError("_pack_ must be an integer") - if _pack_ < 0: + if pack < 0: raise ValueError("_pack_ must be a non-negative integer") - if _pack_ > _INT_MAX: + if pack > _INT_MAX: raise ValueError("_pack_ too big") if not _ms: raise ValueError('_pack_ is not compatible with gcc-sysv layout') @@ -93,118 +89,131 @@ def __init__(self, cls, fields, is_struct, base, _ms): else: format_spec_parts = ["B"] - state_field_size = 0 - # `8 * offset + bitofs` points to where the next field would start. - state_bitofs = 0 - state_offset = 0 - struct_size = 0 # size if this was a struct - if base: - struct_size = state_offset = ctypes.sizeof(base) + last_field_bit_size = 0 # used in MS layout only + + # `8 * next_byte_offset + next_bit_offset` points to where the + # next field would start. + next_bit_offset = 0 + next_byte_offset = 0 + # size if this was a struct (sum of field sizes, plus padding) + struct_size = 0 + # max of field sizes; only meaningful for unions union_size = 0 + + if base: + struct_size = ctypes.sizeof(base) + if _ms: + next_byte_offset = struct_size + else: + next_bit_offset = struct_size * 8 + last_size = struct_size last_field = None for i, field in enumerate(fields): if not is_struct: # Unions start fresh each time - state_field_size = 0 - state_bitofs = 0 - state_offset = 0 + last_field_bit_size = 0 + next_bit_offset = 0 + next_byte_offset = 0 + # Unpack the field field = tuple(field) try: - name, ftype = field + name, ctype = field is_bitfield = False - bit_size = ctypes.sizeof(ftype) * 8 + type_size = ctypes.sizeof(ctype) + bit_size = type_size * 8 except ValueError: - name, ftype, bit_size = field + name, ctype, bit_size = field is_bitfield = True if bit_size <= 0: raise ValueError(f'number of bits invalid for bit field {name!r}') + type_size = ctypes.sizeof(ctype) - type_size = ctypes.sizeof(ftype) - type_align = ctypes.alignment(ftype) + type_bit_size = type_size * 8 + type_align = ctypes.alignment(ctype) or 1 + type_bit_align = type_align * 8 if not _ms: - # We don't use packstate->offset here, so clear it, if it has been set. - state_bitofs += state_offset * 8; - state_offset = 0 - - ## We don't use packstate->offset here, so clear it, if it has been set. - state_bitofs += state_offset * 8; - state_offset = 0; - - assert(_pack_ in (0, None)); ## TODO: This shouldn't be a C assertion - - ## Determine whether the bit field, if placed at the next free bit, - ## fits within a single object of its specified type. - ## That is: determine a "slot", sized & aligned for the specified type, - ## which contains the bitfield's beginning: - slot_start_bit = round_down(state_bitofs, 8 * type_align); - slot_end_bit = slot_start_bit + 8 * type_size; - ## And see if it also contains the bitfield's last bit: - field_end_bit = state_bitofs + bit_size; + # We don't use next_byte_offset here + assert pack is None + assert next_byte_offset == 0 + + # Determine whether the bit field, if placed at the next + # free bit, fits within a single object of its specified type. + # That is: determine a "slot", sized & aligned for the + # specified type, which contains the bitfield's beginning: + slot_start_bit = round_down(next_bit_offset, type_bit_align) + slot_end_bit = slot_start_bit + type_bit_size + # And see if it also contains the bitfield's last bit: + field_end_bit = next_bit_offset + bit_size if field_end_bit > slot_end_bit: - ## It doesn't: add padding (bump up to the next alignment boundary) - state_bitofs = round_up(state_bitofs, 8 * type_align); + # It doesn't: add padding (bump up to the next + # alignment boundary) + next_bit_offset = round_up(next_bit_offset, type_bit_align) - assert(state_offset == 0); - - offset = int(round_down(state_bitofs, 8 * type_align) / 8); + offset = int(round_down(next_bit_offset, type_bit_align) / 8) if is_bitfield: - effective_bitsof = state_bitofs - 8 * offset; - size = build_size(bit_size, effective_bitsof, + effective_bit_offset = next_bit_offset - 8 * offset + size = build_size(bit_size, effective_bit_offset, big_endian, type_size) - assert(effective_bitsof <= type_size * 8); + assert effective_bit_offset <= type_bit_size else: - size = type_size; + assert offset == next_bit_offset / 8 + size = type_size - state_bitofs += bit_size; - struct_size = int(round_up(state_bitofs, 8) / 8); + next_bit_offset += bit_size + struct_size = int(round_up(next_bit_offset, 8) / 8) else: - if _pack_: - type_align = min(_pack_, type_align); - - ## packstate->offset points to end of current bitfield. - ## packstate->bitofs is generally non-positive, - ## and 8 * packstate->offset + packstate->bitofs points just behind - ## the end of the last field we placed. - if ((0 < state_bitofs + bit_size) or (8 * type_size != state_field_size)): - ## Close the previous bitfield (if any). - ## and start a new bitfield: - state_offset = round_up(state_offset, type_align); - - state_offset += type_size; - - state_field_size = type_size * 8; - ## Reminder: 8 * (packstate->offset) + bitofs points to where we would start a - ## new field. Ie just behind where we placed the last field plus an - ## allowance for alignment. - state_bitofs = - state_field_size; - - assert(8 * type_size == state_field_size); - - offset = state_offset - int((state_field_size) // 8); + if pack: + type_align = min(pack, type_align) + + # next_byte_offset points to end of current bitfield. + # next_bit_offset is generally non-positive, + # and 8 * next_byte_offset + next_bit_offset points just behind + # the end of the last field we placed. + if ( + (0 < next_bit_offset + bit_size) + or (type_bit_size != last_field_bit_size) + ): + # Close the previous bitfield (if any) + # and start a new bitfield + next_byte_offset = round_up(next_byte_offset, type_align) + + next_byte_offset += type_size + + last_field_bit_size = type_bit_size + # Reminder: 8 * (next_byte_offset) + next_bit_offset + # points to where we would start a + # new field. I.e. just behind where we placed the last + # field plus an allowance for alignment. + next_bit_offset = - last_field_bit_size + + assert type_bit_size == last_field_bit_size + assert type_bit_size > 0 + + offset = next_byte_offset - last_field_bit_size // 8 if is_bitfield: - assert(0 <= (state_field_size + state_bitofs)); - assert((state_field_size + state_bitofs) < type_size * 8); - size = build_size(bit_size, state_field_size + state_bitofs, + assert 0 <= (last_field_bit_size + next_bit_offset) + size = build_size(bit_size, + last_field_bit_size + next_bit_offset, big_endian, type_size) else: - size = type_size; - assert(state_field_size + state_bitofs <= type_size * 8); + size = type_size + assert (last_field_bit_size + next_bit_offset) < type_bit_size - state_bitofs += bit_size; - struct_size = state_offset; + next_bit_offset += bit_size + struct_size = next_byte_offset + assert((not is_bitfield) or (LOW_BIT(size) <= size * 8)) - assert((not is_bitfield) or (LOW_BIT(size) <= size * 8)); - + # Add the format spec parts if is_struct: padding = offset - last_size format_spec_parts.append(padding_spec(padding)) - fieldfmt, bf_ndim, bf_shape = buffer_info(ftype) + fieldfmt, bf_ndim, bf_shape = buffer_info(ctype) if bf_shape: format_spec_parts.extend(( @@ -219,7 +228,7 @@ def __init__(self, cls, fields, is_struct, base, _ms): last_field = CField( name=name, - type=ftype, + type=ctype, size=size, offset=offset, bit_size=bit_size if is_bitfield else None, @@ -228,7 +237,8 @@ def __init__(self, cls, fields, is_struct, base, _ms): self.fields.append(last_field) align = max(align, type_align) last_size = struct_size - union_size = max(struct_size, union_size); + if not is_struct: + union_size = max(struct_size, union_size) if is_struct: total_size = struct_size @@ -238,6 +248,7 @@ def __init__(self, cls, fields, is_struct, base, _ms): # Adjust the size according to the alignment requirements aligned_size = round_up(total_size, align) + # Finish up the format spec if is_struct: padding = aligned_size - total_size format_spec_parts.append(padding_spec(padding)) @@ -247,6 +258,7 @@ def __init__(self, cls, fields, is_struct, base, _ms): self.align = align self.format_spec = "".join(format_spec_parts) + def padding_spec(padding): if padding <= 0: return "" From af5c52bab15d8bec5500e80392415ccb64d2cff3 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 23:24:00 +0200 Subject: [PATCH 68/94] Simplify C call using borrowed references --- Modules/_ctypes/stgdict.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 35ce23ff38b2f4..b5de6cefe8e5c1 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -280,7 +280,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (!kwnames) { goto error; } - PyObject *base_arg = Py_NewRef(baseinfo ? base : Py_None); layout = PyObject_Vectorcall( layout_class, 1 + (PyObject*[]){ @@ -289,13 +288,11 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct type, fields, /* keyword args */ - Py_GetConstantBorrowed( - isStruct ? Py_CONSTANT_TRUE : Py_CONSTANT_FALSE), - base_arg}, + isStruct ? Py_True : Py_False, + baseinfo ? base : Py_None}, 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); Py_DECREF(kwnames); - Py_DECREF(base_arg); Py_DECREF(layout_class); fields = NULL; // a borrowed reference we won't be using again if (!layout) { From 51a4042a10c3f0e09d00b5769502b9c7d68eede3 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 23 Aug 2024 23:29:32 +0200 Subject: [PATCH 69/94] Use _Py_ID for the literals --- .../pycore_global_objects_fini_generated.h | 6 ++-- Include/internal/pycore_global_strings.h | 6 ++-- .../internal/pycore_runtime_init_generated.h | 6 ++-- .../internal/pycore_unicodeobject_generated.h | 24 +++++++++----- Modules/_ctypes/stgdict.c | 32 +++++++++---------- 5 files changed, 44 insertions(+), 30 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 19f51029827232..0e6064082ef566 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -765,14 +765,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_loop)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_needs_com_addref_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_only_immortal)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_pack_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_restype_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_showwarnmsg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_shutdown)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_slotnames)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_strptime_datetime)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_swappedbytes_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_type_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_uninitialized_submodules)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_warn_unawaited_coroutine)); @@ -806,6 +804,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(before)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(big)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(binary_form)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bit_size)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(block)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bound)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(buffer)); @@ -934,6 +933,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fd2)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fdel)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fget)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fields)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(file)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(file_actions)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(filename)); @@ -950,6 +950,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fold)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(follow_symlinks)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format_spec)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(from_param)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromlist)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromtimestamp)); @@ -986,6 +987,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(importlib)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(in_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(incoming)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(index)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(indexgroup)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inf)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(infer_variance)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 1bf5bdaf488b7a..251bd1c8561fe2 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -254,14 +254,12 @@ struct _Py_global_strings { STRUCT_FOR_ID(_loop) STRUCT_FOR_ID(_needs_com_addref_) STRUCT_FOR_ID(_only_immortal) - STRUCT_FOR_ID(_pack_) STRUCT_FOR_ID(_restype_) STRUCT_FOR_ID(_showwarnmsg) STRUCT_FOR_ID(_shutdown) STRUCT_FOR_ID(_slotnames) STRUCT_FOR_ID(_strptime) STRUCT_FOR_ID(_strptime_datetime) - STRUCT_FOR_ID(_swappedbytes_) STRUCT_FOR_ID(_type_) STRUCT_FOR_ID(_uninitialized_submodules) STRUCT_FOR_ID(_warn_unawaited_coroutine) @@ -295,6 +293,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(before) STRUCT_FOR_ID(big) STRUCT_FOR_ID(binary_form) + STRUCT_FOR_ID(bit_size) STRUCT_FOR_ID(block) STRUCT_FOR_ID(bound) STRUCT_FOR_ID(buffer) @@ -423,6 +422,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(fd2) STRUCT_FOR_ID(fdel) STRUCT_FOR_ID(fget) + STRUCT_FOR_ID(fields) STRUCT_FOR_ID(file) STRUCT_FOR_ID(file_actions) STRUCT_FOR_ID(filename) @@ -439,6 +439,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(fold) STRUCT_FOR_ID(follow_symlinks) STRUCT_FOR_ID(format) + STRUCT_FOR_ID(format_spec) STRUCT_FOR_ID(from_param) STRUCT_FOR_ID(fromlist) STRUCT_FOR_ID(fromtimestamp) @@ -475,6 +476,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(importlib) STRUCT_FOR_ID(in_fd) STRUCT_FOR_ID(incoming) + STRUCT_FOR_ID(index) STRUCT_FOR_ID(indexgroup) STRUCT_FOR_ID(inf) STRUCT_FOR_ID(infer_variance) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 1a1c8408623fb1..7094bb958b5c86 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -763,14 +763,12 @@ extern "C" { INIT_ID(_loop), \ INIT_ID(_needs_com_addref_), \ INIT_ID(_only_immortal), \ - INIT_ID(_pack_), \ INIT_ID(_restype_), \ INIT_ID(_showwarnmsg), \ INIT_ID(_shutdown), \ INIT_ID(_slotnames), \ INIT_ID(_strptime), \ INIT_ID(_strptime_datetime), \ - INIT_ID(_swappedbytes_), \ INIT_ID(_type_), \ INIT_ID(_uninitialized_submodules), \ INIT_ID(_warn_unawaited_coroutine), \ @@ -804,6 +802,7 @@ extern "C" { INIT_ID(before), \ INIT_ID(big), \ INIT_ID(binary_form), \ + INIT_ID(bit_size), \ INIT_ID(block), \ INIT_ID(bound), \ INIT_ID(buffer), \ @@ -932,6 +931,7 @@ extern "C" { INIT_ID(fd2), \ INIT_ID(fdel), \ INIT_ID(fget), \ + INIT_ID(fields), \ INIT_ID(file), \ INIT_ID(file_actions), \ INIT_ID(filename), \ @@ -948,6 +948,7 @@ extern "C" { INIT_ID(fold), \ INIT_ID(follow_symlinks), \ INIT_ID(format), \ + INIT_ID(format_spec), \ INIT_ID(from_param), \ INIT_ID(fromlist), \ INIT_ID(fromtimestamp), \ @@ -984,6 +985,7 @@ extern "C" { INIT_ID(importlib), \ INIT_ID(in_fd), \ INIT_ID(incoming), \ + INIT_ID(index), \ INIT_ID(indexgroup), \ INIT_ID(inf), \ INIT_ID(infer_variance), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 839446c9bc7331..febf2862985dab 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -816,10 +816,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(_pack_); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_restype_); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -844,10 +840,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(_swappedbytes_); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_type_); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -980,6 +972,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(bit_size); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(block); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1492,6 +1488,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(fields); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(file); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1556,6 +1556,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(format_spec); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(from_param); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1700,6 +1704,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(index); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(indexgroup); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index b5de6cefe8e5c1..a8ff1dc881ffe7 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -299,7 +299,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; } - tmp = PyObject_GetAttrString(layout, "align"); + tmp = PyObject_GetAttr(layout, &_Py_ID(align)); if (!tmp) { goto error; } @@ -308,12 +308,12 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (total_align < 0) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_ValueError, - "total_align must be a non-negative integer"); + "align must be a non-negative integer"); } goto error; } - tmp = PyObject_GetAttrString(layout, "size"); + tmp = PyObject_GetAttr(layout, &_Py_ID(size)); if (!tmp) { goto error; } @@ -327,7 +327,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; } - format_spec_obj = PyObject_GetAttrString(layout, "format_spec"); + format_spec_obj = PyObject_GetAttr(layout, &_Py_ID(format_spec)); if (!format_spec_obj) { goto error; } @@ -338,7 +338,18 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; } - PyObject *layout_fields_obj = PyObject_GetAttrString(layout, "fields"); + if (stginfo->format) { + PyMem_Free(stginfo->format); + stginfo->format = NULL; + } + stginfo->format = PyMem_Malloc(format_spec_size + 1); + if (!stginfo->format) { + PyErr_NoMemory(); + goto error; + } + memcpy(stginfo->format, format_spec, format_spec_size + 1); + + PyObject *layout_fields_obj = PyObject_GetAttr(layout, &_Py_ID(fields)); if (!layout_fields_obj) { goto error; } @@ -351,17 +362,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct Py_ssize_t len = PyTuple_GET_SIZE(layout_fields); - if (stginfo->format) { - PyMem_Free(stginfo->format); - stginfo->format = NULL; - } - stginfo->format = PyMem_Malloc(format_spec_size + 1); - if (!stginfo->format) { - PyErr_NoMemory(); - goto error; - } - memcpy(stginfo->format, format_spec, format_spec_size + 1); - if (stginfo->ffi_type_pointer.elements) { PyMem_Free(stginfo->ffi_type_pointer.elements); } From 5a805cd4c4b4b58a0c349d7d8153eedf4f94b3d0 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 26 Aug 2024 16:10:54 +0200 Subject: [PATCH 70/94] Use floordiv, not int(truediv), for non-negative numbers --- Lib/ctypes/_layout.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 15f3d1ac34bad4..af97dfe2862e28 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -153,7 +153,7 @@ def __init__(self, cls, fields, is_struct, base, _ms): # alignment boundary) next_bit_offset = round_up(next_bit_offset, type_bit_align) - offset = int(round_down(next_bit_offset, type_bit_align) / 8) + offset = round_down(next_bit_offset, type_bit_align) // 8 if is_bitfield: effective_bit_offset = next_bit_offset - 8 * offset size = build_size(bit_size, effective_bit_offset, @@ -164,7 +164,7 @@ def __init__(self, cls, fields, is_struct, base, _ms): size = type_size next_bit_offset += bit_size - struct_size = int(round_up(next_bit_offset, 8) / 8) + struct_size = round_up(next_bit_offset, 8) // 8 else: if pack: type_align = min(pack, type_align) From 12fd84aa15050f6a4028d2cadc4e1f51df87a136 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 26 Aug 2024 16:00:44 +0200 Subject: [PATCH 71/94] Use a data class + function --- Lib/ctypes/_layout.py | 439 +++++++++++++++++++------------------- Modules/_ctypes/stgdict.c | 20 +- 2 files changed, 228 insertions(+), 231 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index af97dfe2862e28..7e2667b2752cd8 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -40,223 +40,244 @@ def build_size(bit_size, bit_offset, big_endian, type_size): _INT_MAX = (1 << (ctypes.sizeof(ctypes.c_int) * 8) - 1) - 1 -class _structunion_layout: - """Compute the layout of a struct or union - - This is a callable that returns an object with attributes: - - fields: sequence of CField objects - - size: total size of the aggregate - - align: total alignment requirement of the aggregate - - format_spec: buffer format specification (as a string, UTF-8 but - best kept ASCII-only) - - Technically this is a class, that might change. - """ - def __init__(self, cls, fields, is_struct, base, _ms): - align = getattr(cls, '_align_', 1) - if align < 0: - raise ValueError('_align_ must be a non-negative integer') - elif align == 0: - # Setting `_align_ = 0` amounts to using the default alignment - align == 1 - - if base: - align = max(ctypes.alignment(base), align) - - swapped_bytes = hasattr(cls, '_swappedbytes_') - if swapped_bytes: - big_endian = sys.byteorder == 'little' - else: - big_endian = sys.byteorder == 'big' - - pack = getattr(cls, '_pack_', None) - if pack is not None: - try: - pack = int(pack) - except (TypeError, ValueError): - raise ValueError("_pack_ must be an integer") - if pack < 0: - raise ValueError("_pack_ must be a non-negative integer") - if pack > _INT_MAX: - raise ValueError("_pack_ too big") - if not _ms: - raise ValueError('_pack_ is not compatible with gcc-sysv layout') - - self.fields = [] - if is_struct: - format_spec_parts = ["T{"] +class StructUnionLayout: + def __init__(self, fields, size, align, format_spec): + # sequence of CField objects + self.fields = fields + + # total size of the aggregate (rounded up to alignment) + self.size = size + + # total alignment requirement of the aggregate + self.align = align + + # buffer format specification (as a string, UTF-8 but bes + # kept ASCII-only) + self.format_spec = format_spec + + +def get_layout(cls, input_fields, is_struct, base): + layout = getattr(cls, '_layout_', None) + if layout is None: + if sys.platform == 'win32' or getattr(cls, '_pack_', None): + gcc_layout = False else: - format_spec_parts = ["B"] + gcc_layout = True + elif layout == 'ms': + gcc_layout = False + elif layout == 'gcc-sysv': + gcc_layout = True + else: + raise ValueError(f'unknown _layout_: {layout!r}') + + align = getattr(cls, '_align_', 1) + if align < 0: + raise ValueError('_align_ must be a non-negative integer') + elif align == 0: + # Setting `_align_ = 0` amounts to using the default alignment + align == 1 - last_field_bit_size = 0 # used in MS layout only + if base: + align = max(ctypes.alignment(base), align) - # `8 * next_byte_offset + next_bit_offset` points to where the - # next field would start. - next_bit_offset = 0 - next_byte_offset = 0 + swapped_bytes = hasattr(cls, '_swappedbytes_') + if swapped_bytes: + big_endian = sys.byteorder == 'little' + else: + big_endian = sys.byteorder == 'big' + + pack = getattr(cls, '_pack_', None) + if pack is not None: + try: + pack = int(pack) + except (TypeError, ValueError): + raise ValueError("_pack_ must be an integer") + if pack < 0: + raise ValueError("_pack_ must be a non-negative integer") + if pack > _INT_MAX: + raise ValueError("_pack_ too big") + if gcc_layout: + raise ValueError('_pack_ is not compatible with gcc-sysv layout') + + result_fields = [] + + if is_struct: + format_spec_parts = ["T{"] + else: + format_spec_parts = ["B"] - # size if this was a struct (sum of field sizes, plus padding) - struct_size = 0 - # max of field sizes; only meaningful for unions - union_size = 0 + last_field_bit_size = 0 # used in MS layout only - if base: - struct_size = ctypes.sizeof(base) - if _ms: - next_byte_offset = struct_size - else: - next_bit_offset = struct_size * 8 + # `8 * next_byte_offset + next_bit_offset` points to where the + # next field would start. + next_bit_offset = 0 + next_byte_offset = 0 - last_size = struct_size - last_field = None - for i, field in enumerate(fields): - if not is_struct: - # Unions start fresh each time - last_field_bit_size = 0 - next_bit_offset = 0 - next_byte_offset = 0 - - # Unpack the field - field = tuple(field) - try: - name, ctype = field - is_bitfield = False - type_size = ctypes.sizeof(ctype) - bit_size = type_size * 8 - except ValueError: - name, ctype, bit_size = field - is_bitfield = True - if bit_size <= 0: - raise ValueError(f'number of bits invalid for bit field {name!r}') - type_size = ctypes.sizeof(ctype) - - type_bit_size = type_size * 8 - type_align = ctypes.alignment(ctype) or 1 - type_bit_align = type_align * 8 - - if not _ms: - # We don't use next_byte_offset here - assert pack is None - assert next_byte_offset == 0 - - # Determine whether the bit field, if placed at the next - # free bit, fits within a single object of its specified type. - # That is: determine a "slot", sized & aligned for the - # specified type, which contains the bitfield's beginning: - slot_start_bit = round_down(next_bit_offset, type_bit_align) - slot_end_bit = slot_start_bit + type_bit_size - # And see if it also contains the bitfield's last bit: - field_end_bit = next_bit_offset + bit_size - if field_end_bit > slot_end_bit: - # It doesn't: add padding (bump up to the next - # alignment boundary) - next_bit_offset = round_up(next_bit_offset, type_bit_align) - - offset = round_down(next_bit_offset, type_bit_align) // 8 - if is_bitfield: - effective_bit_offset = next_bit_offset - 8 * offset - size = build_size(bit_size, effective_bit_offset, - big_endian, type_size) - assert effective_bit_offset <= type_bit_size - else: - assert offset == next_bit_offset / 8 - size = type_size - - next_bit_offset += bit_size - struct_size = round_up(next_bit_offset, 8) // 8 + # size if this was a struct (sum of field sizes, plus padding) + struct_size = 0 + # max of field sizes; only meaningful for unions + union_size = 0 + + if base: + struct_size = ctypes.sizeof(base) + if gcc_layout: + next_bit_offset = struct_size * 8 + else: + next_byte_offset = struct_size + + last_size = struct_size + last_field = None + for i, field in enumerate(input_fields): + if not is_struct: + # Unions start fresh each time + last_field_bit_size = 0 + next_bit_offset = 0 + next_byte_offset = 0 + + # Unpack the field + field = tuple(field) + try: + name, ctype = field + is_bitfield = False + type_size = ctypes.sizeof(ctype) + bit_size = type_size * 8 + except ValueError: + name, ctype, bit_size = field + is_bitfield = True + if bit_size <= 0: + raise ValueError(f'number of bits invalid for bit field {name!r}') + type_size = ctypes.sizeof(ctype) + + type_bit_size = type_size * 8 + type_align = ctypes.alignment(ctype) or 1 + type_bit_align = type_align * 8 + + if gcc_layout: + # We don't use next_byte_offset here + assert pack is None + assert next_byte_offset == 0 + + # Determine whether the bit field, if placed at the next + # free bit, fits within a single object of its specified type. + # That is: determine a "slot", sized & aligned for the + # specified type, which contains the bitfield's beginning: + slot_start_bit = round_down(next_bit_offset, type_bit_align) + slot_end_bit = slot_start_bit + type_bit_size + # And see if it also contains the bitfield's last bit: + field_end_bit = next_bit_offset + bit_size + if field_end_bit > slot_end_bit: + # It doesn't: add padding (bump up to the next + # alignment boundary) + next_bit_offset = round_up(next_bit_offset, type_bit_align) + + offset = round_down(next_bit_offset, type_bit_align) // 8 + if is_bitfield: + effective_bit_offset = next_bit_offset - 8 * offset + size = build_size(bit_size, effective_bit_offset, + big_endian, type_size) + assert effective_bit_offset <= type_bit_size else: - if pack: - type_align = min(pack, type_align) - - # next_byte_offset points to end of current bitfield. - # next_bit_offset is generally non-positive, - # and 8 * next_byte_offset + next_bit_offset points just behind - # the end of the last field we placed. - if ( - (0 < next_bit_offset + bit_size) - or (type_bit_size != last_field_bit_size) - ): - # Close the previous bitfield (if any) - # and start a new bitfield - next_byte_offset = round_up(next_byte_offset, type_align) - - next_byte_offset += type_size - - last_field_bit_size = type_bit_size - # Reminder: 8 * (next_byte_offset) + next_bit_offset - # points to where we would start a - # new field. I.e. just behind where we placed the last - # field plus an allowance for alignment. - next_bit_offset = - last_field_bit_size - - assert type_bit_size == last_field_bit_size - assert type_bit_size > 0 - - offset = next_byte_offset - last_field_bit_size // 8 - if is_bitfield: - assert 0 <= (last_field_bit_size + next_bit_offset) - size = build_size(bit_size, - last_field_bit_size + next_bit_offset, - big_endian, type_size) - else: - size = type_size - assert (last_field_bit_size + next_bit_offset) < type_bit_size - - next_bit_offset += bit_size - struct_size = next_byte_offset - - assert((not is_bitfield) or (LOW_BIT(size) <= size * 8)) - - # Add the format spec parts - if is_struct: - padding = offset - last_size - format_spec_parts.append(padding_spec(padding)) - - fieldfmt, bf_ndim, bf_shape = buffer_info(ctype) - - if bf_shape: - format_spec_parts.extend(( - "(", - ','.join(str(n) for n in bf_shape), - ")", - )) - - if fieldfmt is None: - fieldfmt = "B" - format_spec_parts.append(f"{fieldfmt}:{name}:") - - last_field = CField( - name=name, - type=ctype, - size=size, - offset=offset, - bit_size=bit_size if is_bitfield else None, - index=i, - ) - self.fields.append(last_field) - align = max(align, type_align) - last_size = struct_size - if not is_struct: - union_size = max(struct_size, union_size) + assert offset == next_bit_offset / 8 + size = type_size - if is_struct: - total_size = struct_size + next_bit_offset += bit_size + struct_size = round_up(next_bit_offset, 8) // 8 else: - total_size = union_size + if pack: + type_align = min(pack, type_align) + + # next_byte_offset points to end of current bitfield. + # next_bit_offset is generally non-positive, + # and 8 * next_byte_offset + next_bit_offset points just behind + # the end of the last field we placed. + if ( + (0 < next_bit_offset + bit_size) + or (type_bit_size != last_field_bit_size) + ): + # Close the previous bitfield (if any) + # and start a new bitfield + next_byte_offset = round_up(next_byte_offset, type_align) + + next_byte_offset += type_size + + last_field_bit_size = type_bit_size + # Reminder: 8 * (next_byte_offset) + next_bit_offset + # points to where we would start a + # new field. I.e. just behind where we placed the last + # field plus an allowance for alignment. + next_bit_offset = - last_field_bit_size + + assert type_bit_size == last_field_bit_size + assert type_bit_size > 0 + + offset = next_byte_offset - last_field_bit_size // 8 + if is_bitfield: + assert 0 <= (last_field_bit_size + next_bit_offset) + size = build_size(bit_size, + last_field_bit_size + next_bit_offset, + big_endian, type_size) + else: + size = type_size + assert (last_field_bit_size + next_bit_offset) < type_bit_size - # Adjust the size according to the alignment requirements - aligned_size = round_up(total_size, align) + next_bit_offset += bit_size + struct_size = next_byte_offset - # Finish up the format spec + assert((not is_bitfield) or (LOW_BIT(size) <= size * 8)) + + # Add the format spec parts if is_struct: - padding = aligned_size - total_size + padding = offset - last_size format_spec_parts.append(padding_spec(padding)) - format_spec_parts.append("}") - self.size = aligned_size - self.align = align - self.format_spec = "".join(format_spec_parts) + fieldfmt, bf_ndim, bf_shape = buffer_info(ctype) + + if bf_shape: + format_spec_parts.extend(( + "(", + ','.join(str(n) for n in bf_shape), + ")", + )) + + if fieldfmt is None: + fieldfmt = "B" + format_spec_parts.append(f"{fieldfmt}:{name}:") + + last_field = CField( + name=name, + type=ctype, + size=size, + offset=offset, + bit_size=bit_size if is_bitfield else None, + index=i, + ) + result_fields.append(last_field) + align = max(align, type_align) + last_size = struct_size + if not is_struct: + union_size = max(struct_size, union_size) + + if is_struct: + total_size = struct_size + else: + total_size = union_size + + # Adjust the size according to the alignment requirements + aligned_size = round_up(total_size, align) + + # Finish up the format spec + if is_struct: + padding = aligned_size - total_size + format_spec_parts.append(padding_spec(padding)) + format_spec_parts.append("}") + + return StructUnionLayout( + fields=result_fields, + size=aligned_size, + align=align, + format_spec="".join(format_spec_parts), + ) def padding_spec(padding): @@ -265,17 +286,3 @@ def padding_spec(padding): if padding == 1: return "x" return f"{padding}x" - - -def default_layout(cls, *args, **kwargs): - layout = getattr(cls, '_layout_', None) - if layout is None: - if sys.platform == 'win32' or getattr(cls, '_pack_', None): - return _structunion_layout(cls, *args, **kwargs, _ms=True) - return _structunion_layout(cls, *args, **kwargs, _ms=False) - elif layout == 'ms': - return _structunion_layout(cls, *args, **kwargs, _ms=True) - elif layout == 'gcc-sysv': - return _structunion_layout(cls, *args, **kwargs, _ms=False) - else: - raise ValueError(f'unknown _layout_: {layout!r}') diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index a8ff1dc881ffe7..4a3a21b5146b63 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -258,19 +258,9 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; } - PyObject *layout_class; - if (PyObject_GetOptionalAttr(type, &_Py_ID(_layout_), &tmp) < 0) { - goto error; - } - if (!tmp || PyUnicode_Check(tmp)) { - layout_class = _PyImport_GetModuleAttrString("ctypes._layout", - "default_layout"); - Py_XDECREF(tmp); - } - else { - layout_class = tmp; - } - if (!layout_class) { + PyObject *layout_func = _PyImport_GetModuleAttrString("ctypes._layout", + "get_layout"); + if (!layout_func) { goto error; } PyObject *kwnames = PyTuple_Pack( @@ -281,7 +271,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct goto error; } layout = PyObject_Vectorcall( - layout_class, + layout_func, 1 + (PyObject*[]){ NULL, /* positional args */ @@ -293,7 +283,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); Py_DECREF(kwnames); - Py_DECREF(layout_class); + Py_DECREF(layout_func); fields = NULL; // a borrowed reference we won't be using again if (!layout) { goto error; From e4ad8390633bd3f6e121744c79e7243cb9133455 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 26 Aug 2024 16:03:24 +0200 Subject: [PATCH 72/94] Warning comment --- Lib/ctypes/_layout.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 7e2667b2752cd8..07226b2830fc99 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -1,3 +1,9 @@ +"""Python implementation of computing the layout of a struct/union + +This code is internal and tightly coupled to the C part. The interface +may change at any time. +""" + import sys import warnings import struct From c1ba0e89537ab54448927a138b03757d4bb6e166 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 26 Aug 2024 16:22:36 +0200 Subject: [PATCH 73/94] Remove test scaffolding --- Lib/test/test_ctypes/test_bitfields.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 86d6a7021459d1..e6509e6bf89e1d 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -185,9 +185,9 @@ class X(Structure): x.a, x.b = 0, -1 self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) - def fail_fields(self, *fields, expected=None): + def fail_fields(self, *fields): return self.get_except(type(Structure), "X", (), - {"_fields_": fields}, expected=expected) + {"_fields_": fields}) def test_nonint_types(self): # bit fields are not allowed on non-integer types. @@ -220,7 +220,7 @@ def test_single_bitfield_size(self): with self.subTest(c_typ): if sizeof(c_typ) != alignment(c_typ): self.skipTest('assumes size=alignment') - result = self.fail_fields(("a", c_typ, -1), expected=ValueError) + result = self.fail_fields(("a", c_typ, -1)) self.assertEqual(result, (ValueError, "number of bits invalid for bit field 'a'")) @@ -267,12 +267,10 @@ class X(Structure): self.assertEqual(X.b.offset, sizeof(c_short)*1) self.assertEqual(X.c.offset, sizeof(c_short)*2) - def get_except(self, func, *args, expected=None, **kw): + def get_except(self, func, *args, **kw): try: func(*args, **kw) except Exception as detail: - if expected and not isinstance(detail, expected): - raise return detail.__class__, str(detail) def test_mixed_1(self): From 25ceb4bc6639c201b2f46b3aabe663eeec6940cf Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 26 Aug 2024 16:23:22 +0200 Subject: [PATCH 74/94] CField is instantiable now --- Lib/test/test_ctypes/test_struct_fields.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_ctypes/test_struct_fields.py b/Lib/test/test_ctypes/test_struct_fields.py index 1746312155b46f..7d7a518c0138f9 100644 --- a/Lib/test/test_ctypes/test_struct_fields.py +++ b/Lib/test/test_ctypes/test_struct_fields.py @@ -60,7 +60,6 @@ def test_6(self): self.assertRaises(TypeError, CField) def test_cfield_type_flags(self): - #self.assertTrue(CField.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) self.assertTrue(CField.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) def test_cfield_inheritance_hierarchy(self): From 54640c37f473dda9b1df2eb050babf5471b2c915 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 26 Aug 2024 16:24:40 +0200 Subject: [PATCH 75/94] PEP-8 --- Lib/ctypes/_layout.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 07226b2830fc99..a6d0c25bc9fc18 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -152,7 +152,8 @@ def get_layout(cls, input_fields, is_struct, base): name, ctype, bit_size = field is_bitfield = True if bit_size <= 0: - raise ValueError(f'number of bits invalid for bit field {name!r}') + raise ValueError( + f'number of bits invalid for bit field {name!r}') type_size = ctypes.sizeof(ctype) type_bit_size = type_size * 8 From e62bb0298cad285760b0b7fd8d2fbe58fddb2674 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 26 Aug 2024 16:31:36 +0200 Subject: [PATCH 76/94] Move a comment over to its code --- Lib/ctypes/_layout.py | 28 ++++++++++++++++++++++++++++ Modules/_ctypes/cfield.c | 25 ------------------------- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index a6d0c25bc9fc18..20c5fcc2f797d9 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -63,7 +63,35 @@ def __init__(self, fields, size, align, format_spec): self.format_spec = format_spec + def get_layout(cls, input_fields, is_struct, base): + """Return a StructUnionLayout for the given class + + Called by PyCStructUnionType_update_stginfo when _fields_ is assigned + to a class. + """ + # Currently there are two modes, selectable using the '_layout_' attribute: + # + # 'gcc-sysv' mode places fields one after another, bit by bit. + # But "each bit field must fit within a single object of its specified + # type" (GCC manual, section 15.8 "Bit Field Packing"). When it doesn't, + # we insert a few bits of padding to avoid that. + # + # 'ms' mode works similar except for bitfield packing. Adjacent + # bit-fields are packed into the same 1-, 2-, or 4-byte allocation unit + # if the integral types are the same size and if the next bit-field fits + # into the current allocation unit without crossing the boundary imposed + # by the common alignment requirements of the bit-fields. + # + # See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html#index-mms-bitfields + # for details. + + # We do not support zero length bitfields (we use bitsize != 0 + # elsewhere to indicate a bitfield). Here, non-bitfields have bit_size + # set to size*8. + + # For clarity, variables that count bits have `bit` in their names. + layout = getattr(cls, '_layout_', None) if layout is None: if sys.platform == 'win32' or getattr(cls, '_pack_', None): diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index a3f887c1916e78..14914fcb17aba3 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -78,31 +78,6 @@ _cfield_is_bitfield(CFieldObject* self) return self->bit_size >= 0; } -/* PyCField_FromDesc creates and returns a struct/union field descriptor. - -The function expects to be called repeatedly for all fields in a struct or -union. It uses helper functions PyCField_FromDesc_gcc and -PyCField_FromDesc_msvc to simulate the corresponding compilers. - -GCC mode places fields one after another, bit by bit. But "each bit field must -fit within a single object of its specified type" (GCC manual, section 15.8 -"Bit Field Packing"). When it doesn't, we insert a few bits of padding to -avoid that. - -MSVC mode works similar except for bitfield packing. Adjacent bit-fields are -packed into the same 1-, 2-, or 4-byte allocation unit if the integral types -are the same size and if the next bit-field fits into the current allocation -unit without crossing the boundary imposed by the common alignment requirements -of the bit-fields. - -See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html#index-mms-bitfields for details. - -We do not support zero length bitfields. In fact we use bitsize != 0 elsewhere -to indicate a bitfield. Here, non-bitfields need bitsize set to size*8. - -PyCField_FromDesc manages the pack state struct. -*/ - /*[clinic input] @classmethod _ctypes.CField.__new__ as PyCField_new From e0b0de0f9643e28779b6e1b6cfc49710275cbdea Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 26 Aug 2024 16:32:16 +0200 Subject: [PATCH 77/94] Remove unused function --- Modules/_ctypes/cfield.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 14914fcb17aba3..45ec044710ca68 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -72,11 +72,6 @@ Py_ssize_t LOW_BIT(Py_ssize_t offset); static inline Py_ssize_t BUILD_SIZE(Py_ssize_t bitsize, Py_ssize_t offset); -static bool -_cfield_is_bitfield(CFieldObject* self) -{ - return self->bit_size >= 0; -} /*[clinic input] @classmethod From 51b7568542c8464bd91d737f93c3a5a36383e34f Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 26 Aug 2024 16:41:57 +0200 Subject: [PATCH 78/94] PEP-7, typo, align comments --- Modules/_ctypes/cfield.c | 5 +++-- Modules/_ctypes/ctypes.h | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 45ec044710ca68..41da6edfb1998f 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -95,7 +95,8 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, CFieldObject* self = NULL; if (size < 0) { PyErr_Format(PyExc_ValueError, - "size of field %R must not be negative, got %zd", name, size); + "size of field %R must not be negative, got %zd", + name, size); goto error; } // assert: no overflow; @@ -187,7 +188,7 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, self->index = index; - /* Field descriptors for 'c_char * n' are be scpecial cased to + /* Field descriptors for 'c_char * n' are be special cased to return a Python string instead of an Array object instance... */ self->setfunc = NULL; diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index dbe51c23176885..be8d1ed14dcb8e 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -253,13 +253,13 @@ typedef struct CFieldObject { Py_ssize_t size; Py_ssize_t index; /* Index into CDataObject's object array */ - PyObject *proto; /* underlying ctype; must have StgInfo */ + PyObject *proto; /* underlying ctype; must have StgInfo */ GETFUNC getfunc; /* getter function if proto is NULL */ SETFUNC setfunc; /* setter function if proto is NULL */ int anonymous; - Py_ssize_t bit_size; /* -1 if not a bitfield */ - PyObject *name; /* exact PyUnicode */ + Py_ssize_t bit_size; /* -1 if not a bitfield */ + PyObject *name; /* exact PyUnicode */ } CFieldObject; /**************************************************************** From 1092d68a78d30a888c26738558b251d58f90a187 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 27 Aug 2024 16:04:07 +0200 Subject: [PATCH 79/94] Regenerate global objects --- Include/internal/pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - Include/internal/pycore_runtime_init_generated.h | 1 - Include/internal/pycore_unicodeobject_generated.h | 4 ---- 4 files changed, 7 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 0e6064082ef566..6e948e16b7dbe8 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -758,7 +758,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_initializing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_io)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_is_text_encoding)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_layout_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_length_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_limbo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_lock_unlock_module)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 251bd1c8561fe2..5c63a6e519b93d 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -247,7 +247,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(_initializing) STRUCT_FOR_ID(_io) STRUCT_FOR_ID(_is_text_encoding) - STRUCT_FOR_ID(_layout_) STRUCT_FOR_ID(_length_) STRUCT_FOR_ID(_limbo) STRUCT_FOR_ID(_lock_unlock_module) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 7094bb958b5c86..bac6b5b8fcfd9d 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -756,7 +756,6 @@ extern "C" { INIT_ID(_initializing), \ INIT_ID(_io), \ INIT_ID(_is_text_encoding), \ - INIT_ID(_layout_), \ INIT_ID(_length_), \ INIT_ID(_limbo), \ INIT_ID(_lock_unlock_module), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index febf2862985dab..efdbde4c8ea3c6 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -788,10 +788,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(_layout_); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_length_); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); From 6820d23e05358d796756db65878a6a6257b33f6b Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 27 Aug 2024 16:08:47 +0200 Subject: [PATCH 80/94] Avoid BytesWarning --- Lib/ctypes/_layout.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 20c5fcc2f797d9..a3aa6b874ddb5f 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -277,6 +277,11 @@ def get_layout(cls, input_fields, is_struct, base): if fieldfmt is None: fieldfmt = "B" + if isinstance(name, bytes): + # a bytes name would be rejected later, but we check early + # to avoid a BytesWarning with `python -bb` + raise TypeError( + "field {name!r}: name must be a string, not bytes") format_spec_parts.append(f"{fieldfmt}:{name}:") last_field = CField( From 12311adc5244d05d36364122983a26f1a00bdcd0 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 27 Aug 2024 16:15:06 +0200 Subject: [PATCH 81/94] Move an assertion after a check --- Lib/ctypes/_layout.py | 6 ++++-- Lib/test/test_ctypes/test_bitfields.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index a3aa6b874ddb5f..df24597152b302 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -241,10 +241,9 @@ def get_layout(cls, input_fields, is_struct, base): # points to where we would start a # new field. I.e. just behind where we placed the last # field plus an allowance for alignment. - next_bit_offset = - last_field_bit_size + next_bit_offset = -last_field_bit_size assert type_bit_size == last_field_bit_size - assert type_bit_size > 0 offset = next_byte_offset - last_field_bit_size // 8 if is_bitfield: @@ -292,6 +291,9 @@ def get_layout(cls, input_fields, is_struct, base): bit_size=bit_size if is_bitfield else None, index=i, ) + if not gcc_layout: + assert type_bit_size > 0 + result_fields.append(last_field) align = max(align, type_align) last_size = struct_size diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index e6509e6bf89e1d..1adc0591e7bc1f 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -186,8 +186,10 @@ class X(Structure): self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) def fail_fields(self, *fields): - return self.get_except(type(Structure), "X", (), - {"_fields_": fields}) + for layout in "ms", "gcc-sysv": + with self.subTest(layout=layout): + return self.get_except(type(Structure), "X", (), + {"_fields_": fields, "layout": layout}) def test_nonint_types(self): # bit fields are not allowed on non-integer types. From 49214079a0a711272676ad599aca1aa1b15c6fc1 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 27 Aug 2024 16:31:54 +0200 Subject: [PATCH 82/94] Remove unused field, bit_size --- Lib/ctypes/_layout.py | 3 +++ Lib/test/test_ctypes/test_bitfields.py | 8 +++++++- Modules/_ctypes/cfield.c | 23 +++-------------------- Modules/_ctypes/ctypes.h | 1 - 4 files changed, 13 insertions(+), 22 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index df24597152b302..7d0cd347bd1c37 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -183,6 +183,9 @@ def get_layout(cls, input_fields, is_struct, base): raise ValueError( f'number of bits invalid for bit field {name!r}') type_size = ctypes.sizeof(ctype) + if bit_size > type_size * 8: + raise ValueError( + f'number of bits invalid for bit field {name!r}') type_bit_size = type_size * 8 type_align = ctypes.alignment(ctype) or 1 diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 1adc0591e7bc1f..7af1bbf388577c 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -206,9 +206,15 @@ def test_nonint_types(self): result = self.fail_fields(("a", c_char, 1)) self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char')) - class Dummy(Structure): + class Empty(Structure): _fields_ = [] + result = self.fail_fields(("a", Empty, 1)) + self.assertEqual(result, (ValueError, "number of bits invalid for bit field 'a'")) + + class Dummy(Structure): + _fields_ = [("x", c_int)] + result = self.fail_fields(("a", Dummy, 1)) self.assertEqual(result, (TypeError, 'bit fields not allowed for type Dummy')) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 41da6edfb1998f..9ddd55fd5b5a84 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -132,25 +132,8 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, goto error; } - if (bit_size_obj == Py_None) { - self->bit_size = -1; - } - else { - self->bit_size = PyLong_AsSsize_t(bit_size_obj); - if (self->bit_size < 0) { - if (!PyErr_Occurred()) { - PyErr_Format(PyExc_ValueError, - "number of bits invalid for bit field %R", - self->name); - } - goto error; - } - if (self->bit_size > size * 8) { - PyErr_Format(PyExc_ValueError, - "number of bits too large for bit field %R", - self->name); - goto error; - } + Py_ssize_t bit_size = NUM_BITS(size); + if (bit_size) { switch(info->ffi_type_pointer.type) { case FFI_TYPE_UINT8: case FFI_TYPE_UINT16: @@ -174,7 +157,7 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, ((PyTypeObject*)proto)->tp_name); goto error; } - if (self->bit_size <= 0 || self->bit_size > info->size * 8) { + if (bit_size <= 0 || bit_size > info->size * 8) { PyErr_Format(PyExc_ValueError, "number of bits invalid for bit field %R", self->name); diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index be8d1ed14dcb8e..2eb1b6cae4d81b 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -258,7 +258,6 @@ typedef struct CFieldObject { SETFUNC setfunc; /* setter function if proto is NULL */ int anonymous; - Py_ssize_t bit_size; /* -1 if not a bitfield */ PyObject *name; /* exact PyUnicode */ } CFieldObject; From ff4bcf5e000e4ecba946cae3633cf2c80f275a71 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 28 Aug 2024 15:56:56 +0200 Subject: [PATCH 83/94] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/ctypes/_layout.py | 26 +++++++++++++------------- Lib/test/test_ctypes/test_bitfields.py | 2 +- Modules/_ctypes/cfield.c | 4 ++-- Modules/_ctypes/stgdict.c | 3 ++- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index 7d0cd347bd1c37..fa7ac64c88a0a3 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -28,15 +28,15 @@ def NUM_BITS(bitsize): return bitsize >> 16 def BUILD_SIZE(bitsize, offset): - assert(0 <= offset) - assert(offset <= 0xFFFF) - ## We don't support zero length bitfields. - ## And GET_BITFIELD uses NUM_BITS(size)==0, - ## to figure out whether we are handling a bitfield. - assert(0 < bitsize) + assert 0 <= offset, offset + assert offset <= 0xFFFF, offset + # We don't support zero length bitfields. + # And GET_BITFIELD uses NUM_BITS(size) == 0, + # to figure out whether we are handling a bitfield. + assert bitsize > 0, bitsize result = (bitsize << 16) + offset - assert(bitsize == NUM_BITS(result)) - assert(offset == LOW_BIT(result)) + assert bitsize == NUM_BITS(result), (bitsize, result) + assert offset == LOW_BIT(result), (offset, result) return result def build_size(bit_size, bit_offset, big_endian, type_size): @@ -65,7 +65,7 @@ def __init__(self, fields, size, align, format_spec): def get_layout(cls, input_fields, is_struct, base): - """Return a StructUnionLayout for the given class + """Return a StructUnionLayout for the given class. Called by PyCStructUnionType_update_stginfo when _fields_ is assigned to a class. @@ -213,7 +213,7 @@ def get_layout(cls, input_fields, is_struct, base): if is_bitfield: effective_bit_offset = next_bit_offset - 8 * offset size = build_size(bit_size, effective_bit_offset, - big_endian, type_size) + big_endian, type_size) assert effective_bit_offset <= type_bit_size else: assert offset == next_bit_offset / 8 @@ -252,8 +252,8 @@ def get_layout(cls, input_fields, is_struct, base): if is_bitfield: assert 0 <= (last_field_bit_size + next_bit_offset) size = build_size(bit_size, - last_field_bit_size + next_bit_offset, - big_endian, type_size) + last_field_bit_size + next_bit_offset, + big_endian, type_size) else: size = type_size assert (last_field_bit_size + next_bit_offset) < type_bit_size @@ -261,7 +261,7 @@ def get_layout(cls, input_fields, is_struct, base): next_bit_offset += bit_size struct_size = next_byte_offset - assert((not is_bitfield) or (LOW_BIT(size) <= size * 8)) + assert (not is_bitfield) or (LOW_BIT(size) <= size * 8) # Add the format spec parts if is_struct: diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 7af1bbf388577c..4c68bd9926eaf4 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -189,7 +189,7 @@ def fail_fields(self, *fields): for layout in "ms", "gcc-sysv": with self.subTest(layout=layout): return self.get_except(type(Structure), "X", (), - {"_fields_": fields, "layout": layout}) + {"_fields_": fields, "layout": layout}) def test_nonint_types(self): # bit fields are not allowed on non-integer types. diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 9ddd55fd5b5a84..c98fe56fccc68a 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -159,8 +159,8 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, } if (bit_size <= 0 || bit_size > info->size * 8) { PyErr_Format(PyExc_ValueError, - "number of bits invalid for bit field %R", - self->name); + "number of bits invalid for bit field %R", + self->name); goto error; } } diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 4a3a21b5146b63..a1d403f8b2f766 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -354,6 +354,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (stginfo->ffi_type_pointer.elements) { PyMem_Free(stginfo->ffi_type_pointer.elements); + stginfo->ffi_type_pointer.elements = NULL; } if (baseinfo) { @@ -388,7 +389,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct assert(prop_obj); if (!PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type)) { PyErr_Format(PyExc_TypeError, - "fields must be of type CField, got %T", prop_obj); + "fields must be of type CField, got %T", prop_obj); goto error; } From 6b9a8b70b50afed855abb006041e90c215cdbb75 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 28 Aug 2024 15:03:49 +0200 Subject: [PATCH 84/94] Move bit_size checks to assertions; they're checked in Python --- Modules/_ctypes/cfield.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index c98fe56fccc68a..abcab6557de914 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -134,6 +134,8 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, Py_ssize_t bit_size = NUM_BITS(size); if (bit_size) { + assert(bit_size > 0); + assert(bit_size <= info->size * 8); switch(info->ffi_type_pointer.type) { case FFI_TYPE_UINT8: case FFI_TYPE_UINT16: @@ -156,12 +158,6 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, "bit fields not allowed for type %s", ((PyTypeObject*)proto)->tp_name); goto error; - } - if (bit_size <= 0 || bit_size > info->size * 8) { - PyErr_Format(PyExc_ValueError, - "number of bits invalid for bit field %R", - self->name); - goto error; } } From 991bab3e70152b483c6aeac29b0f6824420ab09e Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 28 Aug 2024 15:19:07 +0200 Subject: [PATCH 85/94] Move `i` declaration to for loops --- Modules/_ctypes/stgdict.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index a1d403f8b2f766..9f76f936267edd 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -217,7 +217,6 @@ MakeAnonFields(PyObject *type) int PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct) { - Py_ssize_t i; PyObject *tmp; Py_ssize_t ffi_ofs; int arrays_seen = 0; @@ -384,7 +383,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct ffi_ofs = 0; } - for (i = 0; i < len; ++i) { + for (Py_ssize_t i = 0; i < len; ++i) { PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); assert(prop_obj); if (!PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type)) { @@ -519,7 +518,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct Py_ssize_t struct_index = 0; /* index into dummy structs */ /* first pass to see how much memory to allocate */ - for (i = 0; i < len; ++i) { + for (Py_ssize_t i = 0; i < len; ++i) { PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); // borrowed assert(prop_obj); assert(PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type)); @@ -597,7 +596,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct element_index = ffi_ofs; /* second pass to actually set the type pointers */ - for (i = 0; i < len; ++i) { + for (Py_ssize_t i = 0; i < len; ++i) { PyObject *prop_obj = PyTuple_GET_ITEM(layout_fields, i); // borrowed assert(prop_obj); assert(PyType_IsSubtype(Py_TYPE(prop_obj), st->PyCField_Type)); From 298c8225a0aca6969c172678011a3021c06c4338 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 28 Aug 2024 15:19:22 +0200 Subject: [PATCH 86/94] Remove a clearly unneeded assert --- Modules/_ctypes/stgdict.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 9f76f936267edd..9b391e34fdc5d7 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -416,8 +416,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct stginfo->flags |= TYPEFLAG_HASPOINTER; info->flags |= DICTFLAG_FINAL; /* mark field type final */ - assert(prop); - if (-1 == PyObject_SetAttr(type, prop->name, prop_obj)) { goto error; } From 3088e5db24d84916175ddcb5eca590dbc2a1d2ac Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 28 Aug 2024 15:31:33 +0200 Subject: [PATCH 87/94] Add rudimentary test for bitfields in Union --- Lib/test/test_ctypes/test_bitfields.py | 19 +++++++++++- .../test_ctypes/test_generated_structs.py | 13 ++++++++ Modules/_ctypes/_ctypes_test_generated.c.h | 31 +++++++++++++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 4c68bd9926eaf4..19ba2f4484e7da 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -5,7 +5,9 @@ LittleEndianStructure, BigEndianStructure, c_byte, c_ubyte, c_char, c_char_p, c_void_p, c_wchar, c_uint8, c_uint16, c_uint32, c_uint64, - c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong) + c_short, c_ushort, c_int, c_uint, c_long, c_ulong, + c_longlong, c_ulonglong, + Union) from test import support from test.support import import_helper _ctypes_test = import_helper.import_module("_ctypes_test") @@ -526,6 +528,21 @@ class Big(BigEndianStructure): x.c = 2 self.assertEqual(b, b'\xab\xcd\xef\x12') + def test_union_bitfield(self): + class BitfieldUnion(Union): + _fields_ = [("a", c_uint32, 1), + ("b", c_uint32, 2), + ("c", c_uint32, 3)] + self.assertEqual(sizeof(BitfieldUnion), 4) + b = bytearray(4) + x = BitfieldUnion.from_buffer(b) + x.a = 1 + self.assertEqual(int.from_bytes(b).bit_count(), 1) + x.b = 3 + self.assertEqual(int.from_bytes(b).bit_count(), 2) + x.c = 7 + self.assertEqual(int.from_bytes(b).bit_count(), 3) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py index cbd73c4e911e4e..3870afb779b023 100644 --- a/Lib/test/test_ctypes/test_generated_structs.py +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -161,6 +161,19 @@ class MSStraddlingExample(Structure): ('may_straddle', c_uint, 30), ('last', c_uint, 18)] +@register() +class BitsfieldUnion(Union): + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9), + ("not_a_bitfield", c_int8)] + @register() class IntBits(Structure): _fields_ = [("A", c_int, 1), diff --git a/Modules/_ctypes/_ctypes_test_generated.c.h b/Modules/_ctypes/_ctypes_test_generated.c.h index 46a3e4b01e2259..97b616102f4602 100644 --- a/Modules/_ctypes/_ctypes_test_generated.c.h +++ b/Modules/_ctypes/_ctypes_test_generated.c.h @@ -362,6 +362,37 @@ return result; } + if (PyUnicode_CompareWithASCIIString(name, "BitsfieldUnion") == 0) { + + union BitsfieldUnion { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + int8_t not_a_bitfield; + }; + union BitsfieldUnion value = {0}; + APPEND(PyUnicode_FromString("BitsfieldUnion")); + APPEND(PyLong_FromLong(sizeof(union BitsfieldUnion))); + APPEND(PyLong_FromLong(_Alignof(union BitsfieldUnion))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + TEST_FIELD(int8_t, value.not_a_bitfield); + return result; + } + if (PyUnicode_CompareWithASCIIString(name, "IntBits") == 0) { struct IntBits { From 511e25d5ac91ddf9ff586d7fa6f6111710ce5c95 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 28 Aug 2024 15:50:26 +0200 Subject: [PATCH 88/94] Improve error message for _field_ element unpacking --- Lib/ctypes/_layout.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index fa7ac64c88a0a3..a55f5226f01519 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -173,11 +173,13 @@ def get_layout(cls, input_fields, is_struct, base): field = tuple(field) try: name, ctype = field - is_bitfield = False - type_size = ctypes.sizeof(ctype) - bit_size = type_size * 8 - except ValueError: - name, ctype, bit_size = field + except (ValueError, TypeError): + try: + name, ctype, bit_size = field + except (ValueError, TypeError) as exc: + raise ValueError( + '_fields_ must be a sequence of (name, C type) pairs ' + + 'or (name, C type, bit size) triples') from exc is_bitfield = True if bit_size <= 0: raise ValueError( @@ -186,6 +188,10 @@ def get_layout(cls, input_fields, is_struct, base): if bit_size > type_size * 8: raise ValueError( f'number of bits invalid for bit field {name!r}') + else: + is_bitfield = False + type_size = ctypes.sizeof(ctype) + bit_size = type_size * 8 type_bit_size = type_size * 8 type_align = ctypes.alignment(ctype) or 1 From 0689e59d21c47cf9d5b34eba160a01a0aa3be02e Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 28 Aug 2024 17:32:38 +0200 Subject: [PATCH 89/94] Use correct length modifier --- Modules/_ctypes/stgdict.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 9b391e34fdc5d7..c9cd1c6e7381f6 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -396,7 +396,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (prop->index != i) { PyErr_Format(PyExc_ValueError, - "field %R index mismatch (expected %d, got %d)", + "field %R index mismatch (expected %zd, got %zd)", prop->name, i, prop->index); goto error; } From a7c3724f02b032cce97c7470687e1dcabbe32c52 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 28 Aug 2024 18:00:29 +0200 Subject: [PATCH 90/94] Avoid a case where MSC and GCC's __attribute__((ms_struct)) disagree --- .../test_ctypes/test_generated_structs.py | 21 ++++++---- Modules/_ctypes/_ctypes_test_generated.c.h | 41 +++++++++++-------- 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py index 3870afb779b023..5a9a3e952ed37e 100644 --- a/Lib/test/test_ctypes/test_generated_structs.py +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -163,15 +163,18 @@ class MSStraddlingExample(Structure): @register() class BitsfieldUnion(Union): - _fields_ = [("A", c_int, 1), - ("B", c_int, 2), - ("C", c_int, 3), - ("D", c_int, 4), - ("E", c_int, 5), - ("F", c_int, 6), - ("G", c_int, 7), - ("H", c_int, 8), - ("I", c_int, 9), + # Note that if the bit fields are changed to `c_int`, + # the alignment is unclear: + # MSC reports 1 but GCC with __attribute__((ms_struct)) gives 4. + _fields_ = [("A", c_int8, 1), + ("B", c_int8, 2), + ("C", c_int8, 3), + ("D", c_int8, 4), + ("E", c_int8, 5), + ("F", c_int8, 6), + ("G", c_int8, 7), + ("H", c_int8, 8), + # ("I", c_int, 9), ("not_a_bitfield", c_int8)] @register() diff --git a/Modules/_ctypes/_ctypes_test_generated.c.h b/Modules/_ctypes/_ctypes_test_generated.c.h index 97b616102f4602..ccd288b8af5f4d 100644 --- a/Modules/_ctypes/_ctypes_test_generated.c.h +++ b/Modules/_ctypes/_ctypes_test_generated.c.h @@ -364,32 +364,37 @@ if (PyUnicode_CompareWithASCIIString(name, "BitsfieldUnion") == 0) { + #if (!defined(__xlc__)) + union BitsfieldUnion { - int A :1; - int B :2; - int C :3; - int D :4; - int E :5; - int F :6; - int G :7; - int H :8; - int I :9; + int8_t A :1; + int8_t B :2; + int8_t C :3; + int8_t D :4; + int8_t E :5; + int8_t F :6; + int8_t G :7; + int8_t H :8; int8_t not_a_bitfield; }; union BitsfieldUnion value = {0}; APPEND(PyUnicode_FromString("BitsfieldUnion")); APPEND(PyLong_FromLong(sizeof(union BitsfieldUnion))); APPEND(PyLong_FromLong(_Alignof(union BitsfieldUnion))); - TEST_FIELD(int, value.A); - TEST_FIELD(int, value.B); - TEST_FIELD(int, value.C); - TEST_FIELD(int, value.D); - TEST_FIELD(int, value.E); - TEST_FIELD(int, value.F); - TEST_FIELD(int, value.G); - TEST_FIELD(int, value.H); - TEST_FIELD(int, value.I); + TEST_FIELD(int8_t, value.A); + TEST_FIELD(int8_t, value.B); + TEST_FIELD(int8_t, value.C); + TEST_FIELD(int8_t, value.D); + TEST_FIELD(int8_t, value.E); + TEST_FIELD(int8_t, value.F); + TEST_FIELD(int8_t, value.G); + TEST_FIELD(int8_t, value.H); TEST_FIELD(int8_t, value.not_a_bitfield); + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + return result; } From bd53c2888012797be8164d96e83f813b88491129 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 28 Aug 2024 18:10:08 +0200 Subject: [PATCH 91/94] Adjust Windows assertions --- Lib/ctypes/_layout.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index a55f5226f01519..d385f9c804790e 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -262,7 +262,8 @@ def get_layout(cls, input_fields, is_struct, base): big_endian, type_size) else: size = type_size - assert (last_field_bit_size + next_bit_offset) < type_bit_size + if type_bit_size: + assert (last_field_bit_size + next_bit_offset) < type_bit_size next_bit_offset += bit_size struct_size = next_byte_offset @@ -300,7 +301,7 @@ def get_layout(cls, input_fields, is_struct, base): bit_size=bit_size if is_bitfield else None, index=i, ) - if not gcc_layout: + if is_bitfield and not gcc_layout: assert type_bit_size > 0 result_fields.append(last_field) From b387d30953b6fef68b9a32cb04c7f1a5b8a10ca5 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 30 Aug 2024 11:25:05 +0200 Subject: [PATCH 92/94] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/ctypes/_layout.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index d385f9c804790e..fa117105d3edf6 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -63,7 +63,6 @@ def __init__(self, fields, size, align, format_spec): self.format_spec = format_spec - def get_layout(cls, input_fields, is_struct, base): """Return a StructUnionLayout for the given class. @@ -247,9 +246,9 @@ def get_layout(cls, input_fields, is_struct, base): last_field_bit_size = type_bit_size # Reminder: 8 * (next_byte_offset) + next_bit_offset - # points to where we would start a - # new field. I.e. just behind where we placed the last - # field plus an allowance for alignment. + # points to where we would start a new field, namely + # just behind where we placed the last field plus an + # allowance for alignment. next_bit_offset = -last_field_bit_size assert type_bit_size == last_field_bit_size From 9cf18cdcca0475c176fcb7b7369bc411e9ce25f5 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 28 Aug 2024 18:18:35 +0200 Subject: [PATCH 93/94] Remove unused last_field --- Lib/ctypes/_layout.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index fa117105d3edf6..e30db598ab22e1 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -160,7 +160,6 @@ def get_layout(cls, input_fields, is_struct, base): next_byte_offset = struct_size last_size = struct_size - last_field = None for i, field in enumerate(input_fields): if not is_struct: # Unions start fresh each time @@ -292,18 +291,17 @@ def get_layout(cls, input_fields, is_struct, base): "field {name!r}: name must be a string, not bytes") format_spec_parts.append(f"{fieldfmt}:{name}:") - last_field = CField( + result_fields.append(CField( name=name, type=ctype, size=size, offset=offset, bit_size=bit_size if is_bitfield else None, index=i, - ) + )) if is_bitfield and not gcc_layout: assert type_bit_size > 0 - result_fields.append(last_field) align = max(align, type_align) last_size = struct_size if not is_struct: From 96a5c0d6caac06975b6700304b8f3a43cf4fc874 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 30 Aug 2024 11:26:32 +0200 Subject: [PATCH 94/94] Remove the bitfield-in-union test, for now --- .../test_ctypes/test_generated_structs.py | 16 --------- Modules/_ctypes/_ctypes_test_generated.c.h | 36 ------------------- 2 files changed, 52 deletions(-) diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py index 5a9a3e952ed37e..cbd73c4e911e4e 100644 --- a/Lib/test/test_ctypes/test_generated_structs.py +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -161,22 +161,6 @@ class MSStraddlingExample(Structure): ('may_straddle', c_uint, 30), ('last', c_uint, 18)] -@register() -class BitsfieldUnion(Union): - # Note that if the bit fields are changed to `c_int`, - # the alignment is unclear: - # MSC reports 1 but GCC with __attribute__((ms_struct)) gives 4. - _fields_ = [("A", c_int8, 1), - ("B", c_int8, 2), - ("C", c_int8, 3), - ("D", c_int8, 4), - ("E", c_int8, 5), - ("F", c_int8, 6), - ("G", c_int8, 7), - ("H", c_int8, 8), - # ("I", c_int, 9), - ("not_a_bitfield", c_int8)] - @register() class IntBits(Structure): _fields_ = [("A", c_int, 1), diff --git a/Modules/_ctypes/_ctypes_test_generated.c.h b/Modules/_ctypes/_ctypes_test_generated.c.h index ccd288b8af5f4d..46a3e4b01e2259 100644 --- a/Modules/_ctypes/_ctypes_test_generated.c.h +++ b/Modules/_ctypes/_ctypes_test_generated.c.h @@ -362,42 +362,6 @@ return result; } - if (PyUnicode_CompareWithASCIIString(name, "BitsfieldUnion") == 0) { - - #if (!defined(__xlc__)) - - union BitsfieldUnion { - int8_t A :1; - int8_t B :2; - int8_t C :3; - int8_t D :4; - int8_t E :5; - int8_t F :6; - int8_t G :7; - int8_t H :8; - int8_t not_a_bitfield; - }; - union BitsfieldUnion value = {0}; - APPEND(PyUnicode_FromString("BitsfieldUnion")); - APPEND(PyLong_FromLong(sizeof(union BitsfieldUnion))); - APPEND(PyLong_FromLong(_Alignof(union BitsfieldUnion))); - TEST_FIELD(int8_t, value.A); - TEST_FIELD(int8_t, value.B); - TEST_FIELD(int8_t, value.C); - TEST_FIELD(int8_t, value.D); - TEST_FIELD(int8_t, value.E); - TEST_FIELD(int8_t, value.F); - TEST_FIELD(int8_t, value.G); - TEST_FIELD(int8_t, value.H); - TEST_FIELD(int8_t, value.not_a_bitfield); - #else - APPEND(Py_NewRef(Py_None)); - APPEND(PyUnicode_FromString("skipped on this compiler")); - #endif - - return result; - } - if (PyUnicode_CompareWithASCIIString(name, "IntBits") == 0) { struct IntBits {