From 87a974bc8cfbefc120f85a950b135a4e920c9678 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sat, 15 Mar 2025 17:05:08 +0500 Subject: [PATCH 01/65] Move ctype's pointers cache to StgInfo --- Lib/ctypes/__init__.py | 4 +--- Lib/test/test_ctypes/test_byteswap.py | 2 -- Lib/test/test_ctypes/test_keeprefs.py | 7 +------ Lib/test/test_ctypes/test_pointers.py | 10 +--------- Lib/test/test_ctypes/test_values.py | 3 --- Lib/test/test_ctypes/test_win32.py | 5 ++--- Modules/_ctypes/_ctypes.c | 2 ++ Modules/_ctypes/callproc.c | 26 ++++++++++++++------------ Modules/_ctypes/ctypes.h | 2 ++ 9 files changed, 23 insertions(+), 38 deletions(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 8e2a2926f7a853..274699b59c4509 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -319,11 +319,9 @@ def SetPointerType(pointer, cls): warnings._deprecated("ctypes.SetPointerType", remove=(3, 15)) if _pointer_type_cache.get(cls, None) is not None: raise RuntimeError("This type already exists in the cache") - if id(pointer) not in _pointer_type_cache: - raise RuntimeError("What's this???") + pointer.set_type(cls) _pointer_type_cache[cls] = pointer - del _pointer_type_cache[id(pointer)] def ARRAY(typ, len): return typ * len diff --git a/Lib/test/test_ctypes/test_byteswap.py b/Lib/test/test_ctypes/test_byteswap.py index 78eff0392c4548..38f9be6d50aa1c 100644 --- a/Lib/test/test_ctypes/test_byteswap.py +++ b/Lib/test/test_ctypes/test_byteswap.py @@ -226,7 +226,6 @@ class TestStructure(parent): self.assertEqual(len(data), sizeof(TestStructure)) ptr = POINTER(TestStructure) s = cast(data, ptr)[0] - del ctypes._pointer_type_cache[TestStructure] self.assertEqual(s.point.x, 1) self.assertEqual(s.point.y, 2) @@ -359,7 +358,6 @@ class TestUnion(parent): self.assertEqual(len(data), sizeof(TestUnion)) ptr = POINTER(TestUnion) s = cast(data, ptr)[0] - del ctypes._pointer_type_cache[TestUnion] self.assertEqual(s.point.x, 1) self.assertEqual(s.point.y, 2) diff --git a/Lib/test/test_ctypes/test_keeprefs.py b/Lib/test/test_ctypes/test_keeprefs.py index 23b03b64b4a716..5602460d5ff8e0 100644 --- a/Lib/test/test_ctypes/test_keeprefs.py +++ b/Lib/test/test_ctypes/test_keeprefs.py @@ -1,6 +1,5 @@ import unittest -from ctypes import (Structure, POINTER, pointer, _pointer_type_cache, - c_char_p, c_int) +from ctypes import (Structure, POINTER, pointer, c_char_p, c_int) class SimpleTestCase(unittest.TestCase): @@ -115,10 +114,6 @@ class RECT(Structure): r.a[0].x = 42 r.a[0].y = 99 - # to avoid leaking when tests are run several times - # clean up the types left in the cache. - del _pointer_type_cache[POINT] - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index fc558e10ba40c5..1f95212dc2cb33 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -3,7 +3,7 @@ import sys import unittest from ctypes import (CDLL, CFUNCTYPE, Structure, - POINTER, pointer, _Pointer, _pointer_type_cache, + POINTER, pointer, _Pointer, byref, sizeof, c_void_p, c_char_p, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, @@ -141,8 +141,6 @@ class Table(Structure): pt.contents.c = 33 - del _pointer_type_cache[Table] - def test_basic(self): p = pointer(c_int(42)) # Although a pointer can be indexed, it has no length @@ -210,17 +208,11 @@ def test_pointer_type_name(self): LargeNamedType = type('T' * 2 ** 25, (Structure,), {}) self.assertTrue(POINTER(LargeNamedType)) - # to not leak references, we must clean _pointer_type_cache - del _pointer_type_cache[LargeNamedType] - def test_pointer_type_str_name(self): large_string = 'T' * 2 ** 25 P = POINTER(large_string) self.assertTrue(P) - # to not leak references, we must clean _pointer_type_cache - del _pointer_type_cache[id(P)] - def test_abstract(self): self.assertRaises(TypeError, _Pointer.set_type, 42) diff --git a/Lib/test/test_ctypes/test_values.py b/Lib/test/test_ctypes/test_values.py index 1b757e020d5ce2..1c11ed6120ebc2 100644 --- a/Lib/test/test_ctypes/test_values.py +++ b/Lib/test/test_ctypes/test_values.py @@ -7,7 +7,6 @@ import sys import unittest from ctypes import (Structure, CDLL, POINTER, pythonapi, - _pointer_type_cache, c_ubyte, c_char_p, c_int) from test.support import import_helper @@ -96,8 +95,6 @@ class struct_frozen(Structure): "_PyImport_FrozenBootstrap example " "in Doc/library/ctypes.rst may be out of date") - del _pointer_type_cache[struct_frozen] - def test_undefined(self): self.assertRaises(ValueError, c_int.in_dll, pythonapi, "Undefined_Symbol") diff --git a/Lib/test/test_ctypes/test_win32.py b/Lib/test/test_ctypes/test_win32.py index 54b47dc28fbc73..14e3c222c6ce63 100644 --- a/Lib/test/test_ctypes/test_win32.py +++ b/Lib/test/test_ctypes/test_win32.py @@ -5,7 +5,6 @@ import sys import unittest from ctypes import (CDLL, Structure, POINTER, pointer, sizeof, byref, - _pointer_type_cache, c_void_p, c_char, c_int, c_long) from test import support from test.support import import_helper @@ -145,8 +144,8 @@ class RECT(Structure): self.assertEqual(ret.top, top.value) self.assertEqual(ret.bottom, bottom.value) - # to not leak references, we must clean _pointer_type_cache - del _pointer_type_cache[RECT] + self.assertEqual(id(PointInRect.argtypes[0]), id(ReturnRect.argtypes[2])) + self.assertEqual(id(PointInRect.argtypes[0]), id(ReturnRect.argtypes[5])) if __name__ == '__main__': diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 7c0ac1a57f534c..c8658c53ad5c8a 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -474,6 +474,7 @@ CType_Type_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(info->restype); Py_VISIT(info->checker); Py_VISIT(info->module); + Py_VISIT(info->pointer_type); } Py_VISIT(Py_TYPE(self)); return PyType_Type.tp_traverse(self, visit, arg); @@ -489,6 +490,7 @@ ctype_clear_stginfo(StgInfo *info) Py_CLEAR(info->restype); Py_CLEAR(info->checker); Py_CLEAR(info->module); + Py_CLEAR(info->pointer_type); } static int diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index c6b6460126ca90..7325bea0237096 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1990,7 +1990,6 @@ create_pointer_type(PyObject *module, PyObject *cls) { PyObject *result; PyTypeObject *typ; - PyObject *key; assert(module); ctypes_state *st = get_module_state(module); @@ -1998,6 +1997,16 @@ create_pointer_type(PyObject *module, PyObject *cls) // found or error return result; } + + StgInfo* info = NULL; + if (PyStgInfo_FromAny(st, cls, &info) < 0) { + return NULL; + } + + if (info && info->pointer_type) { + return Py_XNewRef(info->pointer_type); + } + // not found if (PyUnicode_CheckExact(cls)) { PyObject *name = PyUnicode_FromFormat("LP_%U", cls); @@ -2007,11 +2016,6 @@ create_pointer_type(PyObject *module, PyObject *cls) st->PyCPointer_Type); if (result == NULL) return result; - key = PyLong_FromVoidPtr(result); - if (key == NULL) { - Py_DECREF(result); - return NULL; - } } else if (PyType_Check(cls)) { typ = (PyTypeObject *)cls; PyObject *name = PyUnicode_FromFormat("LP_%s", typ->tp_name); @@ -2022,17 +2026,15 @@ create_pointer_type(PyObject *module, PyObject *cls) "_type_", cls); if (result == NULL) return result; - key = Py_NewRef(cls); } else { PyErr_SetString(PyExc_TypeError, "must be a ctypes type"); return NULL; } - if (PyDict_SetItem(st->_ctypes_ptrtype_cache, key, result) < 0) { - Py_DECREF(result); - Py_DECREF(key); - return NULL; + + if (info) { + info->pointer_type = Py_XNewRef(result); } - Py_DECREF(key); + return result; } diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 07049d0968c790..ebf0ef85be4524 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -373,6 +373,7 @@ typedef struct { Py_ssize_t *shape; /* Py_ssize_t *strides; */ /* unused in ctypes */ /* Py_ssize_t *suboffsets; */ /* unused in ctypes */ + PyObject *pointer_type; } StgInfo; extern int PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info); @@ -566,6 +567,7 @@ PyStgInfo_Init(ctypes_state *state, PyTypeObject *type) return NULL; } info->module = Py_NewRef(module); + info->pointer_type = NULL; info->initialized = 1; return info; From 12600a6112837323f7806afc6e2e710abb337473 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sun, 16 Mar 2025 00:47:28 +0500 Subject: [PATCH 02/65] Fix PyCStgInfo_clone for pointer_type --- Modules/_ctypes/stgdict.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 05239d85c44d2c..ee5fd2aafdd3ef 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -41,6 +41,7 @@ PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info) Py_XINCREF(dst_info->restype); Py_XINCREF(dst_info->checker); Py_XINCREF(dst_info->module); + Py_XINCREF(dst_info->pointer_type); if (src_info->format) { dst_info->format = PyMem_Malloc(strlen(src_info->format) + 1); From df2167a6b903ad850f57770d4d6ecdc7aaa8771a Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sun, 16 Mar 2025 01:55:38 +0500 Subject: [PATCH 03/65] No need to use _ctypes_ptrtype_cache for create_pointer_inst --- Modules/_ctypes/callproc.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 7325bea0237096..38acca10640ac1 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -2059,14 +2059,10 @@ create_pointer_inst(PyObject *module, PyObject *arg) PyObject *typ; ctypes_state *st = get_module_state(module); - if (PyDict_GetItemRef(st->_ctypes_ptrtype_cache, (PyObject *)Py_TYPE(arg), &typ) < 0) { + typ = create_pointer_type(module, (PyObject *)Py_TYPE(arg)); + if (typ == NULL) return NULL; - } - if (typ == NULL) { - typ = create_pointer_type(module, (PyObject *)Py_TYPE(arg)); - if (typ == NULL) - return NULL; - } + result = PyObject_CallOneArg(typ, arg); Py_DECREF(typ); return result; From f96ae5902ea5ea09c0558169124e8735133f7206 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sun, 16 Mar 2025 02:07:51 +0500 Subject: [PATCH 04/65] Remove unused st from create_pointer_inst --- Modules/_ctypes/callproc.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 38acca10640ac1..f8ca763fff8bf3 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -2058,7 +2058,6 @@ create_pointer_inst(PyObject *module, PyObject *arg) PyObject *result; PyObject *typ; - ctypes_state *st = get_module_state(module); typ = create_pointer_type(module, (PyObject *)Py_TYPE(arg)); if (typ == NULL) return NULL; From 481cf59554d695d4a70ed9df87ada0eadb72f3a7 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sun, 16 Mar 2025 21:23:23 +0500 Subject: [PATCH 05/65] It is better to check local cache first --- Modules/_ctypes/callproc.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index f8ca763fff8bf3..8fc9f83c6bb0b8 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1993,11 +1993,6 @@ create_pointer_type(PyObject *module, PyObject *cls) assert(module); ctypes_state *st = get_module_state(module); - if (PyDict_GetItemRef(st->_ctypes_ptrtype_cache, cls, &result) != 0) { - // found or error - return result; - } - StgInfo* info = NULL; if (PyStgInfo_FromAny(st, cls, &info) < 0) { return NULL; @@ -2007,6 +2002,11 @@ create_pointer_type(PyObject *module, PyObject *cls) return Py_XNewRef(info->pointer_type); } + if (PyDict_GetItemRef(st->_ctypes_ptrtype_cache, cls, &result) != 0) { + // found or error + return result; + } + // not found if (PyUnicode_CheckExact(cls)) { PyObject *name = PyUnicode_FromFormat("LP_%U", cls); From 96b4dd13d65ca48ef4978fccacad4b209ad293c3 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Mon, 17 Mar 2025 12:21:21 +0500 Subject: [PATCH 06/65] Update Modules/_ctypes/callproc.c Co-authored-by: neonene <53406459+neonene@users.noreply.github.com> --- Modules/_ctypes/callproc.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 8fc9f83c6bb0b8..bbb75b2b57b10f 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1993,13 +1993,14 @@ create_pointer_type(PyObject *module, PyObject *cls) assert(module); ctypes_state *st = get_module_state(module); - StgInfo* info = NULL; - if (PyStgInfo_FromAny(st, cls, &info) < 0) { - return NULL; - } - - if (info && info->pointer_type) { - return Py_XNewRef(info->pointer_type); + StgInfo *info = NULL; + if (PyType_Check(cls)) { + if (PyStgInfo_FromType(st, cls, &info) < 0) { + return NULL; + } + if (info && info->pointer_type) { + return Py_NewRef(info->pointer_type); + } } if (PyDict_GetItemRef(st->_ctypes_ptrtype_cache, cls, &result) != 0) { From 4fb5baa756b0868d8c385ba0f3cdddc491475480 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Mon, 17 Mar 2025 12:21:38 +0500 Subject: [PATCH 07/65] Update Modules/_ctypes/callproc.c Co-authored-by: neonene <53406459+neonene@users.noreply.github.com> --- Modules/_ctypes/callproc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index bbb75b2b57b10f..2f564c941f3f0a 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -2033,7 +2033,7 @@ create_pointer_type(PyObject *module, PyObject *cls) } if (info) { - info->pointer_type = Py_XNewRef(result); + info->pointer_type = Py_NewRef(result); } return result; From 758045f77e7dbf47e6efdb93d0f885052ed03b88 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Mon, 17 Mar 2025 23:33:21 +0500 Subject: [PATCH 08/65] Add news and whatsnew entries --- Doc/whatsnew/3.14.rst | 5 +++++ .../Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst | 2 ++ 2 files changed, 7 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index b1337190636529..26ca4f752fc111 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -510,6 +510,11 @@ ctypes loaded by the current process. (Contributed by Brian Ward in :gh:`119349`.) +* Move :func:`ctypes.POINTER` types cache from the global ``_pointer_type_cache`` + to the corresponding :mod:`ctypes` types. This will stop the cache from + growing without limits in some situations. + (Contributed by Sergey Miryanov in :gh:`100926`). + datetime -------- diff --git a/Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst b/Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst new file mode 100644 index 00000000000000..dfee62f2f1a4d3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst @@ -0,0 +1,2 @@ +Move :func:`ctypes.POINTER` types cache from the global ``_pointer_type_cache`` +to the corresponding :mod:`ctypes` types to avoid unlimited growth of the cache. From 2f9285f484ccb6522260ded35e641a91647fa6d0 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 19 Mar 2025 00:35:58 +0500 Subject: [PATCH 09/65] Update Modules/_ctypes/_ctypes.c Co-authored-by: neonene <53406459+neonene@users.noreply.github.com> --- Modules/_ctypes/_ctypes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index c8658c53ad5c8a..1b58345c8cee1c 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -489,8 +489,8 @@ ctype_clear_stginfo(StgInfo *info) Py_CLEAR(info->converters); Py_CLEAR(info->restype); Py_CLEAR(info->checker); - Py_CLEAR(info->module); Py_CLEAR(info->pointer_type); + Py_CLEAR(info->module); // decref the module last } static int From b070ad58f78ffd1ef03b7a6cc8e839d7bc57f04c Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 19 Mar 2025 00:40:03 +0500 Subject: [PATCH 10/65] Arrange pointer_type declaration --- Modules/_ctypes/_ctypes.c | 2 +- Modules/_ctypes/ctypes.h | 4 ++-- Modules/_ctypes/stgdict.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 1b58345c8cee1c..5a3407c1969f47 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -473,8 +473,8 @@ CType_Type_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(info->converters); Py_VISIT(info->restype); Py_VISIT(info->checker); - Py_VISIT(info->module); Py_VISIT(info->pointer_type); + Py_VISIT(info->module); } Py_VISIT(Py_TYPE(self)); return PyType_Type.tp_traverse(self, visit, arg); diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index ebf0ef85be4524..ab4534b3413335 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -364,6 +364,7 @@ typedef struct { PyObject *converters; /* tuple([t.from_param for t in argtypes]) */ PyObject *restype; /* CDataObject or NULL */ PyObject *checker; + PyObject *pointer_type; PyObject *module; int flags; /* calling convention and such */ @@ -373,7 +374,6 @@ typedef struct { Py_ssize_t *shape; /* Py_ssize_t *strides; */ /* unused in ctypes */ /* Py_ssize_t *suboffsets; */ /* unused in ctypes */ - PyObject *pointer_type; } StgInfo; extern int PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info); @@ -566,8 +566,8 @@ PyStgInfo_Init(ctypes_state *state, PyTypeObject *type) if (!module) { return NULL; } - info->module = Py_NewRef(module); info->pointer_type = NULL; + info->module = Py_NewRef(module); info->initialized = 1; return info; diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index ee5fd2aafdd3ef..a49625bbf2b032 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -40,8 +40,8 @@ PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info) Py_XINCREF(dst_info->converters); Py_XINCREF(dst_info->restype); Py_XINCREF(dst_info->checker); - Py_XINCREF(dst_info->module); Py_XINCREF(dst_info->pointer_type); + Py_XINCREF(dst_info->module); if (src_info->format) { dst_info->format = PyMem_Malloc(strlen(src_info->format) + 1); From ef1e633128e22abb061fc92f4436b93421cc3ea3 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 19 Mar 2025 00:42:03 +0500 Subject: [PATCH 11/65] Use assertIs to check pointer types in tests --- Lib/test/test_ctypes/test_win32.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ctypes/test_win32.py b/Lib/test/test_ctypes/test_win32.py index 14e3c222c6ce63..7d5133221906bb 100644 --- a/Lib/test/test_ctypes/test_win32.py +++ b/Lib/test/test_ctypes/test_win32.py @@ -144,8 +144,8 @@ class RECT(Structure): self.assertEqual(ret.top, top.value) self.assertEqual(ret.bottom, bottom.value) - self.assertEqual(id(PointInRect.argtypes[0]), id(ReturnRect.argtypes[2])) - self.assertEqual(id(PointInRect.argtypes[0]), id(ReturnRect.argtypes[5])) + self.assertIs(PointInRect.argtypes[0], ReturnRect.argtypes[2]) + self.assertIs(PointInRect.argtypes[0], ReturnRect.argtypes[5]) if __name__ == '__main__': From 3754d3d842d9db6a9a2f28cdea6bae000a973be1 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sat, 22 Mar 2025 01:25:16 +0500 Subject: [PATCH 12/65] Implement pointer-type creation via __pointer_type__ --- Lib/ctypes/__init__.py | 22 ++++++++++++ .../test_ctypes/test_c_simple_type_meta.py | 8 ++--- Modules/_ctypes/_ctypes.c | 36 +++++++++++++++++-- Modules/_ctypes/callproc.c | 2 +- 4 files changed, 60 insertions(+), 8 deletions(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 274699b59c4509..9fcf7312c1c6d6 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -266,6 +266,28 @@ class c_bool(_SimpleCData): from _ctypes import POINTER, pointer, _pointer_type_cache +CType_Type = _Pointer.__base__ + +def POINTER(cls): + if cls is None: + return c_void_p + try: + pt = cls.__pointer_type__ + if pt is not None: + return pt + except AttributeError: + pass + if isinstance(cls, str): + return type(f'LP_{cls}', (_Pointer,), {}) + if issubclass(cls, CType_Type): + return type(f'LP_{cls.__name__}', (_Pointer,), {'_type_': cls}) + + raise TypeError(f'must be a ctypes-like type: {cls}') + +def pointer(arg): + typ = POINTER(type(arg)) + return typ(arg) + class c_wchar_p(_SimpleCData): _type_ = "Z" def __repr__(self): diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py index b446fd5c77dde2..2810bf826a543c 100644 --- a/Lib/test/test_ctypes/test_c_simple_type_meta.py +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -36,7 +36,7 @@ def __new__(cls, name, bases, namespace): else: ptr_bases = (self, POINTER(bases[0])) p = p_meta(f"POINTER({self.__name__})", ptr_bases, {}) - ctypes._pointer_type_cache[self] = p + cls.__pointer_type__ = p return self class p_meta(PyCSimpleType, ct_meta): @@ -69,7 +69,7 @@ def __new__(cls, name, bases, namespace): if isinstance(self, p_meta): return self p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {}) - ctypes._pointer_type_cache[self] = p + cls.__pointer_type__ = p return self class p_meta(PyCSimpleType, ct_meta): @@ -103,7 +103,7 @@ def __init__(self, name, bases, namespace): else: ptr_bases = (self, POINTER(bases[0])) p = p_meta(f"POINTER({self.__name__})", ptr_bases, {}) - ctypes._pointer_type_cache[self] = p + type(self).__pointer_type__ = p class p_meta(PyCSimpleType, ct_meta): pass @@ -135,7 +135,7 @@ def __init__(self, name, bases, namespace): if isinstance(self, p_meta): return p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {}) - ctypes._pointer_type_cache[self] = p + type(self).__pointer_type__ = p class p_meta(PyCSimpleType, ct_meta): pass diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 4a0a285d3aebe3..55917bb0e6dcd9 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -564,6 +564,21 @@ _ctypes_CType_Type___sizeof___impl(PyObject *self, PyTypeObject *cls) return PyLong_FromSsize_t(size); } +static PyObject * +ctype_get_pointer_type(PyObject *self, void *Py_UNUSED(ignored)) +{ + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + StgInfo *info; + if (PyStgInfo_FromType(st, self, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL */ + if (info->pointer_type) { + return Py_NewRef(info->pointer_type); + } + Py_RETURN_NONE; +} + static PyObject * CType_Type_repeat(PyObject *self, Py_ssize_t length); @@ -573,12 +588,18 @@ static PyMethodDef ctype_methods[] = { {0}, }; +static PyGetSetDef ctype_getsets[] = { + { "__pointer_type__", ctype_get_pointer_type, NULL, "pointer type", NULL }, + { NULL, NULL } +}; + static PyType_Slot ctype_type_slots[] = { {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_traverse, CType_Type_traverse}, {Py_tp_clear, CType_Type_clear}, {Py_tp_dealloc, CType_Type_dealloc}, {Py_tp_methods, ctype_methods}, + {Py_tp_getset, ctype_getsets}, // Sequence protocol. {Py_sq_repeat, CType_Type_repeat}, {0, NULL}, @@ -1174,7 +1195,7 @@ class _ctypes.PyCPointerType "PyObject *" "clinic_state()->PyCPointerType_Type" static int -PyCPointerType_SetProto(ctypes_state *st, StgInfo *stginfo, PyObject *proto) +PyCPointerType_SetProto(ctypes_state *st, PyObject *self, StgInfo *stginfo, PyObject *proto) { if (!proto || !PyType_Check(proto)) { PyErr_SetString(PyExc_TypeError, @@ -1190,8 +1211,17 @@ PyCPointerType_SetProto(ctypes_state *st, StgInfo *stginfo, PyObject *proto) "_type_ must have storage info"); return -1; } + if (info->pointer_type) { + PyErr_Format(PyExc_TypeError, + "pointer type already set: old=%R, new=%R", + info->pointer_type, self); + return -1; + } + Py_INCREF(proto); Py_XSETREF(stginfo->proto, proto); + + Py_XSETREF(info->pointer_type, Py_NewRef(self)); return 0; } @@ -1243,7 +1273,7 @@ PyCPointerType_init(PyObject *self, PyObject *args, PyObject *kwds) } if (proto) { const char *current_format; - if (PyCPointerType_SetProto(st, stginfo, proto) < 0) { + if (PyCPointerType_SetProto(st, self, stginfo, proto) < 0) { Py_DECREF(proto); return -1; } @@ -1307,7 +1337,7 @@ PyCPointerType_set_type_impl(PyTypeObject *self, PyTypeObject *cls, return NULL; } - if (PyCPointerType_SetProto(st, info, type) < 0) { + if (PyCPointerType_SetProto(st, (PyObject *)self, info, type) < 0) { Py_DECREF(attrdict); return NULL; } diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 2f564c941f3f0a..3a2dce614ea009 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -2033,7 +2033,7 @@ create_pointer_type(PyObject *module, PyObject *cls) } if (info) { - info->pointer_type = Py_NewRef(result); + // info->pointer_type = Py_NewRef(result); } return result; From 10e36e4625ee0697d16b2b37ddf67c5034961705 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sun, 23 Mar 2025 00:20:44 +0500 Subject: [PATCH 13/65] Fix _CType_Type Co-authored-by: neonene <53406459+neonene@users.noreply.github.com> --- Lib/ctypes/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 9fcf7312c1c6d6..1e8cb82692a195 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -266,7 +266,7 @@ class c_bool(_SimpleCData): from _ctypes import POINTER, pointer, _pointer_type_cache -CType_Type = _Pointer.__base__ +_CType_Type = type(_Pointer).__base__ def POINTER(cls): if cls is None: From 29bfe9ccdf3cb499d38b3f9ae6f01ae7edc87389 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sun, 23 Mar 2025 00:25:42 +0500 Subject: [PATCH 14/65] Update Lib/test/test_ctypes/test_c_simple_type_meta.py Co-authored-by: Jun Komoda <45822440+junkmd@users.noreply.github.com> --- Lib/test/test_ctypes/test_c_simple_type_meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py index 2810bf826a543c..06cddbc6d68c15 100644 --- a/Lib/test/test_ctypes/test_c_simple_type_meta.py +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -69,7 +69,7 @@ def __new__(cls, name, bases, namespace): if isinstance(self, p_meta): return self p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {}) - cls.__pointer_type__ = p + self.__pointer_type__ = p return self class p_meta(PyCSimpleType, ct_meta): From f8139ff8385e2c459edd37afa9cc78ab6aa37156 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sun, 23 Mar 2025 00:27:44 +0500 Subject: [PATCH 15/65] Fix POINTER and test_creating_pointer_in_dunder_new_1 --- Lib/ctypes/__init__.py | 2 +- Lib/test/test_ctypes/test_c_simple_type_meta.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 1e8cb82692a195..bfd940175583ef 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -279,7 +279,7 @@ def POINTER(cls): pass if isinstance(cls, str): return type(f'LP_{cls}', (_Pointer,), {}) - if issubclass(cls, CType_Type): + if isinstance(cls, _CType_Type): return type(f'LP_{cls.__name__}', (_Pointer,), {'_type_': cls}) raise TypeError(f'must be a ctypes-like type: {cls}') diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py index 06cddbc6d68c15..831ef4d9befcc6 100644 --- a/Lib/test/test_ctypes/test_c_simple_type_meta.py +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -36,7 +36,7 @@ def __new__(cls, name, bases, namespace): else: ptr_bases = (self, POINTER(bases[0])) p = p_meta(f"POINTER({self.__name__})", ptr_bases, {}) - cls.__pointer_type__ = p + self.__pointer_type__ = p return self class p_meta(PyCSimpleType, ct_meta): From e1aaf4526ae360012feedebbdfb7c20585de37eb Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Mon, 24 Mar 2025 00:19:54 +0500 Subject: [PATCH 16/65] Do not share pointer_type via StgInfo clone Co-authored-by: neonene <53406459+neonene@users.noreply.github.com> --- Modules/_ctypes/stgdict.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 5d731c0d9353f4..da6e0150ee81ad 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -40,7 +40,7 @@ PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info) Py_XINCREF(dst_info->converters); Py_XINCREF(dst_info->restype); Py_XINCREF(dst_info->checker); - Py_XINCREF(dst_info->pointer_type); + dst_info->pointer_type = NULL; // the cache cannot be shared Py_XINCREF(dst_info->module); if (src_info->format) { From 82f74eccbff8761e693a51912cdeb2e0d439f912 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Mon, 24 Mar 2025 00:30:24 +0500 Subject: [PATCH 17/65] Test if PyCStgInfo_clone not share pointer_type cache --- Lib/test/test_ctypes/test_structures.py | 24 ++++++++++++++++++++++++ Modules/_ctypes/_ctypes.c | 22 +++++++++++++++------- Modules/_ctypes/ctypes.h | 1 + Modules/_ctypes/stgdict.c | 10 ++-------- 4 files changed, 42 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_ctypes/test_structures.py b/Lib/test/test_ctypes/test_structures.py index 0ec238e04b74cd..9a4e9ea38c860d 100644 --- a/Lib/test/test_ctypes/test_structures.py +++ b/Lib/test/test_ctypes/test_structures.py @@ -644,6 +644,30 @@ class Test8(Union): self.assertEqual(ctx.exception.args[0], 'item 1 in _argtypes_ passes ' 'a union by value, which is unsupported.') + def test_do_not_share_pointer_type_cache_via_stginfo_clone(self): + # This test case calls PyCStgInfo_clone() + # for the Mid and Vector class definitions + # and checks that pointer_type cache not shared + # between subclasses. + class Base(Structure): + _fields_ = [('y', c_double), + ('x', c_double)] + base_ptr = POINTER(Base) + + class Mid(Base): + pass + Mid._fields_ = [] + mid_ptr = POINTER(Mid) + + class Vector(Mid): + pass + + vector_ptr = POINTER(Vector) + + self.assertIsNot(base_ptr, mid_ptr) + self.assertIsNot(base_ptr, vector_ptr) + self.assertIsNot(mid_ptr, vector_ptr) + if __name__ == '__main__': unittest.main() diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 55917bb0e6dcd9..51caf4d61f5cbb 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -494,6 +494,20 @@ ctype_clear_stginfo(StgInfo *info) Py_CLEAR(info->module); // decref the module last } +void +ctype_free_stginfo_members(StgInfo *info) +{ + assert(info); + + PyMem_Free(info->ffi_type_pointer.elements); + info->ffi_type_pointer.elements = NULL; + PyMem_Free(info->format); + info->format = NULL; + PyMem_Free(info->shape); + info->shape = NULL; + ctype_clear_stginfo(info); +} + static int CType_Type_clear(PyObject *self) { @@ -517,13 +531,7 @@ CType_Type_dealloc(PyObject *self) "deallocating ctypes %R", self); } if (info) { - PyMem_Free(info->ffi_type_pointer.elements); - info->ffi_type_pointer.elements = NULL; - PyMem_Free(info->format); - info->format = NULL; - PyMem_Free(info->shape); - info->shape = NULL; - ctype_clear_stginfo(info); + ctype_free_stginfo_members(info); } PyTypeObject *tp = Py_TYPE(self); diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index ab4534b3413335..54f4e7d5d08753 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -378,6 +378,7 @@ typedef struct { extern int PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info); extern void ctype_clear_stginfo(StgInfo *info); +extern void ctype_free_stginfo_members(StgInfo *info); typedef int(* PPROC)(void); diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index da6e0150ee81ad..dc75f6d90f8a7a 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -25,13 +25,7 @@ PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info) { Py_ssize_t size; - ctype_clear_stginfo(dst_info); - PyMem_Free(dst_info->ffi_type_pointer.elements); - PyMem_Free(dst_info->format); - dst_info->format = NULL; - PyMem_Free(dst_info->shape); - dst_info->shape = NULL; - dst_info->ffi_type_pointer.elements = NULL; + ctype_free_stginfo_members(dst_info); memcpy(dst_info, src_info, sizeof(StgInfo)); @@ -40,8 +34,8 @@ PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info) Py_XINCREF(dst_info->converters); Py_XINCREF(dst_info->restype); Py_XINCREF(dst_info->checker); - dst_info->pointer_type = NULL; // the cache cannot be shared Py_XINCREF(dst_info->module); + dst_info->pointer_type = NULL; // the cache cannot be shared if (src_info->format) { dst_info->format = PyMem_Malloc(strlen(src_info->format) + 1); From eacc72474b6d646f3dcb6c7c7aec27bb4c826ffc Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Mon, 24 Mar 2025 01:16:41 +0500 Subject: [PATCH 18/65] Simplify POINTER Co-authored-by: neonene <53406459+neonene@users.noreply.github.com> --- Lib/ctypes/__init__.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index bfd940175583ef..dc4f28c1f4316d 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -266,8 +266,6 @@ class c_bool(_SimpleCData): from _ctypes import POINTER, pointer, _pointer_type_cache -_CType_Type = type(_Pointer).__base__ - def POINTER(cls): if cls is None: return c_void_p @@ -278,12 +276,8 @@ def POINTER(cls): except AttributeError: pass if isinstance(cls, str): - return type(f'LP_{cls}', (_Pointer,), {}) - if isinstance(cls, _CType_Type): - return type(f'LP_{cls.__name__}', (_Pointer,), {'_type_': cls}) - - raise TypeError(f'must be a ctypes-like type: {cls}') - + return type(f'LP_{cls}', (_Pointer,), {}) # deprecated + return type(f'LP_{cls.__name__}', (_Pointer,), {'_type_': cls}) def pointer(arg): typ = POINTER(type(arg)) return typ(arg) From 0b373d5927662abcfc17c1186738ee28d63cccff Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Mon, 24 Mar 2025 01:22:07 +0500 Subject: [PATCH 19/65] Add some extra checks and tests --- Lib/ctypes/__init__.py | 3 ++- Lib/test/test_ctypes/test_pointers.py | 25 +++++++++++++++++++++++++ Modules/_ctypes/_ctypes.c | 26 +++++++++++++++++++------- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index dc4f28c1f4316d..b2debab64c634b 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -276,8 +276,9 @@ def POINTER(cls): except AttributeError: pass if isinstance(cls, str): - return type(f'LP_{cls}', (_Pointer,), {}) # deprecated + return type(f'LP_{cls}', (_Pointer,), {}) return type(f'LP_{cls.__name__}', (_Pointer,), {'_type_': cls}) + def pointer(arg): typ = POINTER(type(arg)) return typ(arg) diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index 1f95212dc2cb33..970749ea5bab87 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -216,6 +216,31 @@ def test_pointer_type_str_name(self): def test_abstract(self): self.assertRaises(TypeError, _Pointer.set_type, 42) + def test_pointer_types_equal(self): + t1 = POINTER(c_int) + t2 = POINTER(c_int) + + self.assertIs(t1, t2) + + def test_incomplete_pointer_types_not_equal(self): + t1 = POINTER("LP_C") + t2 = POINTER("LP_C") + + self.assertIsNot(t1, t2) + + def test_pointer_set_type_twice(self): + t1 = POINTER(c_int) + t1.set_type(c_int) + + def test_pointer_set_wrong_type(self): + t1 = POINTER(c_int) + with self.assertRaisesRegex(TypeError, "pointer type already set"): + t1.set_type(c_float) + + def test_pointer_not_ctypes_type(self): + with self.assertRaisesRegex(TypeError, "must have storage info"): + POINTER(int) + if __name__ == '__main__': unittest.main() diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 51caf4d61f5cbb..e6f3289c9dab01 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -580,7 +580,11 @@ ctype_get_pointer_type(PyObject *self, void *Py_UNUSED(ignored)) if (PyStgInfo_FromType(st, self, &info) < 0) { return NULL; } - assert(info); /* Cannot be NULL */ + if (!info) { + PyErr_Format(PyExc_TypeError, "%R must have storage info", self); + return NULL; + } + if (info->pointer_type) { return Py_NewRef(info->pointer_type); } @@ -1215,21 +1219,29 @@ PyCPointerType_SetProto(ctypes_state *st, PyObject *self, StgInfo *stginfo, PyOb return -1; } if (!info) { - PyErr_SetString(PyExc_TypeError, - "_type_ must have storage info"); + PyErr_Format(PyExc_TypeError, "%R must have storage info", proto); return -1; } - if (info->pointer_type) { + if (info->pointer_type && info->pointer_type != self) { PyErr_Format(PyExc_TypeError, "pointer type already set: old=%R, new=%R", info->pointer_type, self); return -1; } + if (stginfo->proto && stginfo->proto != proto) { + PyErr_Format(PyExc_TypeError, + "cls type already set: old=%R, new=%R", + stginfo->proto, proto); + return -1; + } - Py_INCREF(proto); - Py_XSETREF(stginfo->proto, proto); + if (!stginfo->proto) { + Py_XSETREF(stginfo->proto, Py_NewRef(proto)); + } - Py_XSETREF(info->pointer_type, Py_NewRef(self)); + if (!info->pointer_type) { + Py_XSETREF(info->pointer_type, Py_NewRef(self)); + } return 0; } From 9c49abd57732a2099c3cab1a123e2b20b6b47314 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Mon, 24 Mar 2025 20:55:52 +0500 Subject: [PATCH 20/65] Fix tests for ct_meta with init --- .../test_ctypes/test_c_simple_type_meta.py | 58 ++++++++++++++++++- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py index 831ef4d9befcc6..64c5c2eec20e0e 100644 --- a/Lib/test/test_ctypes/test_c_simple_type_meta.py +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -45,20 +45,36 @@ class p_meta(PyCSimpleType, ct_meta): class PtrBase(c_void_p, metaclass=p_meta): pass + ptr_base_pointer = POINTER(PtrBase) + class CtBase(object, metaclass=ct_meta): pass + ct_base_pointer = POINTER(CtBase) + class Sub(CtBase): pass + sub_pointer = POINTER(Sub) + class Sub2(Sub): pass + sub2_pointer = POINTER(Sub2) + + self.assertIsNot(ptr_base_pointer, ct_base_pointer) + self.assertIsNot(ct_base_pointer, sub_pointer) + self.assertIsNot(sub_pointer, sub2_pointer) + self.assertIsInstance(POINTER(Sub2), p_meta) self.assertIsSubclass(POINTER(Sub2), Sub2) self.assertIsSubclass(POINTER(Sub2), POINTER(Sub)) self.assertIsSubclass(POINTER(Sub), POINTER(CtBase)) + self.assertIs(POINTER(Sub2), sub2_pointer) + self.assertIs(POINTER(Sub), sub_pointer) + self.assertIs(POINTER(CtBase), ct_base_pointer) + def test_creating_pointer_in_dunder_new_2(self): # A simpler variant of the above, used in `CoClass` of the `comtypes` # project. @@ -78,15 +94,27 @@ class p_meta(PyCSimpleType, ct_meta): class Core(object): pass + with self.assertRaisesRegex(TypeError, "must have storage info"): + POINTER(Core) + class CtBase(Core, metaclass=ct_meta): pass + ct_base_pointer = POINTER(CtBase) + class Sub(CtBase): pass + sub_pointer = POINTER(Sub) + + self.assertIsNot(ct_base_pointer, sub_pointer) + self.assertIsInstance(POINTER(Sub), p_meta) self.assertIsSubclass(POINTER(Sub), Sub) + self.assertIs(POINTER(Sub), sub_pointer) + self.assertIs(POINTER(CtBase), ct_base_pointer) + def test_creating_pointer_in_dunder_init_1(self): class ct_meta(type): def __init__(self, name, bases, namespace): @@ -103,7 +131,7 @@ def __init__(self, name, bases, namespace): else: ptr_bases = (self, POINTER(bases[0])) p = p_meta(f"POINTER({self.__name__})", ptr_bases, {}) - type(self).__pointer_type__ = p + self.__pointer_type__ = p class p_meta(PyCSimpleType, ct_meta): pass @@ -111,20 +139,37 @@ class p_meta(PyCSimpleType, ct_meta): class PtrBase(c_void_p, metaclass=p_meta): pass + ptr_base_pointer = POINTER(PtrBase) + class CtBase(object, metaclass=ct_meta): pass + ct_base_pointer = POINTER(CtBase) + class Sub(CtBase): pass + sub_pointer = POINTER(Sub) + class Sub2(Sub): pass + sub2_pointer = POINTER(Sub2) + + self.assertIsNot(ptr_base_pointer, ct_base_pointer) + self.assertIsNot(ct_base_pointer, sub_pointer) + self.assertIsNot(sub_pointer, sub2_pointer) + self.assertIsInstance(POINTER(Sub2), p_meta) self.assertIsSubclass(POINTER(Sub2), Sub2) self.assertIsSubclass(POINTER(Sub2), POINTER(Sub)) self.assertIsSubclass(POINTER(Sub), POINTER(CtBase)) + self.assertIs(POINTER(PtrBase), ptr_base_pointer) + self.assertIs(POINTER(CtBase), ct_base_pointer) + self.assertIs(POINTER(Sub), sub_pointer) + self.assertIs(POINTER(Sub2), sub2_pointer) + def test_creating_pointer_in_dunder_init_2(self): class ct_meta(type): def __init__(self, name, bases, namespace): @@ -135,7 +180,7 @@ def __init__(self, name, bases, namespace): if isinstance(self, p_meta): return p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {}) - type(self).__pointer_type__ = p + self.__pointer_type__ = p class p_meta(PyCSimpleType, ct_meta): pass @@ -146,12 +191,21 @@ class Core(object): class CtBase(Core, metaclass=ct_meta): pass + ct_base_pointer = POINTER(CtBase) + class Sub(CtBase): pass + sub_pointer = POINTER(Sub) + + self.assertIsNot(ct_base_pointer, sub_pointer) + self.assertIsInstance(POINTER(Sub), p_meta) self.assertIsSubclass(POINTER(Sub), Sub) + self.assertIs(POINTER(CtBase), ct_base_pointer) + self.assertIs(POINTER(Sub), sub_pointer) + def test_bad_type_message(self): """Verify the error message that lists all available type codes""" # (The string is generated at runtime, so this checks the underlying From 3c38aa51351a32a6676f06a175f2f7f92175250a Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Mon, 24 Mar 2025 22:22:06 +0500 Subject: [PATCH 21/65] Do not use Py_XSETREF if dst already NULL --- Modules/_ctypes/_ctypes.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index e6f3289c9dab01..c47b8891cd1b7b 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1236,11 +1236,11 @@ PyCPointerType_SetProto(ctypes_state *st, PyObject *self, StgInfo *stginfo, PyOb } if (!stginfo->proto) { - Py_XSETREF(stginfo->proto, Py_NewRef(proto)); + stginfo->proto = Py_NewRef(proto); } if (!info->pointer_type) { - Py_XSETREF(info->pointer_type, Py_NewRef(self)); + info->pointer_type = Py_NewRef(self); } return 0; } From e251a7d15f971b37434a8633887c2ca4f4069696 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Mon, 24 Mar 2025 22:32:53 +0500 Subject: [PATCH 22/65] Add docstrings for POINTER and pointer --- Lib/ctypes/__init__.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 19d13be44888f9..040e54bbe66f61 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -268,6 +268,19 @@ class c_bool(_SimpleCData): from _ctypes import POINTER, pointer, _pointer_type_cache def POINTER(cls): + """ + Create and return a new ctypes pointer type. + + cls + A ctypes type. + + Pointer types are cached and reused internally, + so calling this function repeatedly is cheap. + + Pointer types for incomplete types are not cached, + so calling this function repeatedly will give + different types. + """ if cls is None: return c_void_p try: @@ -280,9 +293,16 @@ def POINTER(cls): return type(f'LP_{cls}', (_Pointer,), {}) return type(f'LP_{cls.__name__}', (_Pointer,), {'_type_': cls}) -def pointer(arg): - typ = POINTER(type(arg)) - return typ(arg) +def pointer(obj): + """ + Create a new pointer instance, pointing to 'obj'. + + The returned object is of the type POINTER(type(obj)). Note that if you + just want to pass a pointer to an object to a foreign function call, you + should use byref(obj) which is much faster. + """ + typ = POINTER(type(obj)) + return typ(obj) class c_wchar_p(_SimpleCData): _type_ = "Z" From cdba1f65406300eff6181f61002679d11b2ae586 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Mon, 24 Mar 2025 22:34:23 +0500 Subject: [PATCH 23/65] Remove import of c-versions of POINTER and pointer --- Lib/ctypes/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 040e54bbe66f61..df066f7ae1c60a 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -265,8 +265,6 @@ class c_void_p(_SimpleCData): class c_bool(_SimpleCData): _type_ = "?" -from _ctypes import POINTER, pointer, _pointer_type_cache - def POINTER(cls): """ Create and return a new ctypes pointer type. @@ -304,6 +302,9 @@ def pointer(obj): typ = POINTER(type(obj)) return typ(obj) +_pointer_type_cache = {} +"""XXX: Subject to change.""" + class c_wchar_p(_SimpleCData): _type_ = "Z" def __repr__(self): From a241cd912e7750e3c0503ca23a3bbbf7d04b9ae3 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Mon, 24 Mar 2025 22:36:20 +0500 Subject: [PATCH 24/65] Remove c-versions of POINTER and pointer --- Modules/_ctypes/callproc.c | 101 ---------------------------- Modules/_ctypes/clinic/callproc.c.h | 32 --------- 2 files changed, 133 deletions(-) delete mode 100644 Modules/_ctypes/clinic/callproc.c.h diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 428712d8461fa9..7d619a5d08ce1d 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -107,8 +107,6 @@ module _ctypes #include "../_complex.h" // complex #endif -#include "clinic/callproc.c.h" - #define CTYPES_CAPSULE_NAME_PYMEM "_ctypes pymem" @@ -1971,103 +1969,6 @@ unpickle(PyObject *self, PyObject *args) return NULL; } -/*[clinic input] -_ctypes.POINTER as create_pointer_type - - type as cls: object - A ctypes type. - / - -Create and return a new ctypes pointer type. - -Pointer types are cached and reused internally, -so calling this function repeatedly is cheap. -[clinic start generated code]*/ - -static PyObject * -create_pointer_type(PyObject *module, PyObject *cls) -/*[clinic end generated code: output=98c3547ab6f4f40b input=3b81cff5ff9b9d5b]*/ -{ - PyObject *result; - PyTypeObject *typ; - - assert(module); - ctypes_state *st = get_module_state(module); - StgInfo *info = NULL; - if (PyType_Check(cls)) { - if (PyStgInfo_FromType(st, cls, &info) < 0) { - return NULL; - } - if (info && info->pointer_type) { - return Py_NewRef(info->pointer_type); - } - } - - if (PyDict_GetItemRef(st->_ctypes_ptrtype_cache, cls, &result) != 0) { - // found or error - return result; - } - - // not found - if (PyUnicode_CheckExact(cls)) { - PyObject *name = PyUnicode_FromFormat("LP_%U", cls); - result = PyObject_CallFunction((PyObject *)Py_TYPE(st->PyCPointer_Type), - "N(O){}", - name, - st->PyCPointer_Type); - if (result == NULL) - return result; - } else if (PyType_Check(cls)) { - typ = (PyTypeObject *)cls; - PyObject *name = PyUnicode_FromFormat("LP_%s", typ->tp_name); - result = PyObject_CallFunction((PyObject *)Py_TYPE(st->PyCPointer_Type), - "N(O){sO}", - name, - st->PyCPointer_Type, - "_type_", cls); - if (result == NULL) - return result; - } else { - PyErr_SetString(PyExc_TypeError, "must be a ctypes type"); - return NULL; - } - - if (info) { - // info->pointer_type = Py_NewRef(result); - } - - return result; -} - -/*[clinic input] -_ctypes.pointer as create_pointer_inst - - obj as arg: object - / - -Create a new pointer instance, pointing to 'obj'. - -The returned object is of the type POINTER(type(obj)). Note that if you -just want to pass a pointer to an object to a foreign function call, you -should use byref(obj) which is much faster. -[clinic start generated code]*/ - -static PyObject * -create_pointer_inst(PyObject *module, PyObject *arg) -/*[clinic end generated code: output=3b543bc9f0de2180 input=713685fdb4d9bc27]*/ -{ - PyObject *result; - PyObject *typ; - - typ = create_pointer_type(module, (PyObject *)Py_TYPE(arg)); - if (typ == NULL) - return NULL; - - result = PyObject_CallOneArg(typ, arg); - Py_DECREF(typ); - return result; -} - static PyObject * buffer_info(PyObject *self, PyObject *arg) { @@ -2102,8 +2003,6 @@ buffer_info(PyObject *self, PyObject *arg) PyMethodDef _ctypes_module_methods[] = { {"get_errno", get_errno, METH_NOARGS}, {"set_errno", set_errno, METH_VARARGS}, - CREATE_POINTER_TYPE_METHODDEF - CREATE_POINTER_INST_METHODDEF {"_unpickle", unpickle, METH_VARARGS }, {"buffer_info", buffer_info, METH_O, "Return buffer interface information"}, {"resize", resize, METH_VARARGS, "Resize the memory buffer of a ctypes instance"}, diff --git a/Modules/_ctypes/clinic/callproc.c.h b/Modules/_ctypes/clinic/callproc.c.h deleted file mode 100644 index a787693ae67cd8..00000000000000 --- a/Modules/_ctypes/clinic/callproc.c.h +++ /dev/null @@ -1,32 +0,0 @@ -/*[clinic input] -preserve -[clinic start generated code]*/ - -PyDoc_STRVAR(create_pointer_type__doc__, -"POINTER($module, type, /)\n" -"--\n" -"\n" -"Create and return a new ctypes pointer type.\n" -"\n" -" type\n" -" A ctypes type.\n" -"\n" -"Pointer types are cached and reused internally,\n" -"so calling this function repeatedly is cheap."); - -#define CREATE_POINTER_TYPE_METHODDEF \ - {"POINTER", (PyCFunction)create_pointer_type, METH_O, create_pointer_type__doc__}, - -PyDoc_STRVAR(create_pointer_inst__doc__, -"pointer($module, obj, /)\n" -"--\n" -"\n" -"Create a new pointer instance, pointing to \'obj\'.\n" -"\n" -"The returned object is of the type POINTER(type(obj)). Note that if you\n" -"just want to pass a pointer to an object to a foreign function call, you\n" -"should use byref(obj) which is much faster."); - -#define CREATE_POINTER_INST_METHODDEF \ - {"pointer", (PyCFunction)create_pointer_inst, METH_O, create_pointer_inst__doc__}, -/*[clinic end generated code: output=51b311ea369e5adf input=a9049054013a1b77]*/ From 8d856246185fe11a22b9e7d9977c10a4b5da0eaa Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Mon, 24 Mar 2025 22:37:25 +0500 Subject: [PATCH 25/65] Remove c-versions of _pointer_type_cache/_ctypes_ptrtype_cache --- Modules/_ctypes/_ctypes.c | 8 -------- Modules/_ctypes/ctypes.h | 2 -- 2 files changed, 10 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 590728eee7fba0..0b035da63d2bc4 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -6080,7 +6080,6 @@ _ctypes_add_objects(PyObject *mod) } while (0) ctypes_state *st = get_module_state(mod); - MOD_ADD("_pointer_type_cache", Py_NewRef(st->_ctypes_ptrtype_cache)); #ifdef MS_WIN32 MOD_ADD("COMError", Py_NewRef(st->PyComError_Type)); @@ -6141,11 +6140,6 @@ _ctypes_mod_exec(PyObject *mod) return -1; } - st->_ctypes_ptrtype_cache = PyDict_New(); - if (st->_ctypes_ptrtype_cache == NULL) { - return -1; - } - st->PyExc_ArgError = PyErr_NewException("ctypes.ArgumentError", NULL, NULL); if (!st->PyExc_ArgError) { return -1; @@ -6165,7 +6159,6 @@ _ctypes_mod_exec(PyObject *mod) static int module_traverse(PyObject *module, visitproc visit, void *arg) { ctypes_state *st = get_module_state(module); - Py_VISIT(st->_ctypes_ptrtype_cache); Py_VISIT(st->_unpickle); Py_VISIT(st->array_cache); Py_VISIT(st->error_object_name); @@ -6200,7 +6193,6 @@ module_traverse(PyObject *module, visitproc visit, void *arg) { static int module_clear(PyObject *module) { ctypes_state *st = get_module_state(module); - Py_CLEAR(st->_ctypes_ptrtype_cache); Py_CLEAR(st->_unpickle); Py_CLEAR(st->array_cache); Py_CLEAR(st->error_object_name); diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 5483b599ebe984..8a4a9dfa9142aa 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -81,8 +81,6 @@ typedef struct { #ifdef MS_WIN32 PyTypeObject *PyComError_Type; #endif - /* This dict maps ctypes types to POINTER types */ - PyObject *_ctypes_ptrtype_cache; /* a callable object used for unpickling: strong reference to _ctypes._unpickle() function */ PyObject *_unpickle; From aedc4b2e0d544119b89657adce60108c32c95e97 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Tue, 25 Mar 2025 01:21:24 +0500 Subject: [PATCH 26/65] Add extra test for set_type/PyCPointerType_SetProto --- Lib/test/test_ctypes/test_pointers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index 970749ea5bab87..e86bf43f8ec8f8 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -241,6 +241,10 @@ def test_pointer_not_ctypes_type(self): with self.assertRaisesRegex(TypeError, "must have storage info"): POINTER(int) + def test_pointer_set_python_type(self): + p1 = POINTER(c_int) + with self.assertRaisesRegex(TypeError, "must have storage info"): + p1.set_type(int) if __name__ == '__main__': unittest.main() From 6ac84c70e7effdfe846f4b7d7450646317eb6f51 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Tue, 25 Mar 2025 01:22:44 +0500 Subject: [PATCH 27/65] Add test for creating types with factory --- Lib/test/test_ctypes/test_pointers.py | 31 +++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index e86bf43f8ec8f8..5d3810a4ce0466 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -1,5 +1,6 @@ import array import ctypes +import gc import sys import unittest from ctypes import (CDLL, CFUNCTYPE, Structure, @@ -10,6 +11,7 @@ c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) from test.support import import_helper +from weakref import WeakSet _ctypes_test = import_helper.import_module("_ctypes_test") from ._support import (_CData, PyCPointerType, Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE) @@ -246,5 +248,34 @@ def test_pointer_set_python_type(self): with self.assertRaisesRegex(TypeError, "must have storage info"): p1.set_type(int) + def test_pointer_types_factory(self): + """Shouldn't leak""" + def factory(): + class Cls(Structure): + _fields_ = ( + ('a', c_int), + ('b', c_float), + ) + + return Cls + + ws_typ = WeakSet() + ws_ptr = WeakSet() + for _ in range(10): + typ = factory() + ptr = POINTER(typ) + + ws_typ.add(typ) + ws_ptr.add(ptr) + + typ = None + ptr = None + + gc.collect() + + self.assertEqual(len(ws_typ), 0, ws_typ) + self.assertEqual(len(ws_ptr), 0, ws_ptr) + + if __name__ == '__main__': unittest.main() From 2373c63e767da2f60e4b3a684eb689fe74ae7694 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Tue, 25 Mar 2025 22:37:18 +0500 Subject: [PATCH 28/65] Fix docstrings --- Lib/ctypes/__init__.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index df066f7ae1c60a..74c2fc707e6263 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -266,11 +266,7 @@ class c_bool(_SimpleCData): _type_ = "?" def POINTER(cls): - """ - Create and return a new ctypes pointer type. - - cls - A ctypes type. + """Create and return a new ctypes pointer type. Pointer types are cached and reused internally, so calling this function repeatedly is cheap. @@ -292,8 +288,7 @@ def POINTER(cls): return type(f'LP_{cls.__name__}', (_Pointer,), {'_type_': cls}) def pointer(obj): - """ - Create a new pointer instance, pointing to 'obj'. + """Create a new pointer instance, pointing to 'obj'. The returned object is of the type POINTER(type(obj)). Note that if you just want to pass a pointer to an object to a foreign function call, you From fc93bc83eaa52414c46ab9fd32711f8ddf3a16a8 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 26 Mar 2025 00:01:59 +0500 Subject: [PATCH 29/65] Add some tests for pointer --- Lib/test/test_ctypes/test_pointers.py | 33 ++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index 5d3810a4ce0466..439b65f2b9948b 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -127,7 +127,10 @@ def test_basics(self): def test_from_address(self): a = array.array('i', [100, 200, 300, 400, 500]) addr = a.buffer_info()[0] - p = POINTER(POINTER(c_int)) + p1 = POINTER(c_int) + p2 = POINTER(p1) + + self.assertIsNot(p1, p2) def test_other(self): class Table(Structure): @@ -224,15 +227,37 @@ def test_pointer_types_equal(self): self.assertIs(t1, t2) + p1 = t1(c_int(1)) + p2 = t2(c_int(1)) + p3 = pointer(c_int(1)) + + self.assertIsInstance(p1, t1) + self.assertIsInstance(p2, t1) + self.assertIsInstance(p3, t1) + def test_incomplete_pointer_types_not_equal(self): t1 = POINTER("LP_C") t2 = POINTER("LP_C") self.assertIsNot(t1, t2) + def test_incomplete_pointer_types_cannot_instantiate(self): + t1 = POINTER("LP_C") + with self.assertRaisesRegex(TypeError, "has no _type_"): + t1() + + msg = " must have storage info" + with self.assertRaisesRegex(TypeError, msg): + pointer("LP_C") + def test_pointer_set_type_twice(self): t1 = POINTER(c_int) + self.assertIs(c_int.__pointer_type__, t1) + self.assertIs(t1._type_, c_int) + t1.set_type(c_int) + self.assertIs(c_int.__pointer_type__, t1) + self.assertIs(t1._type_, c_int) def test_pointer_set_wrong_type(self): t1 = POINTER(c_int) @@ -243,6 +268,12 @@ def test_pointer_not_ctypes_type(self): with self.assertRaisesRegex(TypeError, "must have storage info"): POINTER(int) + with self.assertRaisesRegex(TypeError, "must have storage info"): + pointer(int) + + with self.assertRaisesRegex(TypeError, "must have storage info"): + pointer(int(1)) + def test_pointer_set_python_type(self): p1 = POINTER(c_int) with self.assertRaisesRegex(TypeError, "must have storage info"): From 57683479bfa1da811ca07837d4949353874dcb46 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 26 Mar 2025 00:14:13 +0500 Subject: [PATCH 30/65] Update news and whatsnew --- Doc/whatsnew/3.14.rst | 3 ++- .../Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 8b9ba78bf5c742..71e8e67afcd0f3 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -532,7 +532,8 @@ ctypes * Move :func:`ctypes.POINTER` types cache from the global ``_pointer_type_cache`` to the corresponding :mod:`ctypes` types. This will stop the cache from - growing without limits in some situations. + growing without limits in some situations. Also :mod:`ctypes` pointer types + can be accessed with ``__pointer_type__`` attribute from corresponding type. (Contributed by Sergey Miryanov in :gh:`100926`). datetime diff --git a/Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst b/Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst index dfee62f2f1a4d3..831b896fb81cde 100644 --- a/Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst +++ b/Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst @@ -1,2 +1,7 @@ Move :func:`ctypes.POINTER` types cache from the global ``_pointer_type_cache`` to the corresponding :mod:`ctypes` types to avoid unlimited growth of the cache. +As from now, :mod:`ctypes` pointer types can be accessed with +``__pointer_type__`` attribute. :func:`ctypes.POINTER` checks this attribute +first before creating a new pointer type, so if needed to mimic :mod:`ctypes` +pointer type then type should have this attribute +(see PyCSimpleTypeAsMetaclassTest for example). From 87f8cf350a9e8e3d0021f6854eb90a6f5fe877a5 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 26 Mar 2025 22:36:38 +0500 Subject: [PATCH 31/65] Fix tests --- Lib/test/test_ctypes/test_pointers.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index 439b65f2b9948b..17ddc1504f9b74 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -127,10 +127,15 @@ def test_basics(self): def test_from_address(self): a = array.array('i', [100, 200, 300, 400, 500]) addr = a.buffer_info()[0] + p = POINTER(POINTER(c_int)) + + def test_pointer_from_pointer(self): p1 = POINTER(c_int) p2 = POINTER(p1) self.assertIsNot(p1, p2) + self.assertIs(p1.__pointer_type__, p2) + self.assertIs(p2._type_, p1) def test_other(self): class Table(Structure): @@ -228,12 +233,13 @@ def test_pointer_types_equal(self): self.assertIs(t1, t2) p1 = t1(c_int(1)) - p2 = t2(c_int(1)) - p3 = pointer(c_int(1)) + p2 = pointer(c_int(1)) self.assertIsInstance(p1, t1) self.assertIsInstance(p2, t1) - self.assertIsInstance(p3, t1) + + self.assertIs(type(p1), t1) + self.assertIs(type(p2), t1) def test_incomplete_pointer_types_not_equal(self): t1 = POINTER("LP_C") @@ -246,10 +252,6 @@ def test_incomplete_pointer_types_cannot_instantiate(self): with self.assertRaisesRegex(TypeError, "has no _type_"): t1() - msg = " must have storage info" - with self.assertRaisesRegex(TypeError, msg): - pointer("LP_C") - def test_pointer_set_type_twice(self): t1 = POINTER(c_int) self.assertIs(c_int.__pointer_type__, t1) From 85ff1a0a527b9ad416f1386922d1ec062ff3a0f0 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 26 Mar 2025 22:53:05 +0500 Subject: [PATCH 32/65] Add few more tests --- Lib/test/test_ctypes/test_pointers.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index 17ddc1504f9b74..57b53057597a5e 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -183,6 +183,7 @@ def test_bug_1467852(self): q = pointer(y) pp[0] = q # <== self.assertEqual(p[0], 6) + def test_c_void_p(self): # http://sourceforge.net/tracker/?func=detail&aid=1518190&group_id=5470&atid=105470 if sizeof(c_void_p) == 4: @@ -201,6 +202,30 @@ def test_c_void_p(self): self.assertRaises(TypeError, c_void_p, 3.14) # make sure floats are NOT accepted self.assertRaises(TypeError, c_void_p, object()) # nor other objects + def test_read_null_pointer(self): + null_ptr = POINTER(c_int)() + with self.assertRaisesRegex(ValueError, "NULL pointer access"): + null_ptr[0] + + def test_write_null_pointer(self): + null_ptr = POINTER(c_int)() + with self.assertRaisesRegex(ValueError, "NULL pointer access"): + null_ptr[0] = 1 + + def test_set_pointer_to_null_and_read(self): + class Bar(Structure): + _fields_ = [("values", POINTER(c_int))] + + bar = Bar() + bar.values = (c_int * 3)(1, 2, 3) + + values = [bar.values[0], bar.values[1], bar.values[2]] + self.assertEqual(values, [1, 2, 3]) + + bar.values = None + with self.assertRaisesRegex(ValueError, "NULL pointer access"): + bar.values[0] + def test_pointers_bool(self): # NULL pointers have a boolean False value, non-NULL pointers True. self.assertEqual(bool(POINTER(c_int)()), False) From d11af803c2d6ad056533289dd559b7332168f60b Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 26 Mar 2025 23:12:25 +0500 Subject: [PATCH 33/65] Add some more tests --- Lib/test/test_ctypes/test_pointers.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index 57b53057597a5e..718bfe26f9a5a0 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -306,6 +306,17 @@ def test_pointer_set_python_type(self): with self.assertRaisesRegex(TypeError, "must have storage info"): p1.set_type(int) + def test_pointer_type_attribute_is_none(self): + class Cls(Structure): + _fields_ = ( + ('a', c_int), + ('b', c_float), + ) + + self.assertIsNone(Cls.__pointer_type__) + p = POINTER(Cls) + self.assertIs(Cls.__pointer_type__, p) + def test_pointer_types_factory(self): """Shouldn't leak""" def factory(): From 6dd1e1a2b0476fa92200f4a0409a42e16d3b4910 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 26 Mar 2025 23:43:41 +0500 Subject: [PATCH 34/65] Update docs --- Doc/library/ctypes.rst | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 1a7b456a8fc6ab..c7b2394a158417 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -871,6 +871,14 @@ invalid non-\ ``NULL`` pointers would crash Python):: ValueError: NULL pointer access >>> +.. _ctypes-pointers-ctypes-like-types: + +Pointers for ctypes-like types +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In some cases you want that :func:`POINTER` can accept non-ctypes types. To do so +just add to your types a class level attribute with the name ``__pointer_type__``. + .. _ctypes-thread-safety: Thread safety without the GIL @@ -2171,9 +2179,10 @@ Utility functions .. function:: POINTER(type, /) - Create and return a new ctypes pointer type. Pointer types are cached and + Create or return a ctypes pointer type. Pointer types are cached and reused internally, so calling this function repeatedly is cheap. - *type* must be a ctypes type. + *type* must be a ctypes-like type. The ctypes-like type is a type that + has class level attribute with the name ``__pointer_type__``. .. function:: pointer(obj, /) @@ -2360,6 +2369,14 @@ Data types valid. This object is only exposed for debugging; never modify the contents of this dictionary. + .. attribute:: __pointer_type__ + + This attributes is a pointer type that was created by calling + :func:`POINTER` for corresponding ctypes data type. If ``POINTER`` was + not called then this attribute contains ``None``. + + .. versionadded:: 3.14 + .. _ctypes-fundamental-data-types-2: From 33ae0387fab00071d36d11265a3851ca33233e61 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 26 Mar 2025 23:58:19 +0500 Subject: [PATCH 35/65] Update news --- .../Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst b/Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst index 831b896fb81cde..b62b2bfaa3ae13 100644 --- a/Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst +++ b/Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst @@ -1,7 +1,7 @@ Move :func:`ctypes.POINTER` types cache from the global ``_pointer_type_cache`` to the corresponding :mod:`ctypes` types to avoid unlimited growth of the cache. -As from now, :mod:`ctypes` pointer types can be accessed with -``__pointer_type__`` attribute. :func:`ctypes.POINTER` checks this attribute +As from now, :mod:`ctypes` pointer types can be accessed with a +:attr:`~_CData.__pointer_type__` attribute. :func:`ctypes.POINTER` checks this attribute first before creating a new pointer type, so if needed to mimic :mod:`ctypes` pointer type then type should have this attribute -(see PyCSimpleTypeAsMetaclassTest for example). +(see :ref:`ctypes-pointers-ctypes-like-types`). From 08bdadaac0ac14a6580b503adea37e0569d9026b Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Thu, 27 Mar 2025 00:04:28 +0500 Subject: [PATCH 36/65] Fix news --- .../next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst b/Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst index b62b2bfaa3ae13..555043b4cccadb 100644 --- a/Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst +++ b/Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst @@ -1,7 +1,7 @@ Move :func:`ctypes.POINTER` types cache from the global ``_pointer_type_cache`` to the corresponding :mod:`ctypes` types to avoid unlimited growth of the cache. As from now, :mod:`ctypes` pointer types can be accessed with a -:attr:`~_CData.__pointer_type__` attribute. :func:`ctypes.POINTER` checks this attribute +``__pointer_type__`` attribute. :func:`ctypes.POINTER` checks this attribute first before creating a new pointer type, so if needed to mimic :mod:`ctypes` pointer type then type should have this attribute (see :ref:`ctypes-pointers-ctypes-like-types`). From 8505d4bae3e2076aeb6bc4dd9d910c84038d55b7 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Fri, 28 Mar 2025 01:56:16 +0500 Subject: [PATCH 37/65] Add some more tests --- Lib/test/test_ctypes/test_pointers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index 718bfe26f9a5a0..29e954b8a11437 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -287,10 +287,16 @@ def test_pointer_set_type_twice(self): self.assertIs(t1._type_, c_int) def test_pointer_set_wrong_type(self): + class C(c_int): + pass + t1 = POINTER(c_int) with self.assertRaisesRegex(TypeError, "pointer type already set"): t1.set_type(c_float) + with self.assertRaisesRegex(TypeError, "cls type already set"): + t1.set_type(C) + def test_pointer_not_ctypes_type(self): with self.assertRaisesRegex(TypeError, "must have storage info"): POINTER(int) From c1bf7ccf1fa5c03d2ec82c45048a2132c688fd97 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sun, 30 Mar 2025 12:54:52 +0500 Subject: [PATCH 38/65] Add more tests for metabases pointers --- .../test_ctypes/test_c_simple_type_meta.py | 125 +++++++++++++++++- 1 file changed, 123 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py index 64c5c2eec20e0e..6df6eed61f1144 100644 --- a/Lib/test/test_ctypes/test_c_simple_type_meta.py +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -1,9 +1,9 @@ import unittest from test.support import MS_WINDOWS import ctypes -from ctypes import POINTER, c_void_p +from ctypes import POINTER, Structure, c_void_p -from ._support import PyCSimpleType +from ._support import PyCSimpleType, PyCPointerType, PyCStructType class PyCSimpleTypeAsMetaclassTest(unittest.TestCase): @@ -222,3 +222,124 @@ class F(metaclass=PyCSimpleType): if not MS_WINDOWS: expected_type_chars.remove('X') self.assertIn("'" + ''.join(expected_type_chars) + "'", message) + + def test_creating_pointer_in_dunder_init_3(self): + + class StructureMeta(PyCStructType): + def __new__(cls, name, bases, dct, /, create_pointer_type=True): + if len(bases) > 1: + bases = (bases[0],) + + return super().__new__(cls, name, bases, dct) + + def __init__(self, name, bases, dct, /, create_pointer_type=True): + + super().__init__(name, bases, dct) + if create_pointer_type: + p_bases = (POINTER(bases[0]),) + p = PointerMeta(f"p{name}", p_bases, {'_type_': self}) + assert isinstance(p, PyCPointerType) + assert self.__pointer_type__ is not None + assert self.__pointer_type__ == p + + + class PointerMeta(PyCPointerType): + def __new__(cls, name, bases, dct): + target = dct.get('_type_', None) + if target is None: + + # Create corresponding interface type and then set it as target + target = StructureMeta( + f"_{name}_", + (bases[0]._type_,), + {}, + create_pointer_type=False + ) + dct['_type_'] = target + + pointer_type = super().__new__(cls, name, bases, dct) + assert target.__pointer_type__ is None + + return pointer_type + + def __init__(self, name, bases, dct, /, create_pointer_type=True): + target = dct.get('_type_', None) + super().__init__(name, bases, dct) + assert target.__pointer_type__ is self + + + class Interface(Structure, metaclass=StructureMeta, create_pointer_type=False): + pass + + class pInterface(POINTER(c_void_p), metaclass=PointerMeta): + _type_ = Interface + + class IUnknown(Interface): + pass + + class pIUnknown(pInterface): + pass + + self.assertTrue(issubclass(POINTER(IUnknown), pInterface)) + self.assertTrue(issubclass(pIUnknown, pInterface)) + + self.assertIs(POINTER(Interface), pInterface) + self.assertIsNot(POINTER(IUnknown), pIUnknown) + + def test_creating_pointer_in_dunder_init_4(self): + + class StructureMeta(PyCStructType): + def __new__(cls, name, bases, dct, /, create_pointer_type=True): + if len(bases) > 1: + bases = (bases[0],) + + return super().__new__(cls, name, bases, dct) + + def __init__(self, name, bases, dct, /, create_pointer_type=True): + + super().__init__(name, bases, dct) + if create_pointer_type: + p_bases = (POINTER(bases[0]),) + p = PointerMeta(f"p{name}", p_bases, {'_type_': self}) + assert isinstance(p, PyCPointerType) + assert self.__pointer_type__ is not None + assert self.__pointer_type__ == p + + + class PointerMeta(PyCPointerType): + def __new__(cls, name, bases, dct): + target = dct.get('_type_', None) + assert target is not None + pointer_type = target.__pointer_type__ + + if pointer_type is None: + pointer_type = super().__new__(cls, name, bases, dct) + + return pointer_type + + def __init__(self, name, bases, dct, /, create_pointer_type=True): + target = dct.get('_type_', None) + if target.__pointer_type__ is None: + # target.__pointer_type__ was created by super().__new__ + super().__init__(name, bases, dct) + + assert target.__pointer_type__ is self + + + class Interface(Structure, metaclass=StructureMeta, create_pointer_type=False): + pass + + class pInterface(POINTER(c_void_p), metaclass=PointerMeta): + _type_ = Interface + + class IUnknown(Interface): + pass + + class pIUnknown(pInterface): + _type_ = IUnknown + + self.assertTrue(issubclass(POINTER(IUnknown), pInterface)) + self.assertTrue(issubclass(pIUnknown, pInterface)) + + self.assertIs(POINTER(Interface), pInterface) + self.assertIs(POINTER(IUnknown), pIUnknown) From 62d2deb5f1d6086faf3bbcba8128c3f7ecdedd95 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sun, 30 Mar 2025 21:40:16 +0500 Subject: [PATCH 39/65] Try to add thread safety to pointer_type --- Modules/_ctypes/_ctypes.c | 36 +++++++++++++++++++++++++----------- Modules/_ctypes/ctypes.h | 4 ++++ 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 74255d135eb252..b5fd96b27920c2 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -588,8 +588,9 @@ ctype_get_pointer_type(PyObject *self, void *Py_UNUSED(ignored)) return NULL; } - if (info->pointer_type) { - return Py_NewRef(info->pointer_type); + PyObject* pointer_type = FT_ATOMIC_LOAD_PTR_ACQUIRE(info->pointer_type); + if (pointer_type) { + return Py_NewRef(pointer_type); } Py_RETURN_NONE; } @@ -1227,26 +1228,39 @@ PyCPointerType_SetProto(ctypes_state *st, PyObject *self, StgInfo *stginfo, PyOb PyErr_Format(PyExc_TypeError, "%R must have storage info", proto); return -1; } - if (info->pointer_type && info->pointer_type != self) { + + PyObject* pointer_type = FT_ATOMIC_LOAD_PTR_ACQUIRE(info->pointer_type); + PyObject* stginfo_proto = FT_ATOMIC_LOAD_PTR_ACQUIRE(stginfo->proto); + + if (pointer_type && pointer_type != self) { PyErr_Format(PyExc_TypeError, "pointer type already set: old=%R, new=%R", - info->pointer_type, self); + pointer_type, self); return -1; } - if (stginfo->proto && stginfo->proto != proto) { + if (stginfo_proto && stginfo_proto != proto) { PyErr_Format(PyExc_TypeError, "cls type already set: old=%R, new=%R", - stginfo->proto, proto); + stginfo_proto, proto); return -1; } - if (!stginfo->proto) { - stginfo->proto = Py_NewRef(proto); - } + if (!stginfo_proto || !pointer_type) { + STGINFO2_LOCK(stginfo, info); + + stginfo_proto = FT_ATOMIC_LOAD_PTR_ACQUIRE(stginfo->proto); + if (!stginfo_proto) { + FT_ATOMIC_STORE_PTR_RELEASE(stginfo->proto, Py_NewRef(proto)); + } + + pointer_type = FT_ATOMIC_LOAD_PTR_ACQUIRE(info->pointer_type); + if (!pointer_type) { + FT_ATOMIC_STORE_PTR_RELEASE(info->pointer_type, Py_NewRef(self)); + } - if (!info->pointer_type) { - info->pointer_type = Py_NewRef(self); + STGINFO2_UNLOCK(); } + return 0; } diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index cca81e7e016605..cdd24c38bb37d9 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -422,6 +422,10 @@ typedef struct { #define STGINFO_LOCK(stginfo) Py_BEGIN_CRITICAL_SECTION_MUT(&(stginfo)->mutex) #define STGINFO_UNLOCK() Py_END_CRITICAL_SECTION() +#define STGINFO2_LOCK(stginfo_a, stginfo_b) \ + Py_BEGIN_CRITICAL_SECTION2_MUT(&(stginfo_a)->mutex, &(stginfo_b)->mutex) +#define STGINFO2_UNLOCK() Py_END_CRITICAL_SECTION2() + static inline uint8_t stginfo_get_dict_final(StgInfo *info) { From 360303fa0cb1b21c018db3bd64f3f1a8d4d1a78e Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Mon, 31 Mar 2025 12:13:25 +0500 Subject: [PATCH 40/65] Revert "Try to add thread safety to pointer_type" This reverts commit 62d2deb5f1d6086faf3bbcba8128c3f7ecdedd95. --- Modules/_ctypes/_ctypes.c | 36 +++++++++++------------------------- Modules/_ctypes/ctypes.h | 4 ---- 2 files changed, 11 insertions(+), 29 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index b5fd96b27920c2..74255d135eb252 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -588,9 +588,8 @@ ctype_get_pointer_type(PyObject *self, void *Py_UNUSED(ignored)) return NULL; } - PyObject* pointer_type = FT_ATOMIC_LOAD_PTR_ACQUIRE(info->pointer_type); - if (pointer_type) { - return Py_NewRef(pointer_type); + if (info->pointer_type) { + return Py_NewRef(info->pointer_type); } Py_RETURN_NONE; } @@ -1228,39 +1227,26 @@ PyCPointerType_SetProto(ctypes_state *st, PyObject *self, StgInfo *stginfo, PyOb PyErr_Format(PyExc_TypeError, "%R must have storage info", proto); return -1; } - - PyObject* pointer_type = FT_ATOMIC_LOAD_PTR_ACQUIRE(info->pointer_type); - PyObject* stginfo_proto = FT_ATOMIC_LOAD_PTR_ACQUIRE(stginfo->proto); - - if (pointer_type && pointer_type != self) { + if (info->pointer_type && info->pointer_type != self) { PyErr_Format(PyExc_TypeError, "pointer type already set: old=%R, new=%R", - pointer_type, self); + info->pointer_type, self); return -1; } - if (stginfo_proto && stginfo_proto != proto) { + if (stginfo->proto && stginfo->proto != proto) { PyErr_Format(PyExc_TypeError, "cls type already set: old=%R, new=%R", - stginfo_proto, proto); + stginfo->proto, proto); return -1; } - if (!stginfo_proto || !pointer_type) { - STGINFO2_LOCK(stginfo, info); - - stginfo_proto = FT_ATOMIC_LOAD_PTR_ACQUIRE(stginfo->proto); - if (!stginfo_proto) { - FT_ATOMIC_STORE_PTR_RELEASE(stginfo->proto, Py_NewRef(proto)); - } - - pointer_type = FT_ATOMIC_LOAD_PTR_ACQUIRE(info->pointer_type); - if (!pointer_type) { - FT_ATOMIC_STORE_PTR_RELEASE(info->pointer_type, Py_NewRef(self)); - } - - STGINFO2_UNLOCK(); + if (!stginfo->proto) { + stginfo->proto = Py_NewRef(proto); } + if (!info->pointer_type) { + info->pointer_type = Py_NewRef(self); + } return 0; } diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index cdd24c38bb37d9..cca81e7e016605 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -422,10 +422,6 @@ typedef struct { #define STGINFO_LOCK(stginfo) Py_BEGIN_CRITICAL_SECTION_MUT(&(stginfo)->mutex) #define STGINFO_UNLOCK() Py_END_CRITICAL_SECTION() -#define STGINFO2_LOCK(stginfo_a, stginfo_b) \ - Py_BEGIN_CRITICAL_SECTION2_MUT(&(stginfo_a)->mutex, &(stginfo_b)->mutex) -#define STGINFO2_UNLOCK() Py_END_CRITICAL_SECTION2() - static inline uint8_t stginfo_get_dict_final(StgInfo *info) { From a2cc9616ef59368b3549f498dc81655101cc506f Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 2 Apr 2025 00:27:45 +0500 Subject: [PATCH 41/65] Add set_non_ctypes_pointer_type --- Lib/ctypes/__init__.py | 7 +++++++ Lib/test/test_ctypes/test_c_simple_type_meta.py | 10 +++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 74c2fc707e6263..17ed36dfc5b84f 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -297,6 +297,13 @@ def pointer(obj): typ = POINTER(type(obj)) return typ(obj) +def set_non_ctypes_pointer_type(cls, pointer_type): + """Set pointer type for cls.""" + assert not isinstance(pointer_type, _Pointer) + assert not isinstance(cls, _SimpleCData) + + cls.__pointer_type__ = pointer_type + _pointer_type_cache = {} """XXX: Subject to change.""" diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py index 6df6eed61f1144..6c60ea9f68ece9 100644 --- a/Lib/test/test_ctypes/test_c_simple_type_meta.py +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -1,7 +1,7 @@ import unittest from test.support import MS_WINDOWS import ctypes -from ctypes import POINTER, Structure, c_void_p +from ctypes import POINTER, Structure, c_void_p, set_non_ctypes_pointer_type from ._support import PyCSimpleType, PyCPointerType, PyCStructType @@ -36,7 +36,7 @@ def __new__(cls, name, bases, namespace): else: ptr_bases = (self, POINTER(bases[0])) p = p_meta(f"POINTER({self.__name__})", ptr_bases, {}) - self.__pointer_type__ = p + set_non_ctypes_pointer_type(self, p) return self class p_meta(PyCSimpleType, ct_meta): @@ -85,7 +85,7 @@ def __new__(cls, name, bases, namespace): if isinstance(self, p_meta): return self p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {}) - self.__pointer_type__ = p + set_non_ctypes_pointer_type(self, p) return self class p_meta(PyCSimpleType, ct_meta): @@ -131,7 +131,7 @@ def __init__(self, name, bases, namespace): else: ptr_bases = (self, POINTER(bases[0])) p = p_meta(f"POINTER({self.__name__})", ptr_bases, {}) - self.__pointer_type__ = p + set_non_ctypes_pointer_type(self, p) class p_meta(PyCSimpleType, ct_meta): pass @@ -180,7 +180,7 @@ def __init__(self, name, bases, namespace): if isinstance(self, p_meta): return p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {}) - self.__pointer_type__ = p + set_non_ctypes_pointer_type(self, p) class p_meta(PyCSimpleType, ct_meta): pass From df565410502ded491421a3bd677b262861e43f18 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Thu, 3 Apr 2025 00:31:59 +0500 Subject: [PATCH 42/65] Add some new test and docstrings for tests --- .../test_ctypes/test_c_simple_type_meta.py | 78 +++++++++++++++---- 1 file changed, 62 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py index 6c60ea9f68ece9..12b5aee0f96a14 100644 --- a/Lib/test/test_ctypes/test_c_simple_type_meta.py +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -224,12 +224,13 @@ class F(metaclass=PyCSimpleType): self.assertIn("'" + ''.join(expected_type_chars) + "'", message) def test_creating_pointer_in_dunder_init_3(self): + """Check if interfcase subclasses properly creates according internal + pointer types. But not the same as external pointer types. + """ class StructureMeta(PyCStructType): def __new__(cls, name, bases, dct, /, create_pointer_type=True): - if len(bases) > 1: - bases = (bases[0],) - + assert len(bases) == 1, bases return super().__new__(cls, name, bases, dct) def __init__(self, name, bases, dct, /, create_pointer_type=True): @@ -237,11 +238,11 @@ def __init__(self, name, bases, dct, /, create_pointer_type=True): super().__init__(name, bases, dct) if create_pointer_type: p_bases = (POINTER(bases[0]),) - p = PointerMeta(f"p{name}", p_bases, {'_type_': self}) - assert isinstance(p, PyCPointerType) + ns = {'_type_': self} + internal_pointer_type = PointerMeta(f"p{name}", p_bases, ns) + assert isinstance(internal_pointer_type, PyCPointerType) assert self.__pointer_type__ is not None - assert self.__pointer_type__ == p - + assert self.__pointer_type__ is internal_pointer_type class PointerMeta(PyCPointerType): def __new__(cls, name, bases, dct): @@ -264,6 +265,7 @@ def __new__(cls, name, bases, dct): def __init__(self, name, bases, dct, /, create_pointer_type=True): target = dct.get('_type_', None) + assert target.__pointer_type__ is None super().__init__(name, bases, dct) assert target.__pointer_type__ is self @@ -281,17 +283,18 @@ class pIUnknown(pInterface): pass self.assertTrue(issubclass(POINTER(IUnknown), pInterface)) - self.assertTrue(issubclass(pIUnknown, pInterface)) self.assertIs(POINTER(Interface), pInterface) + self.assertIs(POINTER(IUnknown), POINTER(IUnknown)) self.assertIsNot(POINTER(IUnknown), pIUnknown) def test_creating_pointer_in_dunder_init_4(self): - + """Check if interfcase subclasses properly creates according internal + pointer types, the same as external pointer types. + """ class StructureMeta(PyCStructType): def __new__(cls, name, bases, dct, /, create_pointer_type=True): - if len(bases) > 1: - bases = (bases[0],) + assert len(bases) == 1, bases return super().__new__(cls, name, bases, dct) @@ -300,11 +303,11 @@ def __init__(self, name, bases, dct, /, create_pointer_type=True): super().__init__(name, bases, dct) if create_pointer_type: p_bases = (POINTER(bases[0]),) - p = PointerMeta(f"p{name}", p_bases, {'_type_': self}) - assert isinstance(p, PyCPointerType) + ns = {'_type_': self} + internal_pointer_type = PointerMeta(f"p{name}", p_bases, ns) + assert isinstance(internal_pointer_type, PyCPointerType) assert self.__pointer_type__ is not None - assert self.__pointer_type__ == p - + assert self.__pointer_type__ is internal_pointer_type class PointerMeta(PyCPointerType): def __new__(cls, name, bases, dct): @@ -339,7 +342,50 @@ class pIUnknown(pInterface): _type_ = IUnknown self.assertTrue(issubclass(POINTER(IUnknown), pInterface)) - self.assertTrue(issubclass(pIUnknown, pInterface)) self.assertIs(POINTER(Interface), pInterface) self.assertIs(POINTER(IUnknown), pIUnknown) + self.assertIs(POINTER(IUnknown), pIUnknown) + + def test_custom_pointer_cache_for_ctypes_type1(self): + # Check if PyCPointerType.__init__() caches a pointer type + # customized in the metatype's __new__(). + class PointerMeta(PyCPointerType): + def __new__(cls, name, bases, namespace): + namespace["_type_"] = C + return super().__new__(cls, name, bases, namespace) + + def __init__(self, name, bases, namespace): + assert C.__pointer_type__ is None + super().__init__(name, bases, namespace) + assert C.__pointer_type__ is self + + class C(c_void_p): # ctypes type + pass + + class P(ctypes._Pointer, metaclass=PointerMeta): + pass + + self.assertIs(P._type_, C) + self.assertIs(P, POINTER(C)) + self.assertIs(P, POINTER(C)) + + def test_custom_pointer_cache_for_ctypes_type2(self): + # Check if PyCPointerType.__init__() caches a pointer type + # customized in the metatype's __init__(). + class PointerMeta(PyCPointerType): + def __init__(self, name, bases, namespace): + self._type_ = namespace["_type_"] = C + assert C.__pointer_type__ is None + super().__init__(name, bases, namespace) + assert C.__pointer_type__ is self + + class C(c_void_p): # ctypes type + pass + + class P(ctypes._Pointer, metaclass=PointerMeta): + pass + + self.assertIs(P._type_, C) + self.assertIs(P, POINTER(C)) + self.assertIs(P, POINTER(C)) From 7d42666c9e579795d7c7180ba5225562605bd88c Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Thu, 3 Apr 2025 11:44:50 +0500 Subject: [PATCH 43/65] Address some neonene comments --- Lib/ctypes/__init__.py | 7 ------- Lib/test/test_ctypes/test_c_simple_type_meta.py | 9 ++++----- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 17ed36dfc5b84f..74c2fc707e6263 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -297,13 +297,6 @@ def pointer(obj): typ = POINTER(type(obj)) return typ(obj) -def set_non_ctypes_pointer_type(cls, pointer_type): - """Set pointer type for cls.""" - assert not isinstance(pointer_type, _Pointer) - assert not isinstance(cls, _SimpleCData) - - cls.__pointer_type__ = pointer_type - _pointer_type_cache = {} """XXX: Subject to change.""" diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py index 12b5aee0f96a14..b3a91cb754b445 100644 --- a/Lib/test/test_ctypes/test_c_simple_type_meta.py +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -1,11 +1,14 @@ import unittest from test.support import MS_WINDOWS import ctypes -from ctypes import POINTER, Structure, c_void_p, set_non_ctypes_pointer_type +from ctypes import POINTER, Structure, c_void_p from ._support import PyCSimpleType, PyCPointerType, PyCStructType +def set_non_ctypes_pointer_type(cls, pointer_type): + cls.__pointer_type__ = pointer_type + class PyCSimpleTypeAsMetaclassTest(unittest.TestCase): def tearDown(self): # to not leak references, we must clean _pointer_type_cache @@ -285,7 +288,6 @@ class pIUnknown(pInterface): self.assertTrue(issubclass(POINTER(IUnknown), pInterface)) self.assertIs(POINTER(Interface), pInterface) - self.assertIs(POINTER(IUnknown), POINTER(IUnknown)) self.assertIsNot(POINTER(IUnknown), pIUnknown) def test_creating_pointer_in_dunder_init_4(self): @@ -345,7 +347,6 @@ class pIUnknown(pInterface): self.assertIs(POINTER(Interface), pInterface) self.assertIs(POINTER(IUnknown), pIUnknown) - self.assertIs(POINTER(IUnknown), pIUnknown) def test_custom_pointer_cache_for_ctypes_type1(self): # Check if PyCPointerType.__init__() caches a pointer type @@ -368,7 +369,6 @@ class P(ctypes._Pointer, metaclass=PointerMeta): self.assertIs(P._type_, C) self.assertIs(P, POINTER(C)) - self.assertIs(P, POINTER(C)) def test_custom_pointer_cache_for_ctypes_type2(self): # Check if PyCPointerType.__init__() caches a pointer type @@ -388,4 +388,3 @@ class P(ctypes._Pointer, metaclass=PointerMeta): self.assertIs(P._type_, C) self.assertIs(P, POINTER(C)) - self.assertIs(P, POINTER(C)) From e31c7e95987c566ba0da152e7bbb89bcafc875a4 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sun, 27 Apr 2025 00:09:37 +0500 Subject: [PATCH 44/65] Move __pointer_type__ docs to common class variables section --- Doc/library/ctypes.rst | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 9bf19890a96c6b..932776528e69ca 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -2349,6 +2349,16 @@ Data types library. *name* is the name of the symbol that exports the data, *library* is the loaded shared library. + Common class variables of ctypes data types: + + .. attribute:: __pointer_type__ + + This attributes is a pointer type that was created by calling + :func:`POINTER` for corresponding ctypes data type. If ``POINTER`` was + not called for this type then attribute contains ``None``. + + .. versionadded:: next + Common instance variables of ctypes data types: .. attribute:: _b_base_ @@ -2370,14 +2380,6 @@ Data types valid. This object is only exposed for debugging; never modify the contents of this dictionary. - .. attribute:: __pointer_type__ - - This attributes is a pointer type that was created by calling - :func:`POINTER` for corresponding ctypes data type. If ``POINTER`` was - not called then this attribute contains ``None``. - - .. versionadded:: 3.14 - .. _ctypes-fundamental-data-types-2: From 56d41e28319f3feccd18b3fb1ddabe8421af0df9 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 30 Apr 2025 21:35:28 +0500 Subject: [PATCH 45/65] Apply suggestions from code review Co-authored-by: Petr Viktorin --- Doc/library/ctypes.rst | 17 +++++++++++++---- Doc/whatsnew/3.14.rst | 8 ++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 932776528e69ca..412a745fe43293 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -2182,8 +2182,17 @@ Utility functions Create or return a ctypes pointer type. Pointer types are cached and reused internally, so calling this function repeatedly is cheap. - *type* must be a ctypes-like type. The ctypes-like type is a type that - has class level attribute with the name ``__pointer_type__``. + *type* must be a ctypes type. + + .. impl-detail:: + + The resulting pointer type is cached in the ``__pointer_type__`` + attribute of *type*. + It is possible to set this attribute before the first call to + ``POINTER`` in order to set a custom pointer type. + However, doing this is discouraged: manually creating a suitable + pointer type is difficult without relying on implementation + details that may change in future Python versions. .. function:: pointer(obj, /) @@ -2354,8 +2363,8 @@ Data types .. attribute:: __pointer_type__ This attributes is a pointer type that was created by calling - :func:`POINTER` for corresponding ctypes data type. If ``POINTER`` was - not called for this type then attribute contains ``None``. + :func:`POINTER` for corresponding ctypes data type. If a pointer type + was not yet created, the attribute is missing. .. versionadded:: next diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 0d58e0108d5dd4..fa621f1c19b0e5 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -732,10 +732,10 @@ ctypes loaded by the current process. (Contributed by Brian Ward in :gh:`119349`.) -* Move :func:`ctypes.POINTER` types cache from the global ``_pointer_type_cache`` - to the corresponding :mod:`ctypes` types. This will stop the cache from - growing without limits in some situations. Also :mod:`ctypes` pointer types - can be accessed with ``__pointer_type__`` attribute from corresponding type. +* Move :func:`ctypes.POINTER` types cache from a global internal cache + (``_pointer_type_cache``) to the :attr:`ctypes._CData.__pointer_type__` + attribute of the corresponding :mod:`ctypes` types. + This will stop the cache from growing without limits in some situations. (Contributed by Sergey Miryanov in :gh:`100926`). * The :class:`ctypes.py_object` type now supports subscription, From 4316ad9b7429ae244fb6c8d08165d37088aa5f2a Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 30 Apr 2025 21:43:01 +0500 Subject: [PATCH 46/65] Remove Pointers for ctypes-like types section --- Doc/library/ctypes.rst | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 412a745fe43293..0086a24459ae1e 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -871,14 +871,6 @@ invalid non-\ ``NULL`` pointers would crash Python):: ValueError: NULL pointer access >>> -.. _ctypes-pointers-ctypes-like-types: - -Pointers for ctypes-like types -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In some cases you want that :func:`POINTER` can accept non-ctypes types. To do so -just add to your types a class level attribute with the name ``__pointer_type__``. - .. _ctypes-thread-safety: Thread safety without the GIL @@ -2183,7 +2175,7 @@ Utility functions Create or return a ctypes pointer type. Pointer types are cached and reused internally, so calling this function repeatedly is cheap. *type* must be a ctypes type. - + .. impl-detail:: The resulting pointer type is cached in the ``__pointer_type__`` From 9c99f9ce5f06244a1aa4219c480e7546cb897e49 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 30 Apr 2025 21:44:39 +0500 Subject: [PATCH 47/65] Update news entry --- .../2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst b/Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst index 555043b4cccadb..6a71415fbd87c4 100644 --- a/Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst +++ b/Misc/NEWS.d/next/Library/2025-03-17-23-07-57.gh-issue-100926.B8gcbz.rst @@ -1,7 +1,4 @@ -Move :func:`ctypes.POINTER` types cache from the global ``_pointer_type_cache`` -to the corresponding :mod:`ctypes` types to avoid unlimited growth of the cache. -As from now, :mod:`ctypes` pointer types can be accessed with a -``__pointer_type__`` attribute. :func:`ctypes.POINTER` checks this attribute -first before creating a new pointer type, so if needed to mimic :mod:`ctypes` -pointer type then type should have this attribute -(see :ref:`ctypes-pointers-ctypes-like-types`). +Move :func:`ctypes.POINTER` types cache from a global internal cache +(``_pointer_type_cache``) to the :attr:`ctypes._CData.__pointer_type__` +attribute of the corresponding :mod:`ctypes` types. +This will stop the cache from growing without limits in some situations. From 5753fc967101c822e6105e72c11f8377b552d8d3 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 30 Apr 2025 22:04:17 +0500 Subject: [PATCH 48/65] Update docstring for POINTER and add comments --- Lib/ctypes/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index a96c1f2ccedce5..c19d718c709b5e 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -271,10 +271,6 @@ def POINTER(cls): Pointer types are cached and reused internally, so calling this function repeatedly is cheap. - - Pointer types for incomplete types are not cached, - so calling this function repeatedly will give - different types. """ if cls is None: return c_void_p @@ -285,7 +281,12 @@ def POINTER(cls): except AttributeError: pass if isinstance(cls, str): + # handle old-style incomplete types + # in this case pointer type is not cached and calling this function + # repeatedly will give different result return type(f'LP_{cls}', (_Pointer,), {}) + + # create pointer type and set __pointer_type__ for cls return type(f'LP_{cls.__name__}', (_Pointer,), {'_type_': cls}) def pointer(obj): From 58a1507d7adbda85823077c4d3735449e8650c63 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 30 Apr 2025 22:44:07 +0500 Subject: [PATCH 49/65] Implement _pointer_type_cache as PointerTypeCache proxy --- Lib/ctypes/__init__.py | 36 ++++++++++++-- .../test_ctypes/test_c_simple_type_meta.py | 1 - Lib/test/test_ctypes/test_incomplete.py | 4 +- Lib/test/test_ctypes/test_pointers.py | 49 +++++++++++++++++++ 4 files changed, 84 insertions(+), 6 deletions(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index c19d718c709b5e..6d6e06ac84aa23 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -299,8 +299,38 @@ def pointer(obj): typ = POINTER(type(obj)) return typ(obj) -_pointer_type_cache = {} -"""XXX: Subject to change.""" +class PointerTypeCache: + def __setitem__(self, cls, pointer_type): + import warnings + warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19)) + try: + cls.__pointer_type__ = pointer_type + except AttributeError: + pass + + def __getitem__(self, cls): + import warnings + warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19)) + try: + return cls.__pointer_type__ + except AttributeError: + raise KeyError(cls) + + _sentinel = object() + def get(self, cls, default=_sentinel): + import warnings + warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19)) + try: + return cls.__pointer_type__ + except AttributeError: + if default is self._sentinel: + raise KeyError(cls) + return default + + def __contains__(self, cls): + return hasattr(cls, '__pointer_type__') + +_pointer_type_cache = PointerTypeCache() class c_wchar_p(_SimpleCData): _type_ = "Z" @@ -311,7 +341,6 @@ class c_wchar(_SimpleCData): _type_ = "u" def _reset_cache(): - _pointer_type_cache.clear() _c_functype_cache.clear() if _os.name == "nt": _win_functype_cache.clear() @@ -319,7 +348,6 @@ def _reset_cache(): POINTER(c_wchar).from_param = c_wchar_p.from_param # _SimpleCData.c_char_p_from_param POINTER(c_char).from_param = c_char_p.from_param - _pointer_type_cache[None] = c_void_p def create_unicode_buffer(init, size=None): """create_unicode_buffer(aString) -> character array diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py index 03cd69023e626e..bf43c3b4076b93 100644 --- a/Lib/test/test_ctypes/test_c_simple_type_meta.py +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -11,7 +11,6 @@ def set_non_ctypes_pointer_type(cls, pointer_type): class PyCSimpleTypeAsMetaclassTest(unittest.TestCase): def tearDown(self): - # to not leak references, we must clean _pointer_type_cache ctypes._reset_cache() def test_creating_pointer_in_dunder_new_1(self): diff --git a/Lib/test/test_ctypes/test_incomplete.py b/Lib/test/test_ctypes/test_incomplete.py index 9f859793d88a22..501c33240d1bb5 100644 --- a/Lib/test/test_ctypes/test_incomplete.py +++ b/Lib/test/test_ctypes/test_incomplete.py @@ -7,7 +7,6 @@ # The incomplete pointer example from the tutorial class TestSetPointerType(unittest.TestCase): def tearDown(self): - # to not leak references, we must clean _pointer_type_cache ctypes._reset_cache() def test_incomplete_example(self): @@ -20,6 +19,8 @@ class cell(Structure): warnings.simplefilter('ignore', DeprecationWarning) ctypes.SetPointerType(lpcell, cell) + self.assertIs(POINTER(cell), lpcell) + c1 = cell() c1.name = b"foo" c2 = cell() @@ -45,6 +46,7 @@ class cell(Structure): with self.assertWarns(DeprecationWarning): ctypes.SetPointerType(lpcell, cell) + self.assertIs(POINTER(cell), lpcell) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index 29e954b8a11437..e61f99edf72445 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -10,6 +10,7 @@ c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) +from ctypes import _pointer_type_cache from test.support import import_helper from weakref import WeakSet _ctypes_test = import_helper.import_module("_ctypes_test") @@ -352,5 +353,53 @@ class Cls(Structure): self.assertEqual(len(ws_ptr), 0, ws_ptr) +class PointerTypeCacheTestCase(unittest.TestCase): + # dummy tests to check warnings and base behavior + + def test_deprecated_cache_with_not_ctypes_type(self): + class C: + pass + + P = POINTER("C") + with self.assertWarns(DeprecationWarning): + _pointer_type_cache[C] = P + + self.assertIs(C.__pointer_type__, P) + with self.assertWarns(DeprecationWarning): + self.assertIs(_pointer_type_cache[C], P) + + with self.assertWarns(DeprecationWarning): + self.assertIs(_pointer_type_cache.get(C), P) + + def test_deprecated_cache_with_ctypes_type(self): + class C(Structure): + _fields_ = [("a", c_int), + ("b", c_int), + ("c", c_int)] + + P1 = POINTER(C) + P2 = POINTER("C") + with self.assertWarns(DeprecationWarning): + _pointer_type_cache[C] = P1 + + with self.assertWarns(DeprecationWarning): + _pointer_type_cache[C] = P2 # silently do nothing + + self.assertIs(C.__pointer_type__, P1) + self.assertIsNot(C.__pointer_type__, P2) + + with self.assertWarns(DeprecationWarning): + self.assertIs(_pointer_type_cache[C], P1) + + with self.assertWarns(DeprecationWarning): + self.assertIs(_pointer_type_cache.get(C), P1) + + def test_get_not_registered(self): + with self.assertRaises(KeyError), self.assertWarns(DeprecationWarning): + _pointer_type_cache.get(str) + + with self.assertWarns(DeprecationWarning): + self.assertIsNone(_pointer_type_cache.get(str, None)) + if __name__ == '__main__': unittest.main() From 70820092c848d03af2fe055c0fa5d055ee9b723e Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 30 Apr 2025 23:00:32 +0500 Subject: [PATCH 50/65] Raise AttributeError if __pointer_type__ is not set --- Lib/test/test_ctypes/test_c_simple_type_meta.py | 12 ++++++------ Lib/test/test_ctypes/test_pointers.py | 4 +++- Modules/_ctypes/_ctypes.c | 6 +++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py index bf43c3b4076b93..4dff68e26eaeba 100644 --- a/Lib/test/test_ctypes/test_c_simple_type_meta.py +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -261,13 +261,13 @@ def __new__(cls, name, bases, dct): dct['_type_'] = target pointer_type = super().__new__(cls, name, bases, dct) - assert target.__pointer_type__ is None + assert not hasattr(target, '__pointer_type__') return pointer_type def __init__(self, name, bases, dct, /, create_pointer_type=True): target = dct.get('_type_', None) - assert target.__pointer_type__ is None + assert not hasattr(target, '__pointer_type__') super().__init__(name, bases, dct) assert target.__pointer_type__ is self @@ -314,7 +314,7 @@ class PointerMeta(PyCPointerType): def __new__(cls, name, bases, dct): target = dct.get('_type_', None) assert target is not None - pointer_type = target.__pointer_type__ + pointer_type = getattr(target, '__pointer_type__', None) if pointer_type is None: pointer_type = super().__new__(cls, name, bases, dct) @@ -323,7 +323,7 @@ def __new__(cls, name, bases, dct): def __init__(self, name, bases, dct, /, create_pointer_type=True): target = dct.get('_type_', None) - if target.__pointer_type__ is None: + if not hasattr(target, '__pointer_type__'): # target.__pointer_type__ was created by super().__new__ super().__init__(name, bases, dct) @@ -356,7 +356,7 @@ def __new__(cls, name, bases, namespace): return super().__new__(cls, name, bases, namespace) def __init__(self, name, bases, namespace): - assert C.__pointer_type__ is None + assert not hasattr(C, '__pointer_type__') super().__init__(name, bases, namespace) assert C.__pointer_type__ is self @@ -375,7 +375,7 @@ def test_custom_pointer_cache_for_ctypes_type2(self): class PointerMeta(PyCPointerType): def __init__(self, name, bases, namespace): self._type_ = namespace["_type_"] = C - assert C.__pointer_type__ is None + assert not hasattr(C, '__pointer_type__') super().__init__(name, bases, namespace) assert C.__pointer_type__ is self diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index e61f99edf72445..e8863af2fdf01a 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -320,7 +320,9 @@ class Cls(Structure): ('b', c_float), ) - self.assertIsNone(Cls.__pointer_type__) + with self.assertRaisesRegex(AttributeError, ".Cls'> has no attribute '__pointer_type__'"): + Cls.__pointer_type__ + p = POINTER(Cls) self.assertIs(Cls.__pointer_type__, p) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 16bf7a6bcdc8d0..c60649de5fc2a0 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -592,7 +592,11 @@ ctype_get_pointer_type(PyObject *self, void *Py_UNUSED(ignored)) if (info->pointer_type) { return Py_NewRef(info->pointer_type); } - Py_RETURN_NONE; + + PyErr_Format(PyExc_AttributeError, + "%R has no attribute '__pointer_type__'", + self); + return NULL; } static PyObject * From f89da961e45ca52f63b337b3356d1d2e6497072d Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Thu, 1 May 2025 12:43:46 +0500 Subject: [PATCH 51/65] Fixed POINTER if __pointer_type__ already initialized --- Lib/ctypes/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 6d6e06ac84aa23..61f0508b895053 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -275,9 +275,7 @@ def POINTER(cls): if cls is None: return c_void_p try: - pt = cls.__pointer_type__ - if pt is not None: - return pt + return cls.__pointer_type__ except AttributeError: pass if isinstance(cls, str): From 1afa7b36c57249c3bcfc187ce744a12cc4da31bd Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Thu, 1 May 2025 12:44:08 +0500 Subject: [PATCH 52/65] Fixed default value for PointerTypeCache.get --- Lib/ctypes/__init__.py | 5 +---- Lib/test/test_ctypes/test_pointers.py | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 61f0508b895053..09599ed628030b 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -314,15 +314,12 @@ def __getitem__(self, cls): except AttributeError: raise KeyError(cls) - _sentinel = object() - def get(self, cls, default=_sentinel): + def get(self, cls, default=None): import warnings warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19)) try: return cls.__pointer_type__ except AttributeError: - if default is self._sentinel: - raise KeyError(cls) return default def __contains__(self, cls): diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index e8863af2fdf01a..aa510b0b2ea20a 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -397,8 +397,8 @@ class C(Structure): self.assertIs(_pointer_type_cache.get(C), P1) def test_get_not_registered(self): - with self.assertRaises(KeyError), self.assertWarns(DeprecationWarning): - _pointer_type_cache.get(str) + with self.assertWarns(DeprecationWarning): + self.assertIsNone(_pointer_type_cache.get(str)) with self.assertWarns(DeprecationWarning): self.assertIsNone(_pointer_type_cache.get(str, None)) From e99277c7a6d72c7d9ade3ac4b0395bc9c72f1b5a Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Thu, 1 May 2025 14:50:53 +0500 Subject: [PATCH 53/65] Remove setting item in _pointer_type_cache from SetPointerType --- Lib/ctypes/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 09599ed628030b..f3c1a0ac56c854 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -380,7 +380,6 @@ def SetPointerType(pointer, cls): raise RuntimeError("This type already exists in the cache") pointer.set_type(cls) - _pointer_type_cache[cls] = pointer def ARRAY(typ, len): return typ * len From ebed23e9ed88d434ce50250fb35f88e8bf612a29 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 30 Apr 2025 16:48:19 +0200 Subject: [PATCH 54/65] tests: Remove teardown code that now does nothing --- Lib/test/test_ctypes/test_c_simple_type_meta.py | 3 --- Lib/test/test_ctypes/test_incomplete.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py index 4dff68e26eaeba..43b3044323e2da 100644 --- a/Lib/test/test_ctypes/test_c_simple_type_meta.py +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -10,9 +10,6 @@ def set_non_ctypes_pointer_type(cls, pointer_type): cls.__pointer_type__ = pointer_type class PyCSimpleTypeAsMetaclassTest(unittest.TestCase): - def tearDown(self): - ctypes._reset_cache() - def test_creating_pointer_in_dunder_new_1(self): # Test metaclass whose instances are C types; when the type is # created it automatically creates a pointer type for itself. diff --git a/Lib/test/test_ctypes/test_incomplete.py b/Lib/test/test_ctypes/test_incomplete.py index 501c33240d1bb5..b045ccd5f026e3 100644 --- a/Lib/test/test_ctypes/test_incomplete.py +++ b/Lib/test/test_ctypes/test_incomplete.py @@ -6,9 +6,6 @@ # The incomplete pointer example from the tutorial class TestSetPointerType(unittest.TestCase): - def tearDown(self): - ctypes._reset_cache() - def test_incomplete_example(self): lpcell = POINTER("cell") class cell(Structure): From c4ee75a052a1b6032993b6c399bd2edae4b51bd8 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 30 Apr 2025 16:48:59 +0200 Subject: [PATCH 55/65] Remove _pointer_type_cache setting --- Lib/ctypes/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index f3c1a0ac56c854..0ef6c051cbc8a6 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -376,9 +376,6 @@ def create_unicode_buffer(init, size=None): def SetPointerType(pointer, cls): import warnings warnings._deprecated("ctypes.SetPointerType", remove=(3, 15)) - if _pointer_type_cache.get(cls, None) is not None: - raise RuntimeError("This type already exists in the cache") - pointer.set_type(cls) def ARRAY(typ, len): From 66c68b08f6edc17e779b6d868214ace6b90bc110 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 1 May 2025 16:21:41 +0200 Subject: [PATCH 56/65] Handle old-style incomplete types --- Lib/ctypes/__init__.py | 22 +++++++++----- Lib/test/test_ctypes/test_incomplete.py | 16 ++++++++-- Lib/test/test_ctypes/test_pointers.py | 40 +++++++++++++++++-------- 3 files changed, 55 insertions(+), 23 deletions(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 0ef6c051cbc8a6..fade8deac3e29e 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -279,10 +279,15 @@ def POINTER(cls): except AttributeError: pass if isinstance(cls, str): - # handle old-style incomplete types - # in this case pointer type is not cached and calling this function - # repeatedly will give different result - return type(f'LP_{cls}', (_Pointer,), {}) + # handle old-style incomplete types (see test_ctypes.test_incomplete) + import warnings + warnings._deprecated("ctypes.POINTER with string", remove=(3, 19)) + try: + return _pointer_type_cache_fallback[cls] + except KeyError: + result = type(f'LP_{cls}', (_Pointer,), {}) + _pointer_type_cache_fallback[cls] = result + return result # create pointer type and set __pointer_type__ for cls return type(f'LP_{cls.__name__}', (_Pointer,), {'_type_': cls}) @@ -297,14 +302,14 @@ def pointer(obj): typ = POINTER(type(obj)) return typ(obj) -class PointerTypeCache: +class _PointerTypeCache: def __setitem__(self, cls, pointer_type): import warnings warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19)) try: cls.__pointer_type__ = pointer_type except AttributeError: - pass + _pointer_type_cache_fallback[cls] = pointer_type def __getitem__(self, cls): import warnings @@ -312,7 +317,7 @@ def __getitem__(self, cls): try: return cls.__pointer_type__ except AttributeError: - raise KeyError(cls) + return _pointer_type_cache_fallback[cls] def get(self, cls, default=None): import warnings @@ -325,7 +330,8 @@ def get(self, cls, default=None): def __contains__(self, cls): return hasattr(cls, '__pointer_type__') -_pointer_type_cache = PointerTypeCache() +_pointer_type_cache_fallback = {} +_pointer_type_cache = _PointerTypeCache() class c_wchar_p(_SimpleCData): _type_ = "Z" diff --git a/Lib/test/test_ctypes/test_incomplete.py b/Lib/test/test_ctypes/test_incomplete.py index b045ccd5f026e3..9933ab7f8d2670 100644 --- a/Lib/test/test_ctypes/test_incomplete.py +++ b/Lib/test/test_ctypes/test_incomplete.py @@ -3,11 +3,20 @@ import warnings from ctypes import Structure, POINTER, pointer, c_char_p +# String-based "incomplete pointers" wers implemented ctypes 0.6.3 (2003, when +# ctypes was an external project). They made obsolete by the current +# incomplete *types* (setting `_fields_` late) in 0.9.5 (2005). +# ctypes was added to Python 2.5 (2006), without any mention in docs. -# The incomplete pointer example from the tutorial +# This tests incomplete pointer example from the old tutorial +# (https://svn.python.org/projects/ctypes/tags/release_0_6_3/ctypes/docs/tutorial.stx) class TestSetPointerType(unittest.TestCase): + def tearDown(self): + ctypes._pointer_type_cache_fallback.clear() + def test_incomplete_example(self): - lpcell = POINTER("cell") + with self.assertWarns(DeprecationWarning): + lpcell = POINTER("cell") class cell(Structure): _fields_ = [("name", c_char_p), ("next", lpcell)] @@ -35,7 +44,8 @@ class cell(Structure): self.assertEqual(result, [b"foo", b"bar"] * 4) def test_deprecation(self): - lpcell = POINTER("cell") + with self.assertWarns(DeprecationWarning): + lpcell = POINTER("cell") class cell(Structure): _fields_ = [("name", c_char_p), ("next", lpcell)] diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index aa510b0b2ea20a..439cd956237c07 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -10,7 +10,7 @@ c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) -from ctypes import _pointer_type_cache +from ctypes import _pointer_type_cache, _pointer_type_cache_fallback from test.support import import_helper from weakref import WeakSet _ctypes_test = import_helper.import_module("_ctypes_test") @@ -25,6 +25,9 @@ class PointersTestCase(unittest.TestCase): + def tearDown(self): + _pointer_type_cache_fallback.clear() + def test_inheritance_hierarchy(self): self.assertEqual(_Pointer.mro(), [_Pointer, _CData, object]) @@ -246,7 +249,8 @@ def test_pointer_type_name(self): def test_pointer_type_str_name(self): large_string = 'T' * 2 ** 25 - P = POINTER(large_string) + with self.assertWarns(DeprecationWarning): + P = POINTER(large_string) self.assertTrue(P) def test_abstract(self): @@ -267,14 +271,17 @@ def test_pointer_types_equal(self): self.assertIs(type(p1), t1) self.assertIs(type(p2), t1) - def test_incomplete_pointer_types_not_equal(self): - t1 = POINTER("LP_C") - t2 = POINTER("LP_C") + def test_incomplete_pointer_types_still_equal(self): + with self.assertWarns(DeprecationWarning): + t1 = POINTER("LP_C") + with self.assertWarns(DeprecationWarning): + t2 = POINTER("LP_C") - self.assertIsNot(t1, t2) + self.assertIs(t1, t2) def test_incomplete_pointer_types_cannot_instantiate(self): - t1 = POINTER("LP_C") + with self.assertWarns(DeprecationWarning): + t1 = POINTER("LP_C") with self.assertRaisesRegex(TypeError, "has no _type_"): t1() @@ -357,21 +364,31 @@ class Cls(Structure): class PointerTypeCacheTestCase(unittest.TestCase): # dummy tests to check warnings and base behavior + def tearDown(self): + _pointer_type_cache_fallback.clear() def test_deprecated_cache_with_not_ctypes_type(self): class C: pass - P = POINTER("C") with self.assertWarns(DeprecationWarning): - _pointer_type_cache[C] = P + P = POINTER("C") + with self.assertWarns(DeprecationWarning): + self.assertIs(_pointer_type_cache["C"], P) + + with self.assertWarns(DeprecationWarning): + _pointer_type_cache[C] = P self.assertIs(C.__pointer_type__, P) with self.assertWarns(DeprecationWarning): self.assertIs(_pointer_type_cache[C], P) + def test_deprecated_cache_with_ints(self): + with self.assertWarns(DeprecationWarning): + _pointer_type_cache[123] = 456 + with self.assertWarns(DeprecationWarning): - self.assertIs(_pointer_type_cache.get(C), P) + self.assertEqual(_pointer_type_cache[123], 456) def test_deprecated_cache_with_ctypes_type(self): class C(Structure): @@ -380,9 +397,8 @@ class C(Structure): ("c", c_int)] P1 = POINTER(C) - P2 = POINTER("C") with self.assertWarns(DeprecationWarning): - _pointer_type_cache[C] = P1 + P2 = POINTER("C") with self.assertWarns(DeprecationWarning): _pointer_type_cache[C] = P2 # silently do nothing From 0d2c75b93c8a7c9d57e19b506fec76b58944ac72 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 1 May 2025 16:30:23 +0200 Subject: [PATCH 57/65] Document deprecations --- Doc/whatsnew/3.14.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index fa621f1c19b0e5..bf30319a2d60f9 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1580,6 +1580,14 @@ Deprecated as a single positional argument. (Contributed by Serhiy Storchaka in :gh:`109218`.) +* :mod:`ctypes`: + Calling :func:`ctypes.POINTER` on a string is deprecated. + Use :ref:`ctypes-incomplete-types` for self-referential structures. + Also, ``ctypes._pointer_type_cache`` is deprecated as a courtesy to + existing users of this internal API. + See :func:`ctypes.POINTER` for updated implementation details. + (Contributed by Sergey Myrianov in :gh:`100926`.) + * :mod:`functools`: Calling the Python implementation of :func:`functools.reduce` with *function* or *sequence* as keyword arguments is now deprecated. From 8597f6bcdb61b9098d71f05855781ffbdda1c182 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 1 May 2025 17:10:36 +0200 Subject: [PATCH 58/65] Allow setting/deleting __pointer_type__ --- Lib/test/test_ctypes/test_pointers.py | 40 +++++++++++++++++++++++---- Modules/_ctypes/_ctypes.c | 30 ++++++++++++++------ Modules/_ctypes/ctypes.h | 3 +- 3 files changed, 57 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index 439cd956237c07..9af7543559a890 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -299,7 +299,7 @@ class C(c_int): pass t1 = POINTER(c_int) - with self.assertRaisesRegex(TypeError, "pointer type already set"): + with self.assertRaisesRegex(TypeError, "cls type already set"): t1.set_type(c_float) with self.assertRaisesRegex(TypeError, "cls type already set"): @@ -333,6 +333,34 @@ class Cls(Structure): p = POINTER(Cls) self.assertIs(Cls.__pointer_type__, p) + def test_arbitrary_pointer_type_attribute(self): + class Cls(Structure): + _fields_ = ( + ('a', c_int), + ('b', c_float), + ) + + garbage = 'garbage' + + P = POINTER(Cls) + self.assertIs(Cls.__pointer_type__, P) + Cls.__pointer_type__ = garbage + self.assertIs(Cls.__pointer_type__, garbage) + self.assertIs(POINTER(Cls), garbage) + self.assertIs(P._type_, Cls) + + instance = Cls(1, 2.0) + pointer = P(instance) + self.assertEqual(pointer[0].a, 1) + self.assertEqual(pointer[0].b, 2) + + del Cls.__pointer_type__ + + NewP = POINTER(Cls) + self.assertIsNot(NewP, P) + self.assertIs(Cls.__pointer_type__, NewP) + self.assertIs(P._type_, Cls) + def test_pointer_types_factory(self): """Shouldn't leak""" def factory(): @@ -401,16 +429,16 @@ class C(Structure): P2 = POINTER("C") with self.assertWarns(DeprecationWarning): - _pointer_type_cache[C] = P2 # silently do nothing + _pointer_type_cache[C] = P2 - self.assertIs(C.__pointer_type__, P1) - self.assertIsNot(C.__pointer_type__, P2) + self.assertIs(C.__pointer_type__, P2) + self.assertIsNot(C.__pointer_type__, P1) with self.assertWarns(DeprecationWarning): - self.assertIs(_pointer_type_cache[C], P1) + self.assertIs(_pointer_type_cache[C], P2) with self.assertWarns(DeprecationWarning): - self.assertIs(_pointer_type_cache.get(C), P1) + self.assertIs(_pointer_type_cache.get(C), P2) def test_get_not_registered(self): with self.assertWarns(DeprecationWarning): diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index c60649de5fc2a0..a5b9ef3d818b59 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -599,6 +599,23 @@ ctype_get_pointer_type(PyObject *self, void *Py_UNUSED(ignored)) return NULL; } +static int +ctype_set_pointer_type(PyObject *self, PyObject *tp, void *Py_UNUSED(ignored)) +{ + ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); + StgInfo *info; + if (PyStgInfo_FromType(st, self, &info) < 0) { + return -1; + } + if (!info) { + PyErr_Format(PyExc_TypeError, "%R must have storage info", self); + return -1; + } + + Py_XSETREF(info->pointer_type, Py_XNewRef(tp)); + return 0; +} + static PyObject * CType_Type_repeat(PyObject *self, Py_ssize_t length); @@ -609,7 +626,8 @@ static PyMethodDef ctype_methods[] = { }; static PyGetSetDef ctype_getsets[] = { - { "__pointer_type__", ctype_get_pointer_type, NULL, "pointer type", NULL }, + { "__pointer_type__", ctype_get_pointer_type, ctype_set_pointer_type, + "pointer type", NULL }, { NULL, NULL } }; @@ -1235,12 +1253,6 @@ PyCPointerType_SetProto(ctypes_state *st, PyObject *self, StgInfo *stginfo, PyOb PyErr_Format(PyExc_TypeError, "%R must have storage info", proto); return -1; } - if (info->pointer_type && info->pointer_type != self) { - PyErr_Format(PyExc_TypeError, - "pointer type already set: old=%R, new=%R", - info->pointer_type, self); - return -1; - } if (stginfo->proto && stginfo->proto != proto) { PyErr_Format(PyExc_TypeError, "cls type already set: old=%R, new=%R", @@ -1252,8 +1264,8 @@ PyCPointerType_SetProto(ctypes_state *st, PyObject *self, StgInfo *stginfo, PyOb stginfo->proto = Py_NewRef(proto); } - if (!info->pointer_type) { - info->pointer_type = Py_NewRef(self); + if (info->pointer_type == NULL) { + Py_XSETREF(info->pointer_type, Py_XNewRef(self)); } return 0; } diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 5a3ac409ff32e7..9aceeceb88a49f 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -388,7 +388,8 @@ typedef struct { PyObject *converters; /* tuple([t.from_param for t in argtypes]) */ PyObject *restype; /* CDataObject or NULL */ PyObject *checker; - PyObject *pointer_type; + PyObject *pointer_type; /* __pointer_type__ attribute; + arbitrary object or NULL */ PyObject *module; int flags; /* calling convention and such */ #ifdef Py_GIL_DISABLED From 0b5de2769306cb0a3d3ff11bf4bef65976512f11 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 1 May 2025 18:03:47 +0200 Subject: [PATCH 59/65] Allow arbitrary set_type as before --- Lib/test/test_ctypes/test_pointers.py | 26 ++++++++++++++++++++------ Modules/_ctypes/_ctypes.c | 14 ++------------ 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index 9af7543559a890..99b9d1b8e3a8f7 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -295,15 +295,29 @@ def test_pointer_set_type_twice(self): self.assertIs(t1._type_, c_int) def test_pointer_set_wrong_type(self): - class C(c_int): - pass - - t1 = POINTER(c_int) - with self.assertRaisesRegex(TypeError, "cls type already set"): + int_ptr = POINTER(c_int) + float_ptr = POINTER(float_ptr) + try: + class C(c_int): + pass + + t1 = POINTER(c_int) + t2 = POINTER(c_float) t1.set_type(c_float) + self.assertEqual(t1(c_float(1.5))[0], 1.5) + self.assertIs(c_int._type_, c_float) + self.assertIs(c_int.__pointer_type__, t1) + self.assertIs(c_float.__pointer_type__, float_ptr) - with self.assertRaisesRegex(TypeError, "cls type already set"): t1.set_type(C) + self.assertEqual(t1(C(123))[0].value, 123) + self.assertIs(c_int.__pointer_type__, t1) + self.assertIs(c_float.__pointer_type__, float_ptr) + finally: + POINTER(c_int).set_type(c_int) + self.assertIs(POINTER(c_int), int_ptr) + self.assertIs(POINTER(c_int)._type_, c_int) + self.assertIs(c_int.__pointer_type__, int_ptr) def test_pointer_not_ctypes_type(self): with self.assertRaisesRegex(TypeError, "must have storage info"): diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index a5b9ef3d818b59..2c0b1301203f2e 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1253,19 +1253,9 @@ PyCPointerType_SetProto(ctypes_state *st, PyObject *self, StgInfo *stginfo, PyOb PyErr_Format(PyExc_TypeError, "%R must have storage info", proto); return -1; } - if (stginfo->proto && stginfo->proto != proto) { - PyErr_Format(PyExc_TypeError, - "cls type already set: old=%R, new=%R", - stginfo->proto, proto); - return -1; - } - - if (!stginfo->proto) { - stginfo->proto = Py_NewRef(proto); - } - + Py_XSETREF(stginfo->proto, Py_NewRef(proto)); if (info->pointer_type == NULL) { - Py_XSETREF(info->pointer_type, Py_XNewRef(self)); + Py_XSETREF(info->pointer_type, Py_NewRef(self)); } return 0; } From 78b6c159905c0d643288de49986c4d97cbe47f83 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Thu, 1 May 2025 21:58:49 +0500 Subject: [PATCH 60/65] Apply suggestions from code review Co-authored-by: Petr Viktorin --- Doc/library/ctypes.rst | 2 +- Lib/test/test_ctypes/test_c_simple_type_meta.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 0086a24459ae1e..2825590400c70b 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -2354,7 +2354,7 @@ Data types .. attribute:: __pointer_type__ - This attributes is a pointer type that was created by calling + The pointer type that was created by calling :func:`POINTER` for corresponding ctypes data type. If a pointer type was not yet created, the attribute is missing. diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py index 4dff68e26eaeba..129b4f0d41dd7e 100644 --- a/Lib/test/test_ctypes/test_c_simple_type_meta.py +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -243,7 +243,6 @@ def __init__(self, name, bases, dct, /, create_pointer_type=True): ns = {'_type_': self} internal_pointer_type = PointerMeta(f"p{name}", p_bases, ns) assert isinstance(internal_pointer_type, PyCPointerType) - assert self.__pointer_type__ is not None assert self.__pointer_type__ is internal_pointer_type class PointerMeta(PyCPointerType): @@ -307,7 +306,6 @@ def __init__(self, name, bases, dct, /, create_pointer_type=True): ns = {'_type_': self} internal_pointer_type = PointerMeta(f"p{name}", p_bases, ns) assert isinstance(internal_pointer_type, PyCPointerType) - assert self.__pointer_type__ is not None assert self.__pointer_type__ is internal_pointer_type class PointerMeta(PyCPointerType): From 2fc600bd540e628e5528f3d0a1f4987f93722394 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Fri, 2 May 2025 13:31:32 +0500 Subject: [PATCH 61/65] Fix typos in test_pointer_set_wrong_type test --- Lib/test/test_ctypes/test_pointers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index 99b9d1b8e3a8f7..66a6569ee2240c 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -296,7 +296,7 @@ def test_pointer_set_type_twice(self): def test_pointer_set_wrong_type(self): int_ptr = POINTER(c_int) - float_ptr = POINTER(float_ptr) + float_ptr = POINTER(c_float) try: class C(c_int): pass @@ -305,7 +305,7 @@ class C(c_int): t2 = POINTER(c_float) t1.set_type(c_float) self.assertEqual(t1(c_float(1.5))[0], 1.5) - self.assertIs(c_int._type_, c_float) + self.assertIs(t1._type_, c_float) self.assertIs(c_int.__pointer_type__, t1) self.assertIs(c_float.__pointer_type__, float_ptr) From cb78a036a380e2e4bfb99f50af711a451a6851bc Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Fri, 2 May 2025 13:32:04 +0500 Subject: [PATCH 62/65] set_pointer for type should reset type cache --- Modules/_ctypes/_ctypes.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 2c0b1301203f2e..9fa64e5814a6e2 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1383,6 +1383,8 @@ PyCPointerType_set_type_impl(PyTypeObject *self, PyTypeObject *cls, return NULL; } + PyType_Modified(self); + Py_DECREF(attrdict); Py_RETURN_NONE; } From b33890d368841605e6c1fa607f8c43f0d9f80f51 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Fri, 2 May 2025 13:52:50 +0500 Subject: [PATCH 63/65] get and getitem should consistent --- Lib/ctypes/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index fade8deac3e29e..d73560a952e260 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -325,7 +325,7 @@ def get(self, cls, default=None): try: return cls.__pointer_type__ except AttributeError: - return default + return _pointer_type_cache_fallback.get(cls, default) def __contains__(self, cls): return hasattr(cls, '__pointer_type__') From 3129ef5771934433e45dd3dad880381000978029 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 2 May 2025 16:22:16 +0200 Subject: [PATCH 64/65] Wording tweak --- Doc/whatsnew/3.14.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index bdd54ff4756dfd..c8f97a2405ed5b 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1683,8 +1683,7 @@ Deprecated * :mod:`ctypes`: Calling :func:`ctypes.POINTER` on a string is deprecated. Use :ref:`ctypes-incomplete-types` for self-referential structures. - Also, ``ctypes._pointer_type_cache`` is deprecated as a courtesy to - existing users of this internal API. + Also, the internal ``ctypes._pointer_type_cache`` is deprecated. See :func:`ctypes.POINTER` for updated implementation details. (Contributed by Sergey Myrianov in :gh:`100926`.) From 8efa28a5d488bbde23fc4cb1cb77f3d12470f3bb Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 2 May 2025 17:09:00 +0200 Subject: [PATCH 65/65] Apply suggestions from code review Co-authored-by: neonene <53406459+neonene@users.noreply.github.com> --- Lib/ctypes/__init__.py | 1 + Lib/test/test_ctypes/test_incomplete.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index d73560a952e260..823a3692fd1bbf 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -342,6 +342,7 @@ class c_wchar(_SimpleCData): _type_ = "u" def _reset_cache(): + _pointer_type_cache_fallback.clear() _c_functype_cache.clear() if _os.name == "nt": _win_functype_cache.clear() diff --git a/Lib/test/test_ctypes/test_incomplete.py b/Lib/test/test_ctypes/test_incomplete.py index 9933ab7f8d2670..fefdfe9102e668 100644 --- a/Lib/test/test_ctypes/test_incomplete.py +++ b/Lib/test/test_ctypes/test_incomplete.py @@ -3,7 +3,7 @@ import warnings from ctypes import Structure, POINTER, pointer, c_char_p -# String-based "incomplete pointers" wers implemented ctypes 0.6.3 (2003, when +# String-based "incomplete pointers" were implemented in ctypes 0.6.3 (2003, when # ctypes was an external project). They made obsolete by the current # incomplete *types* (setting `_fields_` late) in 0.9.5 (2005). # ctypes was added to Python 2.5 (2006), without any mention in docs.