From 799326b0a93ae6375f153d5a6607e7dc5e0690b2 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 11 Nov 2025 13:52:13 +0100 Subject: [PATCH 01/58] gh-141169: Re-raise exception from findfuncptr (GH-141349) --- Include/internal/pycore_importdl.h | 32 ++++++++++++++++++++++++++---- Python/importdl.c | 24 +++------------------- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index 12a32a5f70e51f..f60c5510d20075 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -14,6 +14,34 @@ extern "C" { extern const char *_PyImport_DynLoadFiletab[]; +#ifdef HAVE_DYNAMIC_LOADING +/* ./configure sets HAVE_DYNAMIC_LOADING if dynamic loading of modules is + supported on this platform. configure will then compile and link in one + of the dynload_*.c files, as appropriate. We will call a function in + those modules to get a function pointer to the module's init function. + + The function should return: + - The function pointer on success + - NULL with exception set if the library cannot be loaded + - NULL *without* an extension set if the library could be loaded but the + function cannot be found in it. +*/ +#ifdef MS_WINDOWS +#include +typedef FARPROC dl_funcptr; +extern dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, + const char *shortname, + PyObject *pathname, + FILE *fp); +#else +typedef void (*dl_funcptr)(void); +extern dl_funcptr _PyImport_FindSharedFuncptr(const char *prefix, + const char *shortname, + const char *pathname, FILE *fp); +#endif + +#endif /* HAVE_DYNAMIC_LOADING */ + typedef enum ext_module_kind { _Py_ext_module_kind_UNKNOWN = 0, @@ -112,8 +140,6 @@ extern int _PyImport_RunModInitFunc( #define MAXSUFFIXSIZE 12 #ifdef MS_WINDOWS -#include -typedef FARPROC dl_funcptr; #ifdef Py_DEBUG # define PYD_DEBUG_SUFFIX "_d" @@ -136,8 +162,6 @@ typedef FARPROC dl_funcptr; #define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX "." PYD_SOABI ".pyd" #define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" -#else -typedef void (*dl_funcptr)(void); #endif diff --git a/Python/importdl.c b/Python/importdl.c index 23a55c39677100..61a9cdaf3754c9 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -10,27 +10,6 @@ #include "pycore_runtime.h" // _Py_ID() -/* ./configure sets HAVE_DYNAMIC_LOADING if dynamic loading of modules is - supported on this platform. configure will then compile and link in one - of the dynload_*.c files, as appropriate. We will call a function in - those modules to get a function pointer to the module's init function. -*/ -#ifdef HAVE_DYNAMIC_LOADING - -#ifdef MS_WINDOWS -extern dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, - const char *shortname, - PyObject *pathname, - FILE *fp); -#else -extern dl_funcptr _PyImport_FindSharedFuncptr(const char *prefix, - const char *shortname, - const char *pathname, FILE *fp); -#endif - -#endif /* HAVE_DYNAMIC_LOADING */ - - /***********************************/ /* module info to use when loading */ /***********************************/ @@ -414,6 +393,9 @@ _PyImport_GetModuleExportHooks( *modexport = (PyModExportFunction)exportfunc; return 2; } + if (PyErr_Occurred()) { + return -1; + } exportfunc = findfuncptr( info->hook_prefixes->init_prefix, From 7211a34fe1d9704935342af8c9b46725629f2d97 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Tue, 11 Nov 2025 20:02:32 +0530 Subject: [PATCH 02/58] gh-132657: optimize `PySet_Contains` for `frozenset` (#141183) --- Objects/setobject.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 213bd821d8a1b9..2401176576eb62 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2747,7 +2747,9 @@ PySet_Contains(PyObject *anyset, PyObject *key) PyErr_BadInternalCall(); return -1; } - + if (PyFrozenSet_CheckExact(anyset)) { + return set_contains_key((PySetObject *)anyset, key); + } int rv; Py_BEGIN_CRITICAL_SECTION(anyset); rv = set_contains_key((PySetObject *)anyset, key); From d890aba748e5213585f9f906888999227dc3fa9c Mon Sep 17 00:00:00 2001 From: John Franey <1728528+johnfraney@users.noreply.github.com> Date: Tue, 11 Nov 2025 10:33:56 -0400 Subject: [PATCH 03/58] gh-140942: Add MIME type for .cjs extension (#140937) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/whatsnew/3.15.rst | 1 + Lib/mimetypes.py | 1 + .../2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst | 2 ++ 3 files changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 1ba394a1967403..ef18d36e4d4748 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -451,6 +451,7 @@ math mimetypes --------- +* Add ``application/node`` MIME type for ``.cjs`` extension. (Contributed by John Franey in :gh:`140937`.) * Add ``application/toml``. (Contributed by Gil Forcada in :gh:`139959`.) * Rename ``application/x-texinfo`` to ``application/texinfo``. (Contributed by Charlie Lin in :gh:`140165`) diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 48a9f430d45262..d6896fc4042cb4 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -486,6 +486,7 @@ def _default_mime_types(): '.wiz' : 'application/msword', '.nq' : 'application/n-quads', '.nt' : 'application/n-triples', + '.cjs' : 'application/node', '.bin' : 'application/octet-stream', '.a' : 'application/octet-stream', '.dll' : 'application/octet-stream', diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst new file mode 100644 index 00000000000000..20cfeca1e71dca --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst @@ -0,0 +1,2 @@ +Add ``.cjs`` to :mod:`mimetypes` to give CommonJS modules a MIME type of +``application/node``. From 759a048d4bea522fda2fe929be0fba1650c62b0e Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 11 Nov 2025 12:22:16 -0500 Subject: [PATCH 04/58] gh-141004: Document `PyType_Unwatch` (GH-141414) --- Doc/c-api/type.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 479ede70b01f5d..29ffeb7c483dce 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -116,6 +116,20 @@ Type Objects .. versionadded:: 3.12 +.. c:function:: int PyType_Unwatch(int watcher_id, PyObject *type) + + Mark *type* as not watched. This undoes a previous call to + :c:func:`PyType_Watch`. *type* must not be ``NULL``. + + An extension should never call this function with a *watcher_id* that was + not returned to it by a previous call to :c:func:`PyType_AddWatcher`. + + On success, this function returns ``0``. On failure, this function returns + ``-1`` with an exception set. + + .. versionadded:: 3.12 + + .. c:type:: int (*PyType_WatchCallback)(PyObject *type) Type of a type-watcher callback function. From 713edbcebfdb5aa83e9bf376ebc40255ccacd235 Mon Sep 17 00:00:00 2001 From: Tan Long Date: Wed, 12 Nov 2025 03:27:21 +0800 Subject: [PATCH 05/58] gh-141415: Remove unused variables and comment in `_pyrepl.windows_console.py` (#141416) --- Lib/_pyrepl/windows_console.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index c56dcd6d7dd434..f9f5988af0b9ef 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -249,22 +249,10 @@ def input_hook(self): def __write_changed_line( self, y: int, oldline: str, newline: str, px_coord: int ) -> None: - # this is frustrating; there's no reason to test (say) - # self.dch1 inside the loop -- but alternative ways of - # structuring this function are equally painful (I'm trying to - # avoid writing code generators these days...) minlen = min(wlen(oldline), wlen(newline)) x_pos = 0 x_coord = 0 - px_pos = 0 - j = 0 - for c in oldline: - if j >= px_coord: - break - j += wlen(c) - px_pos += 1 - # reuse the oldline as much as possible, but stop as soon as we # encounter an ESCAPE, because it might be the start of an escape # sequence @@ -358,7 +346,6 @@ def prepare(self) -> None: self.height, self.width = self.getheightwidth() self.posxy = 0, 0 - self.__gone_tall = 0 self.__offset = 0 if self.__vt_support: From 0f09bda643d778fb20fb79fecdfd09f20f9d9717 Mon Sep 17 00:00:00 2001 From: yihong Date: Wed, 12 Nov 2025 03:27:56 +0800 Subject: [PATCH 06/58] gh-140193: Forward port test_exec_set_nomemory_hang from 3.13 (GH-140187) * chore: test_exec_set_nomemory_hang from 3.13 Signed-off-by: yihong0618 * fix: apply comments Signed-off-by: yihong0618 * Update Lib/test/test_exceptions.py Co-authored-by: Peter Bierma * Update Lib/test/test_exceptions.py Co-authored-by: Peter Bierma * fix: windows too long name 60 times is enough Signed-off-by: yihong0618 --------- Signed-off-by: yihong0618 Co-authored-by: Peter Bierma --- Lib/test/test_exceptions.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 5262b58908a509..6f212d2f91efb1 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1923,6 +1923,39 @@ def test_keyerror_context(self): exc2 = None + @cpython_only + # Python built with Py_TRACE_REFS fail with a fatal error in + # _PyRefchain_Trace() on memory allocation error. + @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + def test_exec_set_nomemory_hang(self): + import_module("_testcapi") + # gh-134163: A MemoryError inside code that was wrapped by a try/except + # block would lead to an infinite loop. + + # The frame_lasti needs to be greater than 257 to prevent + # PyLong_FromLong() from returning cached integers, which + # don't require a memory allocation. Prepend some dummy code + # to artificially increase the instruction index. + warmup_code = "a = list(range(0, 1))\n" * 60 + user_input = warmup_code + dedent(""" + try: + import _testcapi + _testcapi.set_nomemory(0) + b = list(range(1000, 2000)) + except Exception as e: + import traceback + traceback.print_exc() + """) + with SuppressCrashReport(): + with script_helper.spawn_python('-c', user_input) as p: + p.wait() + output = p.stdout.read() + + self.assertIn(p.returncode, (0, 1)) + self.assertGreater(len(output), 0) # At minimum, should not hang + self.assertIn(b"MemoryError", output) + + class NameErrorTests(unittest.TestCase): def test_name_error_has_name(self): try: From c903d768322989e9f8ba79e38ee87e14c85c5430 Mon Sep 17 00:00:00 2001 From: Marco Barbosa Date: Tue, 11 Nov 2025 16:35:55 -0300 Subject: [PATCH 07/58] gh-139533: fix refs to code without proper markups on turtledemo doc (GH-139534) gh-139533: fix refs to code without proper markups on turtledemo documentation --- Doc/library/turtle.rst | 124 ++++++++++++++++++++--------------------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 58b99e0d44173a..95a57c57e71d56 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -2801,68 +2801,68 @@ The demo scripts are: .. tabularcolumns:: |l|L|L| -+----------------+------------------------------+-----------------------+ -| Name | Description | Features | -+================+==============================+=======================+ -| bytedesign | complex classical | :func:`tracer`, delay,| -| | turtle graphics pattern | :func:`update` | -+----------------+------------------------------+-----------------------+ -| chaos | graphs Verhulst dynamics, | world coordinates | -| | shows that computer's | | -| | computations can generate | | -| | results sometimes against the| | -| | common sense expectations | | -+----------------+------------------------------+-----------------------+ -| clock | analog clock showing time | turtles as clock's | -| | of your computer | hands, ontimer | -+----------------+------------------------------+-----------------------+ -| colormixer | experiment with r, g, b | :func:`ondrag` | -+----------------+------------------------------+-----------------------+ -| forest | 3 breadth-first trees | randomization | -+----------------+------------------------------+-----------------------+ -| fractalcurves | Hilbert & Koch curves | recursion | -+----------------+------------------------------+-----------------------+ -| lindenmayer | ethnomathematics | L-System | -| | (indian kolams) | | -+----------------+------------------------------+-----------------------+ -| minimal_hanoi | Towers of Hanoi | Rectangular Turtles | -| | | as Hanoi discs | -| | | (shape, shapesize) | -+----------------+------------------------------+-----------------------+ -| nim | play the classical nim game | turtles as nimsticks, | -| | with three heaps of sticks | event driven (mouse, | -| | against the computer. | keyboard) | -+----------------+------------------------------+-----------------------+ -| paint | super minimalistic | :func:`onclick` | -| | drawing program | | -+----------------+------------------------------+-----------------------+ -| peace | elementary | turtle: appearance | -| | | and animation | -+----------------+------------------------------+-----------------------+ -| penrose | aperiodic tiling with | :func:`stamp` | -| | kites and darts | | -+----------------+------------------------------+-----------------------+ -| planet_and_moon| simulation of | compound shapes, | -| | gravitational system | :class:`Vec2D` | -+----------------+------------------------------+-----------------------+ -| rosette | a pattern from the wikipedia | :func:`clone`, | -| | article on turtle graphics | :func:`undo` | -+----------------+------------------------------+-----------------------+ -| round_dance | dancing turtles rotating | compound shapes, clone| -| | pairwise in opposite | shapesize, tilt, | -| | direction | get_shapepoly, update | -+----------------+------------------------------+-----------------------+ -| sorting_animate| visual demonstration of | simple alignment, | -| | different sorting methods | randomization | -+----------------+------------------------------+-----------------------+ -| tree | a (graphical) breadth | :func:`clone` | -| | first tree (using generators)| | -+----------------+------------------------------+-----------------------+ -| two_canvases | simple design | turtles on two | -| | | canvases | -+----------------+------------------------------+-----------------------+ -| yinyang | another elementary example | :func:`circle` | -+----------------+------------------------------+-----------------------+ ++------------------------+------------------------------+--------------------------------------+ +| Name | Description | Features | ++========================+==============================+======================================+ +| ``bytedesign`` | complex classical | :func:`tracer`, :func:`delay`, | +| | turtle graphics pattern | :func:`update` | ++------------------------+------------------------------+--------------------------------------+ +| ``chaos`` | graphs Verhulst dynamics, | world coordinates | +| | shows that computer's | | +| | computations can generate | | +| | results sometimes against the| | +| | common sense expectations | | ++------------------------+------------------------------+--------------------------------------+ +| ``clock`` | analog clock showing time | turtles as clock's | +| | of your computer | hands, :func:`ontimer` | ++------------------------+------------------------------+--------------------------------------+ +| ``colormixer`` | experiment with r, g, b | :func:`ondrag` | ++------------------------+------------------------------+--------------------------------------+ +| ``forest`` | 3 breadth-first trees | randomization | ++------------------------+------------------------------+--------------------------------------+ +| ``fractalcurves`` | Hilbert & Koch curves | recursion | ++------------------------+------------------------------+--------------------------------------+ +| ``lindenmayer`` | ethnomathematics | L-System | +| | (indian kolams) | | ++------------------------+------------------------------+--------------------------------------+ +| ``minimal_hanoi`` | Towers of Hanoi | Rectangular Turtles | +| | | as Hanoi discs | +| | | (:func:`shape`, :func:`shapesize`) | ++------------------------+------------------------------+--------------------------------------+ +| ``nim`` | play the classical nim game | turtles as nimsticks, | +| | with three heaps of sticks | event driven (mouse, | +| | against the computer. | keyboard) | ++------------------------+------------------------------+--------------------------------------+ +| ``paint`` | super minimalistic | :func:`onclick` | +| | drawing program | | ++------------------------+------------------------------+--------------------------------------+ +| ``peace`` | elementary | turtle: appearance | +| | | and animation | ++------------------------+------------------------------+--------------------------------------+ +| ``penrose`` | aperiodic tiling with | :func:`stamp` | +| | kites and darts | | ++------------------------+------------------------------+--------------------------------------+ +| ``planet_and_moon`` | simulation of | compound shapes, | +| | gravitational system | :class:`Vec2D` | ++------------------------+------------------------------+--------------------------------------+ +| ``rosette`` | a pattern from the wikipedia | :func:`clone`, | +| | article on turtle graphics | :func:`undo` | ++------------------------+------------------------------+--------------------------------------+ +| ``round_dance`` | dancing turtles rotating | compound shapes, :func:`clone` | +| | pairwise in opposite | :func:`shapesize`, :func:`tilt`, | +| | direction | :func:`get_shapepoly`, :func:`update`| ++------------------------+------------------------------+--------------------------------------+ +| ``sorting_animate`` | visual demonstration of | simple alignment, | +| | different sorting methods | randomization | ++------------------------+------------------------------+--------------------------------------+ +| ``tree`` | a (graphical) breadth | :func:`clone` | +| | first tree (using generators)| | ++------------------------+------------------------------+--------------------------------------+ +| ``two_canvases`` | simple design | turtles on two | +| | | canvases | ++------------------------+------------------------------+--------------------------------------+ +| ``yinyang`` | another elementary example | :func:`circle` | ++------------------------+------------------------------+--------------------------------------+ Have fun! From 336154f4b0dbcf1d9dbb461ae962d558ba60f452 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 11 Nov 2025 20:02:49 +0000 Subject: [PATCH 08/58] Add documentation for Python install manager's install_dir, global_dir and download_dir (GH-140223) --- Doc/using/windows.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 0b98cfb8d270a4..e6619b73bd2c26 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -457,6 +457,25 @@ customization. - Specify the default format used by the ``py list`` command. By default, ``table``. + * - ``install_dir`` + - (none) + - Specify the root directory that runtimes will be installed into. + If you change this setting, previously installed runtimes will not be + usable unless you move them to the new location. + + * - ``global_dir`` + - (none) + - Specify the directory where global commands (such as ``python3.14.exe``) + are stored. + This directory should be added to your :envvar:`PATH` to make the + commands available from your terminal. + + * - ``download_dir`` + - (none) + - Specify the directory where downloaded files are stored. + This directory is a temporary cache, and can be cleaned up from time to + time. + Dotted names should be nested inside JSON objects, for example, ``list.format`` would be specified as ``{"list": {"format": "table"}}``. From 2fb2b82161c6df57c4a247cb743816b79134e932 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Tue, 11 Nov 2025 23:16:46 +0300 Subject: [PATCH 09/58] gh-141367: Use actual SPECIALIZATION_THRESHOLD value in specialization related test (GH-141417) --- Lib/test/test_list.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 223f34fb696a6e..642b54d34849da 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -349,10 +349,12 @@ def test_deopt_from_append_list(self): # gh-132011: it used to crash, because # of `CALL_LIST_APPEND` specialization failure. code = textwrap.dedent(""" + import _testinternalcapi + l = [] def lappend(l, x, y): l.append((x, y)) - for x in range(3): + for x in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): lappend(l, None, None) try: lappend(list, None, None) From b5196fa15a6c5aaa90eafff06206f8e44a9da216 Mon Sep 17 00:00:00 2001 From: Aniket <148300120+Aniketsy@users.noreply.github.com> Date: Wed, 12 Nov 2025 01:55:26 +0530 Subject: [PATCH 10/58] gh-137339: Clarify host and port parameter behavior in smtplib.SMTP{_SSL} initialization (#137340) This also documents the previously undocumented default_port parameter. Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/library/smtplib.rst | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index c5a3de52090cee..3ee8b82a1880f3 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -24,10 +24,13 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). .. class:: SMTP(host='', port=0, local_hostname=None[, timeout], source_address=None) An :class:`SMTP` instance encapsulates an SMTP connection. It has methods - that support a full repertoire of SMTP and ESMTP operations. If the optional - *host* and *port* parameters are given, the SMTP :meth:`connect` method is - called with those parameters during initialization. If specified, - *local_hostname* is used as the FQDN of the local host in the HELO/EHLO + that support a full repertoire of SMTP and ESMTP operations. + + If the host parameter is set to a truthy value, :meth:`SMTP.connect` is called with + host and port automatically when the object is created; otherwise, :meth:`!connect` must + be called manually. + + If specified, *local_hostname* is used as the FQDN of the local host in the HELO/EHLO command. Otherwise, the local hostname is found using :func:`socket.getfqdn`. If the :meth:`connect` call returns anything other than a success code, an :exc:`SMTPConnectError` is raised. The optional @@ -62,6 +65,10 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). ``smtplib.SMTP.send`` with arguments ``self`` and ``data``, where ``data`` is the bytes about to be sent to the remote host. + .. attribute:: SMTP.default_port + + The default port used for SMTP connections (25). + .. versionchanged:: 3.3 Support for the :keyword:`with` statement was added. @@ -80,15 +87,23 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). An :class:`SMTP_SSL` instance behaves exactly the same as instances of :class:`SMTP`. :class:`SMTP_SSL` should be used for situations where SSL is - required from the beginning of the connection and using :meth:`~SMTP.starttls` - is not appropriate. If *host* is not specified, the local host is used. If - *port* is zero, the standard SMTP-over-SSL port (465) is used. The optional - arguments *local_hostname*, *timeout* and *source_address* have the same + required from the beginning of the connection and using :meth:`SMTP.starttls` is + not appropriate. + + If the host parameter is set to a truthy value, :meth:`SMTP.connect` is called with host + and port automatically when the object is created; otherwise, :meth:`!SMTP.connect` must + be called manually. + + The optional arguments *local_hostname*, *timeout* and *source_address* have the same meaning as they do in the :class:`SMTP` class. *context*, also optional, can contain a :class:`~ssl.SSLContext` and allows configuring various aspects of the secure connection. Please read :ref:`ssl-security` for best practices. + .. attribute:: SMTP_SSL.default_port + + The default port used for SMTP-over-SSL connections (465). + .. versionchanged:: 3.3 *context* was added. @@ -259,6 +274,9 @@ An :class:`SMTP` instance has the following methods: 2-tuple of the response code and message sent by the server in its connection response. + If port is not changed from its default value of 0, the value of the :attr:`default_port` + attribute is used. + .. audit-event:: smtplib.connect self,host,port smtplib.SMTP.connect From 298e9074cdffb09d518e6aceea556e8f4a8a745d Mon Sep 17 00:00:00 2001 From: Alper Date: Tue, 11 Nov 2025 12:27:21 -0800 Subject: [PATCH 11/58] gh-140476: optimize `PySet_Add` for `frozenset` in free-threading (#140440) Avoids critical section in `PySet_Add` when adding items to newly created frozensets. Co-authored-by: Kumar Aditya --- ...-10-22-12-48-05.gh-issue-140476.F3-d1P.rst | 2 ++ Objects/setobject.c | 25 ++++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst new file mode 100644 index 00000000000000..a24033208c558c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst @@ -0,0 +1,2 @@ +Optimize :c:func:`PySet_Add` for :class:`frozenset` in :term:`free threaded +` build. diff --git a/Objects/setobject.c b/Objects/setobject.c index 2401176576eb62..85f4d7d403178a 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2775,17 +2775,24 @@ PySet_Discard(PyObject *set, PyObject *key) int PySet_Add(PyObject *anyset, PyObject *key) { - if (!PySet_Check(anyset) && - (!PyFrozenSet_Check(anyset) || !_PyObject_IsUniquelyReferenced(anyset))) { - PyErr_BadInternalCall(); - return -1; + if (PySet_Check(anyset)) { + int rv; + Py_BEGIN_CRITICAL_SECTION(anyset); + rv = set_add_key((PySetObject *)anyset, key); + Py_END_CRITICAL_SECTION(); + return rv; } - int rv; - Py_BEGIN_CRITICAL_SECTION(anyset); - rv = set_add_key((PySetObject *)anyset, key); - Py_END_CRITICAL_SECTION(); - return rv; + if (PyFrozenSet_Check(anyset) && _PyObject_IsUniquelyReferenced(anyset)) { + // We can only change frozensets if they are uniquely referenced. The + // API limits the usage of `PySet_Add` to "fill in the values of brand + // new frozensets before they are exposed to other code". In this case, + // this can be done without a lock. + return set_add_key((PySetObject *)anyset, key); + } + + PyErr_BadInternalCall(); + return -1; } int From 2befce86e699fdbb6610949b029bad56a0d0780f Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 11 Nov 2025 15:31:29 -0500 Subject: [PATCH 12/58] gh-141004: Document `PyFile_OpenCode` and `PyFile_OpenCodeObject` (GH-141413) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/file.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index e9019a0d500f7e..9d01254ddb2a11 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -93,6 +93,29 @@ the :mod:`io` APIs instead. .. versionadded:: 3.8 +.. c:function:: PyObject *PyFile_OpenCodeObject(PyObject *path) + + Open *path* with the mode ``'rb'``. *path* must be a Python :class:`str` + object. The behavior of this function may be overridden by + :c:func:`PyFile_SetOpenCodeHook` to allow for some preprocessing of the + text. + + This is analogous to :func:`io.open_code` in Python. + + On success, this function returns a :term:`strong reference` to a Python + file object. On failure, this function returns ``NULL`` with an exception + set. + + .. versionadded:: 3.8 + + +.. c:function:: PyObject *PyFile_OpenCode(const char *path) + + Similar to :c:func:`PyFile_OpenCodeObject`, but *path* is a + UTF-8 encoded :c:expr:`const char*`. + + .. versionadded:: 3.8 + .. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags) From c13b59204af562bfb022eb8f6a5c03eb82659531 Mon Sep 17 00:00:00 2001 From: Alper Date: Tue, 11 Nov 2025 12:31:55 -0800 Subject: [PATCH 13/58] gh-116738: use `PyMutex` in `lzma` module (#140711) Co-authored-by: Kumar Aditya --- Lib/test/test_free_threading/test_lzma.py | 56 +++++++++++++++++++++++ Modules/_lzmamodule.c | 46 ++++++------------- 2 files changed, 69 insertions(+), 33 deletions(-) create mode 100644 Lib/test/test_free_threading/test_lzma.py diff --git a/Lib/test/test_free_threading/test_lzma.py b/Lib/test/test_free_threading/test_lzma.py new file mode 100644 index 00000000000000..38d7e5db489426 --- /dev/null +++ b/Lib/test/test_free_threading/test_lzma.py @@ -0,0 +1,56 @@ +import unittest + +from test.support import import_helper, threading_helper +from test.support.threading_helper import run_concurrently + +lzma = import_helper.import_module("lzma") +from lzma import LZMACompressor, LZMADecompressor + +from test.test_lzma import INPUT + + +NTHREADS = 10 + + +@threading_helper.requires_working_threading() +class TestLZMA(unittest.TestCase): + def test_compressor(self): + lzc = LZMACompressor() + + # First compress() outputs LZMA header + header = lzc.compress(INPUT) + self.assertGreater(len(header), 0) + + def worker(): + # it should return empty bytes as it buffers data internally + data = lzc.compress(INPUT) + self.assertEqual(data, b"") + + run_concurrently(worker_func=worker, nthreads=NTHREADS - 1) + full_compressed = header + lzc.flush() + decompressed = lzma.decompress(full_compressed) + # The decompressed data should be INPUT repeated NTHREADS times + self.assertEqual(decompressed, INPUT * NTHREADS) + + def test_decompressor(self): + chunk_size = 128 + chunks = [bytes([ord("a") + i]) * chunk_size for i in range(NTHREADS)] + input_data = b"".join(chunks) + compressed = lzma.compress(input_data) + + lzd = LZMADecompressor() + output = [] + + def worker(): + data = lzd.decompress(compressed, chunk_size) + self.assertEqual(len(data), chunk_size) + output.append(data) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + self.assertEqual(len(output), NTHREADS) + # Verify the expected chunks (order doesn't matter due to append race) + self.assertSetEqual(set(output), set(chunks)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index 6fc072f6d0a382..5876623399837b 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -72,13 +72,6 @@ OutputBuffer_OnError(_BlocksOutputBuffer *buffer) } -#define ACQUIRE_LOCK(obj) do { \ - if (!PyThread_acquire_lock((obj)->lock, 0)) { \ - Py_BEGIN_ALLOW_THREADS \ - PyThread_acquire_lock((obj)->lock, 1); \ - Py_END_ALLOW_THREADS \ - } } while (0) -#define RELEASE_LOCK(obj) PyThread_release_lock((obj)->lock) typedef struct { PyTypeObject *lzma_compressor_type; @@ -111,7 +104,7 @@ typedef struct { lzma_allocator alloc; lzma_stream lzs; int flushed; - PyThread_type_lock lock; + PyMutex mutex; } Compressor; typedef struct { @@ -124,7 +117,7 @@ typedef struct { char needs_input; uint8_t *input_buffer; size_t input_buffer_size; - PyThread_type_lock lock; + PyMutex mutex; } Decompressor; #define Compressor_CAST(op) ((Compressor *)(op)) @@ -617,14 +610,14 @@ _lzma_LZMACompressor_compress_impl(Compressor *self, Py_buffer *data) { PyObject *result = NULL; - ACQUIRE_LOCK(self); + PyMutex_Lock(&self->mutex); if (self->flushed) { PyErr_SetString(PyExc_ValueError, "Compressor has been flushed"); } else { result = compress(self, data->buf, data->len, LZMA_RUN); } - RELEASE_LOCK(self); + PyMutex_Unlock(&self->mutex); return result; } @@ -644,14 +637,14 @@ _lzma_LZMACompressor_flush_impl(Compressor *self) { PyObject *result = NULL; - ACQUIRE_LOCK(self); + PyMutex_Lock(&self->mutex); if (self->flushed) { PyErr_SetString(PyExc_ValueError, "Repeated call to flush()"); } else { self->flushed = 1; result = compress(self, NULL, 0, LZMA_FINISH); } - RELEASE_LOCK(self); + PyMutex_Unlock(&self->mutex); return result; } @@ -820,12 +813,7 @@ Compressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) self->alloc.free = PyLzma_Free; self->lzs.allocator = &self->alloc; - self->lock = PyThread_allocate_lock(); - if (self->lock == NULL) { - Py_DECREF(self); - PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - return NULL; - } + self->mutex = (PyMutex){0}; self->flushed = 0; switch (format) { @@ -867,10 +855,8 @@ static void Compressor_dealloc(PyObject *op) { Compressor *self = Compressor_CAST(op); + assert(!PyMutex_IsLocked(&self->mutex)); lzma_end(&self->lzs); - if (self->lock != NULL) { - PyThread_free_lock(self->lock); - } PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); Py_DECREF(tp); @@ -1146,12 +1132,12 @@ _lzma_LZMADecompressor_decompress_impl(Decompressor *self, Py_buffer *data, { PyObject *result = NULL; - ACQUIRE_LOCK(self); + PyMutex_Lock(&self->mutex); if (self->eof) PyErr_SetString(PyExc_EOFError, "Already at end of stream"); else result = decompress(self, data->buf, data->len, max_length); - RELEASE_LOCK(self); + PyMutex_Unlock(&self->mutex); return result; } @@ -1244,12 +1230,7 @@ _lzma_LZMADecompressor_impl(PyTypeObject *type, int format, self->lzs.allocator = &self->alloc; self->lzs.next_in = NULL; - self->lock = PyThread_allocate_lock(); - if (self->lock == NULL) { - Py_DECREF(self); - PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - return NULL; - } + self->mutex = (PyMutex){0}; self->check = LZMA_CHECK_UNKNOWN; self->needs_input = 1; @@ -1304,14 +1285,13 @@ static void Decompressor_dealloc(PyObject *op) { Decompressor *self = Decompressor_CAST(op); + assert(!PyMutex_IsLocked(&self->mutex)); + if(self->input_buffer != NULL) PyMem_Free(self->input_buffer); lzma_end(&self->lzs); Py_CLEAR(self->unused_data); - if (self->lock != NULL) { - PyThread_free_lock(self->lock); - } PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); Py_DECREF(tp); From 37e2762ee12c2d7fc465938d7161a9a0640bd71f Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 11 Nov 2025 15:32:54 -0500 Subject: [PATCH 14/58] gh-141004: Document `PyBytes_Repr` and `PyBytes_DecodeEscape` (GH-141407) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/bytes.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 865a9e5d2bf5d5..82c2557368371f 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -228,6 +228,42 @@ called with a non-bytes parameter. The function is :term:`soft deprecated`, use the :c:type:`PyBytesWriter` API instead. + +.. c:function:: PyObject *PyBytes_Repr(PyObject *bytes, int smartquotes) + + Get the string representation of *bytes*. This function is currently used to + implement :meth:`!bytes.__repr__` in Python. + + This function does not do type checking; it is undefined behavior to pass + *bytes* as a non-bytes object or ``NULL``. + + If *smartquotes* is true, the representation will use a double-quoted string + instead of single-quoted string when single-quotes are present in *bytes*. + For example, the byte string ``'Python'`` would be represented as + ``b"'Python'"`` when *smartquotes* is true, or ``b'\'Python\''`` when it is + false. + + On success, this function returns a :term:`strong reference` to a + :class:`str` object containing the representation. On failure, this + returns ``NULL`` with an exception set. + + +.. c:function:: PyObject *PyBytes_DecodeEscape(const char *s, Py_ssize_t len, const char *errors, Py_ssize_t unicode, const char *recode_encoding) + + Unescape a backslash-escaped string *s*. *s* must not be ``NULL``. + *len* must be the size of *s*. + + *errors* must be one of ``"strict"``, ``"replace"``, or ``"ignore"``. If + *errors* is ``NULL``, then ``"strict"`` is used by default. + + On success, this function returns a :term:`strong reference` to a Python + :class:`bytes` object containing the unescaped string. On failure, this + function returns ``NULL`` with an exception set. + + .. versionchanged:: 3.9 + *unicode* and *recode_encoding* are now unused. + + .. _pybyteswriter: PyBytesWriter From af80fac42548719ede7241bfbab3c2c0775b4760 Mon Sep 17 00:00:00 2001 From: Mohsin Mehmood <55545648+mohsinm-dev@users.noreply.github.com> Date: Wed, 12 Nov 2025 02:49:54 +0500 Subject: [PATCH 15/58] gh-141314: Fix TextIOWrapper.tell() assertion failure with standalone carriage return (GH-141331) The assertion was checking wrong variable (skip_back vs skip_bytes). --- Lib/test/test_io/test_textio.py | 19 +++++++++++++++++++ ...-11-10-01-47-18.gh-issue-141314.baaa28.rst | 1 + Modules/_io/textio.c | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst diff --git a/Lib/test/test_io/test_textio.py b/Lib/test/test_io/test_textio.py index d8d0928b4ba69b..6331ed2b958552 100644 --- a/Lib/test/test_io/test_textio.py +++ b/Lib/test/test_io/test_textio.py @@ -686,6 +686,25 @@ def test_multibyte_seek_and_tell(self): self.assertEqual(f.tell(), p1) f.close() + def test_tell_after_readline_with_cr(self): + # Test for gh-141314: TextIOWrapper.tell() assertion failure + # when dealing with standalone carriage returns + data = b'line1\r' + with self.open(os_helper.TESTFN, "wb") as f: + f.write(data) + + with self.open(os_helper.TESTFN, "r") as f: + # Read line that ends with \r + line = f.readline() + self.assertEqual(line, "line1\n") + # This should not cause an assertion failure + pos = f.tell() + # Verify we can seek back to this position + f.seek(pos) + remaining = f.read() + self.assertEqual(remaining, "") + + def test_seek_with_encoder_state(self): f = self.open(os_helper.TESTFN, "w", encoding="euc_jis_2004") f.write("\u00e6\u0300") diff --git a/Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst b/Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst new file mode 100644 index 00000000000000..37acaabfa3eada --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst @@ -0,0 +1 @@ +Fix assertion failure in :meth:`io.TextIOWrapper.tell` when reading files with standalone carriage return (``\r``) line endings. diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 84b7d9df400578..65da300abcf3bc 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2845,7 +2845,7 @@ _io_TextIOWrapper_tell_impl(textio *self) current pos */ skip_bytes = (Py_ssize_t) (self->b2cratio * chars_to_skip); skip_back = 1; - assert(skip_back <= PyBytes_GET_SIZE(next_input)); + assert(skip_bytes <= PyBytes_GET_SIZE(next_input)); input = PyBytes_AS_STRING(next_input); while (skip_bytes > 0) { /* Decode up to temptative start point */ From c744ccb2c92746bc7be6316ab478dbc13e176e97 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 11 Nov 2025 21:51:22 +0000 Subject: [PATCH 16/58] GH-139596: Cease caching config.cache & ccache in GH Actions (GH-139623) * Cease caching config.cache in GH Actions\ * Remove ccache action --- .github/workflows/build.yml | 52 --------------------------- .github/workflows/reusable-macos.yml | 5 --- .github/workflows/reusable-san.yml | 10 ------ .github/workflows/reusable-ubuntu.yml | 10 ------ .github/workflows/reusable-wasi.yml | 21 +---------- 5 files changed, 1 insertion(+), 97 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6aa99928278294..a0f60c30ac8a60 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -109,20 +109,10 @@ jobs: python-version: '3.x' - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}-${{ env.pythonLocation }} - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Add ccache to PATH run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Configure CPython run: | # Build Python with the libpython dynamic library @@ -278,11 +268,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies @@ -304,10 +289,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Configure CPython run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$OPENSSL_DIR" - name: Build CPython @@ -339,11 +320,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies @@ -370,10 +346,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Configure CPython run: | ./configure CFLAGS="-fdiagnostics-format=json" \ @@ -479,10 +451,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" @@ -493,11 +461,6 @@ jobs: run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -581,11 +544,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies @@ -611,11 +569,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Configure CPython run: ./configure --config-cache --with-address-sanitizer --without-pymalloc - name: Build CPython @@ -662,11 +615,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Set build dir diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 3d310ae695bfe0..d85c46b96f873d 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -36,11 +36,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - name: Install Homebrew dependencies run: | brew install pkg-config openssl@3.0 xz gdbm tcl-tk@9 make diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index e6ff02e4838ee6..7fe96d1b238b04 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -34,11 +34,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.sanitizer }}-${{ inputs.config_hash }} - name: Install dependencies run: | sudo ./.github/workflows/posix-deps-apt.sh @@ -77,11 +72,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Configure CPython run: >- ./configure diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 7f8b9fdf5d6639..7b93b5f51b00df 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -64,11 +64,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" @@ -79,11 +74,6 @@ jobs: run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} # `test_unpickle_module_race` writes to the source directory, which is diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 18feb564822116..8f412288f530bc 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -42,11 +42,6 @@ jobs: mkdir "${WASI_SDK_PATH}" && \ curl -s -S --location "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-arm64-linux.tar.gz" | \ tar --strip-components 1 --directory "${WASI_SDK_PATH}" --extract --gunzip - - name: "Configure ccache action" - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: "Add ccache to PATH" run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: "Install Python" @@ -55,24 +50,10 @@ jobs: python-version: '3.x' - name: "Runner image version" run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: "Restore Python build config.cache" - uses: actions/cache@v4 - with: - path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache - # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python. - # Include the hash of `Tools/wasm/wasi/__main__.py` as it may change the environment variables. - # (Make sure to keep the key in sync with the other config.cache step below.) - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi/__main__.py') }}-${{ env.pythonLocation }} - name: "Configure build Python" run: python3 Tools/wasm/wasi configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" - run: python3 Tools/wasm/wasi make-build-python - - name: "Restore host config.cache" - uses: actions/cache@v4 - with: - path: ${{ env.CROSS_BUILD_WASI }}/config.cache - # Should be kept in sync with the other config.cache step above. - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi/__main__.py') }}-${{ env.pythonLocation }} + run: python3 Tools/wasm/wasi.py make-build-python - name: "Configure host" # `--with-pydebug` inferred from configure-build-python run: python3 Tools/wasm/wasi configure-host -- --config-cache From 7906f4d96a8fffbee9f4d4991019595878ad54e9 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 12 Nov 2025 00:01:25 +0200 Subject: [PATCH 17/58] gh-132686: Add parameters inherit_class_doc and fallback_to_class_doc for inspect.getdoc() (GH-132691) --- Doc/library/inspect.rst | 17 +++- Doc/whatsnew/3.15.rst | 8 ++ Lib/inspect.py | 34 +++++-- Lib/pydoc.py | 92 +------------------ Lib/test/test_inspect/test_inspect.py | 27 ++++++ ...-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst | 2 + 6 files changed, 79 insertions(+), 101 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 2b3b294ff33a64..aff53b78c4a774 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -619,17 +619,26 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Retrieving source code ---------------------- -.. function:: getdoc(object) +.. function:: getdoc(object, *, inherit_class_doc=True, fallback_to_class_doc=True) Get the documentation string for an object, cleaned up with :func:`cleandoc`. - If the documentation string for an object is not provided and the object is - a class, a method, a property or a descriptor, retrieve the documentation - string from the inheritance hierarchy. + If the documentation string for an object is not provided: + + * if the object is a class and *inherit_class_doc* is true (by default), + retrieve the documentation string from the inheritance hierarchy; + * if the object is a method, a property or a descriptor, retrieve + the documentation string from the inheritance hierarchy; + * otherwise, if *fallback_to_class_doc* is true (by default), retrieve + the documentation string from the class of the object. + Return ``None`` if the documentation string is invalid or missing. .. versionchanged:: 3.5 Documentation strings are now inherited if not overridden. + .. versionchanged:: next + Added parameters *inherit_class_doc* and *fallback_to_class_doc*. + .. function:: getcomments(object) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index ef18d36e4d4748..ecab0d03e105e6 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -429,6 +429,14 @@ http.cookies (Contributed by Nick Burns and Senthil Kumaran in :gh:`92936`.) +inspect +------- + +* Add parameters *inherit_class_doc* and *fallback_to_class_doc* + for :func:`~inspect.getdoc`. + (Contributed by Serhiy Storchaka in :gh:`132686`.) + + locale ------ diff --git a/Lib/inspect.py b/Lib/inspect.py index bb22bab3040fcb..bb17848b444b67 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -706,8 +706,8 @@ def _findclass(func): return None return cls -def _finddoc(obj): - if isclass(obj): +def _finddoc(obj, *, search_in_class=True): + if search_in_class and isclass(obj): for base in obj.__mro__: if base is not object: try: @@ -767,19 +767,37 @@ def _finddoc(obj): return doc return None -def getdoc(object): +def _getowndoc(obj): + """Get the documentation string for an object if it is not + inherited from its class.""" + try: + doc = object.__getattribute__(obj, '__doc__') + if doc is None: + return None + if obj is not type: + typedoc = type(obj).__doc__ + if isinstance(typedoc, str) and typedoc == doc: + return None + return doc + except AttributeError: + return None + +def getdoc(object, *, fallback_to_class_doc=True, inherit_class_doc=True): """Get the documentation string for an object. All tabs are expanded to spaces. To clean up docstrings that are indented to line up with blocks of code, any whitespace than can be uniformly removed from the second line onwards is removed.""" - try: - doc = object.__doc__ - except AttributeError: - return None + if fallback_to_class_doc: + try: + doc = object.__doc__ + except AttributeError: + return None + else: + doc = _getowndoc(object) if doc is None: try: - doc = _finddoc(object) + doc = _finddoc(object, search_in_class=inherit_class_doc) except (AttributeError, TypeError): return None if not isinstance(doc, str): diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 989fbd517d8d83..45ff5fca308c14 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -108,96 +108,10 @@ def pathdirs(): normdirs.append(normdir) return dirs -def _findclass(func): - cls = sys.modules.get(func.__module__) - if cls is None: - return None - for name in func.__qualname__.split('.')[:-1]: - cls = getattr(cls, name) - if not inspect.isclass(cls): - return None - return cls - -def _finddoc(obj): - if inspect.ismethod(obj): - name = obj.__func__.__name__ - self = obj.__self__ - if (inspect.isclass(self) and - getattr(getattr(self, name, None), '__func__') is obj.__func__): - # classmethod - cls = self - else: - cls = self.__class__ - elif inspect.isfunction(obj): - name = obj.__name__ - cls = _findclass(obj) - if cls is None or getattr(cls, name) is not obj: - return None - elif inspect.isbuiltin(obj): - name = obj.__name__ - self = obj.__self__ - if (inspect.isclass(self) and - self.__qualname__ + '.' + name == obj.__qualname__): - # classmethod - cls = self - else: - cls = self.__class__ - # Should be tested before isdatadescriptor(). - elif isinstance(obj, property): - name = obj.__name__ - cls = _findclass(obj.fget) - if cls is None or getattr(cls, name) is not obj: - return None - elif inspect.ismethoddescriptor(obj) or inspect.isdatadescriptor(obj): - name = obj.__name__ - cls = obj.__objclass__ - if getattr(cls, name) is not obj: - return None - if inspect.ismemberdescriptor(obj): - slots = getattr(cls, '__slots__', None) - if isinstance(slots, dict) and name in slots: - return slots[name] - else: - return None - for base in cls.__mro__: - try: - doc = _getowndoc(getattr(base, name)) - except AttributeError: - continue - if doc is not None: - return doc - return None - -def _getowndoc(obj): - """Get the documentation string for an object if it is not - inherited from its class.""" - try: - doc = object.__getattribute__(obj, '__doc__') - if doc is None: - return None - if obj is not type: - typedoc = type(obj).__doc__ - if isinstance(typedoc, str) and typedoc == doc: - return None - return doc - except AttributeError: - return None - def _getdoc(object): - """Get the documentation string for an object. - - All tabs are expanded to spaces. To clean up docstrings that are - indented to line up with blocks of code, any whitespace than can be - uniformly removed from the second line onwards is removed.""" - doc = _getowndoc(object) - if doc is None: - try: - doc = _finddoc(object) - except (AttributeError, TypeError): - return None - if not isinstance(doc, str): - return None - return inspect.cleandoc(doc) + return inspect.getdoc(object, + fallback_to_class_doc=False, + inherit_class_doc=False) def getdoc(object): """Get the doc string or comments for an object.""" diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index d42f2dbff99cae..24fd4a2fa626d4 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -688,10 +688,37 @@ def test_getdoc_inherited(self): self.assertEqual(inspect.getdoc(mod.FesteringGob.contradiction), 'The automatic gainsaying.') + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_getdoc_inherited_class_doc(self): + class A: + """Common base class""" + class B(A): + pass + + a = A() + self.assertEqual(inspect.getdoc(A), 'Common base class') + self.assertEqual(inspect.getdoc(A, inherit_class_doc=False), + 'Common base class') + self.assertEqual(inspect.getdoc(a), 'Common base class') + self.assertIsNone(inspect.getdoc(a, fallback_to_class_doc=False)) + a.__doc__ = 'Instance' + self.assertEqual(inspect.getdoc(a, fallback_to_class_doc=False), + 'Instance') + + b = B() + self.assertEqual(inspect.getdoc(B), 'Common base class') + self.assertIsNone(inspect.getdoc(B, inherit_class_doc=False)) + self.assertIsNone(inspect.getdoc(b)) + self.assertIsNone(inspect.getdoc(b, fallback_to_class_doc=False)) + b.__doc__ = 'Instance' + self.assertEqual(inspect.getdoc(b, fallback_to_class_doc=False), 'Instance') + @unittest.skipIf(MISSING_C_DOCSTRINGS, "test requires docstrings") def test_finddoc(self): finddoc = inspect._finddoc self.assertEqual(finddoc(int), int.__doc__) + self.assertIsNone(finddoc(int, search_in_class=False)) self.assertEqual(finddoc(int.to_bytes), int.to_bytes.__doc__) self.assertEqual(finddoc(int().to_bytes), int.to_bytes.__doc__) self.assertEqual(finddoc(int.from_bytes), int.from_bytes.__doc__) diff --git a/Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst b/Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst new file mode 100644 index 00000000000000..d0c8e2d705cc73 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst @@ -0,0 +1,2 @@ +Add parameters *inherit_class_doc* and *fallback_to_class_doc* for +:func:`inspect.getdoc`. From 9e7340cd3b5531784291088b504882cfb4d4c78c Mon Sep 17 00:00:00 2001 From: J Berg Date: Tue, 11 Nov 2025 22:09:58 +0000 Subject: [PATCH 18/58] gh-139462: Make the ProcessPoolExecutor BrokenProcessPool exception report which child process terminated (GH-139486) Report which process terminated as cause of BPE --- Doc/whatsnew/3.15.rst | 10 ++++++++++ Lib/concurrent/futures/process.py | 18 ++++++++++++++++-- .../test_process_pool.py | 15 +++++++++++++++ ...5-10-02-22-29-00.gh-issue-139462.VZXUHe.rst | 3 +++ 4 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index ecab0d03e105e6..c543b6e6c2a779 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -369,6 +369,16 @@ collections.abc :mod:`!collections.abc` module. +concurrent.futures +------------------ + +* Improved error reporting when a child process in a + :class:`concurrent.futures.ProcessPoolExecutor` terminates abruptly. + The resulting traceback will now tell you the PID and exit code of the + terminated process. + (Contributed by Jonathan Berg in :gh:`139486`.) + + dataclasses ----------- diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index a14650bf5fa47c..a42afa68efcb14 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -474,9 +474,23 @@ def _terminate_broken(self, cause): bpe = BrokenProcessPool("A process in the process pool was " "terminated abruptly while the future was " "running or pending.") + cause_str = None if cause is not None: - bpe.__cause__ = _RemoteTraceback( - f"\n'''\n{''.join(cause)}'''") + cause_str = ''.join(cause) + else: + # No cause known, so report any processes that have + # terminated with nonzero exit codes, e.g. from a + # segfault. Multiple may terminate simultaneously, + # so include all of them in the traceback. + errors = [] + for p in self.processes.values(): + if p.exitcode is not None and p.exitcode != 0: + errors.append(f"Process {p.pid} terminated abruptly " + f"with exit code {p.exitcode}") + if errors: + cause_str = "\n".join(errors) + if cause_str: + bpe.__cause__ = _RemoteTraceback(f"\n'''\n{cause_str}'''") # Mark pending tasks as failed. for work_id, work_item in self.pending_work_items.items(): diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index 9685f980119a0e..731419a48bd128 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -106,6 +106,21 @@ def test_traceback(self): self.assertIn('raise RuntimeError(123) # some comment', f1.getvalue()) + def test_traceback_when_child_process_terminates_abruptly(self): + # gh-139462 enhancement - BrokenProcessPool exceptions + # should describe which process terminated. + exit_code = 99 + with self.executor_type(max_workers=1) as executor: + future = executor.submit(os._exit, exit_code) + with self.assertRaises(BrokenProcessPool) as bpe: + future.result() + + cause = bpe.exception.__cause__ + self.assertIsInstance(cause, futures.process._RemoteTraceback) + self.assertIn( + f"terminated abruptly with exit code {exit_code}", cause.tb + ) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @hashlib_helper.requires_hashdigest('md5') def test_ressources_gced_in_workers(self): diff --git a/Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst b/Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst new file mode 100644 index 00000000000000..390a6124386151 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst @@ -0,0 +1,3 @@ +When a child process in a :class:`concurrent.futures.ProcessPoolExecutor` +terminates abruptly, the resulting traceback will now tell you the PID +and exit code of the terminated process. Contributed by Jonathan Berg. From 4359706ac8d5589fc37e2f1460a0d07a2319df15 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 12 Nov 2025 00:27:13 +0200 Subject: [PATCH 19/58] gh-120950: Fix overflow in math.log() with large int-like argument (GH-121011) Handling of arbitrary large int-like argument is now consistent with handling arbitrary large int arguments. --- Lib/test/test_math.py | 59 ++++++++++++++ ...-06-26-16-16-43.gh-issue-121011.qW54eh.rst | 2 + Modules/mathmodule.c | 80 ++++++++++++------- 3 files changed, 111 insertions(+), 30 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index ddeb8ad7cd62f3..68f41a2e62034d 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -189,6 +189,22 @@ def __init__(self, value): def __index__(self): return self.value +class IndexableFloatLike: + def __init__(self, float_value, index_value): + self.float_value = float_value + self.index_value = index_value + + def __float__(self): + if isinstance(self.float_value, BaseException): + raise self.float_value + return self.float_value + + def __index__(self): + if isinstance(self.index_value, BaseException): + raise self.index_value + return self.index_value + + class BadDescr: def __get__(self, obj, objtype=None): raise ValueError @@ -1192,13 +1208,32 @@ def testLog(self): self.ftest('log(10**40, 10**20)', math.log(10**40, 10**20), 2) self.ftest('log(10**1000)', math.log(10**1000), 2302.5850929940457) + self.ftest('log(10**2000, 10**1000)', math.log(10**2000, 10**1000), 2) + self.ftest('log(MyIndexable(32), MyIndexable(2))', + math.log(MyIndexable(32), MyIndexable(2)), 5) + self.ftest('log(MyIndexable(10**1000))', + math.log(MyIndexable(10**1000)), + 2302.5850929940457) + self.ftest('log(MyIndexable(10**2000), MyIndexable(10**1000))', + math.log(MyIndexable(10**2000), MyIndexable(10**1000)), + 2) + self.assertRaises(ValueError, math.log, 0.0) + self.assertRaises(ValueError, math.log, 0) + self.assertRaises(ValueError, math.log, MyIndexable(0)) self.assertRaises(ValueError, math.log, -1.5) + self.assertRaises(ValueError, math.log, -1) + self.assertRaises(ValueError, math.log, MyIndexable(-1)) self.assertRaises(ValueError, math.log, -10**1000) + self.assertRaises(ValueError, math.log, MyIndexable(-10**1000)) self.assertRaises(ValueError, math.log, 10, -10) self.assertRaises(ValueError, math.log, NINF) self.assertEqual(math.log(INF), INF) self.assertTrue(math.isnan(math.log(NAN))) + self.assertEqual(math.log(IndexableFloatLike(math.e, 10**1000)), 1.0) + self.assertAlmostEqual(math.log(IndexableFloatLike(OverflowError(), 10**1000)), + 2302.5850929940457) + def testLog1p(self): self.assertRaises(TypeError, math.log1p) for n in [2, 2**90, 2**300]: @@ -1214,16 +1249,28 @@ def testLog2(self): self.assertEqual(math.log2(1), 0.0) self.assertEqual(math.log2(2), 1.0) self.assertEqual(math.log2(4), 2.0) + self.assertEqual(math.log2(MyIndexable(4)), 2.0) # Large integer values self.assertEqual(math.log2(2**1023), 1023.0) self.assertEqual(math.log2(2**1024), 1024.0) self.assertEqual(math.log2(2**2000), 2000.0) + self.assertEqual(math.log2(MyIndexable(2**2000)), 2000.0) + self.assertRaises(ValueError, math.log2, 0.0) + self.assertRaises(ValueError, math.log2, 0) + self.assertRaises(ValueError, math.log2, MyIndexable(0)) self.assertRaises(ValueError, math.log2, -1.5) + self.assertRaises(ValueError, math.log2, -1) + self.assertRaises(ValueError, math.log2, MyIndexable(-1)) + self.assertRaises(ValueError, math.log2, -2**2000) + self.assertRaises(ValueError, math.log2, MyIndexable(-2**2000)) self.assertRaises(ValueError, math.log2, NINF) self.assertTrue(math.isnan(math.log2(NAN))) + self.assertEqual(math.log2(IndexableFloatLike(8.0, 2**2000)), 3.0) + self.assertEqual(math.log2(IndexableFloatLike(OverflowError(), 2**2000)), 2000.0) + @requires_IEEE_754 # log2() is not accurate enough on Mac OS X Tiger (10.4) @support.requires_mac_ver(10, 5) @@ -1239,12 +1286,24 @@ def testLog10(self): self.ftest('log10(1)', math.log10(1), 0) self.ftest('log10(10)', math.log10(10), 1) self.ftest('log10(10**1000)', math.log10(10**1000), 1000.0) + self.ftest('log10(MyIndexable(10))', math.log10(MyIndexable(10)), 1) + self.ftest('log10(MyIndexable(10**1000))', + math.log10(MyIndexable(10**1000)), 1000.0) + self.assertRaises(ValueError, math.log10, 0.0) + self.assertRaises(ValueError, math.log10, 0) + self.assertRaises(ValueError, math.log10, MyIndexable(0)) self.assertRaises(ValueError, math.log10, -1.5) + self.assertRaises(ValueError, math.log10, -1) + self.assertRaises(ValueError, math.log10, MyIndexable(-1)) self.assertRaises(ValueError, math.log10, -10**1000) + self.assertRaises(ValueError, math.log10, MyIndexable(-10**1000)) self.assertRaises(ValueError, math.log10, NINF) self.assertEqual(math.log(INF), INF) self.assertTrue(math.isnan(math.log10(NAN))) + self.assertEqual(math.log10(IndexableFloatLike(100.0, 10**1000)), 2.0) + self.assertEqual(math.log10(IndexableFloatLike(OverflowError(), 10**1000)), 1000.0) + @support.bigmemtest(2**32, memuse=0.2) def test_log_huge_integer(self, size): v = 1 << size diff --git a/Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst b/Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst new file mode 100644 index 00000000000000..aee7fe2bcb5c60 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst @@ -0,0 +1,2 @@ +:func:`math.log` now supports arbitrary large integer-like arguments in the +same way as arbitrary large integer arguments. diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 82846843cfb0b2..de1886451eda8f 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -57,6 +57,7 @@ raised for division by zero and mod by zero. #endif #include "Python.h" +#include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_bitutils.h" // _Py_bit_length() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_import.h" // _PyImport_SetModuleString() @@ -1578,43 +1579,62 @@ math_modf_impl(PyObject *module, double x) in that int is larger than PY_SSIZE_T_MAX. */ static PyObject* -loghelper(PyObject* arg, double (*func)(double)) +loghelper_int(PyObject* arg, double (*func)(double)) { /* If it is int, do it ourselves. */ - if (PyLong_Check(arg)) { - double x, result; - int64_t e; + double x, result; + int64_t e; - /* Negative or zero inputs give a ValueError. */ - if (!_PyLong_IsPositive((PyLongObject *)arg)) { - /* The input can be an arbitrary large integer, so we - don't include it's value in the error message. */ - PyErr_SetString(PyExc_ValueError, - "expected a positive input"); - return NULL; - } + /* Negative or zero inputs give a ValueError. */ + if (!_PyLong_IsPositive((PyLongObject *)arg)) { + PyErr_SetString(PyExc_ValueError, + "expected a positive input"); + return NULL; + } - x = PyLong_AsDouble(arg); - if (x == -1.0 && PyErr_Occurred()) { - if (!PyErr_ExceptionMatches(PyExc_OverflowError)) - return NULL; - /* Here the conversion to double overflowed, but it's possible - to compute the log anyway. Clear the exception and continue. */ - PyErr_Clear(); - x = _PyLong_Frexp((PyLongObject *)arg, &e); - assert(e >= 0); - assert(!PyErr_Occurred()); - /* Value is ~= x * 2**e, so the log ~= log(x) + log(2) * e. */ - result = fma(func(2.0), (double)e, func(x)); - } - else - /* Successfully converted x to a double. */ - result = func(x); - return PyFloat_FromDouble(result); + x = PyLong_AsDouble(arg); + if (x == -1.0 && PyErr_Occurred()) { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return NULL; + /* Here the conversion to double overflowed, but it's possible + to compute the log anyway. Clear the exception and continue. */ + PyErr_Clear(); + x = _PyLong_Frexp((PyLongObject *)arg, &e); + assert(!PyErr_Occurred()); + /* Value is ~= x * 2**e, so the log ~= log(x) + log(2) * e. */ + result = fma(func(2.0), (double)e, func(x)); } + else + /* Successfully converted x to a double. */ + result = func(x); + return PyFloat_FromDouble(result); +} +static PyObject* +loghelper(PyObject* arg, double (*func)(double)) +{ + /* If it is int, do it ourselves. */ + if (PyLong_Check(arg)) { + return loghelper_int(arg, func); + } /* Else let libm handle it by itself. */ - return math_1(arg, func, 0, "expected a positive input, got %s"); + PyObject *res = math_1(arg, func, 0, "expected a positive input, got %s"); + if (res == NULL && + PyErr_ExceptionMatches(PyExc_OverflowError) && + PyIndex_Check(arg)) + { + /* Here the conversion to double overflowed, but it's possible + to compute the log anyway. Clear the exception, convert to + integer and continue. */ + PyErr_Clear(); + arg = _PyNumber_Index(arg); + if (arg == NULL) { + return NULL; + } + res = loghelper_int(arg, func); + Py_DECREF(arg); + } + return res; } From f5c2a41f9a6b3be95c5be9dbae0a4a3342d356dc Mon Sep 17 00:00:00 2001 From: yihong Date: Wed, 12 Nov 2025 07:47:57 +0800 Subject: [PATCH 20/58] gh-138775: fix handle `python -m base64` stdin correct with EOF signal (GH-138776) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: handle stdin correct with EOF single. * fix: flollow the comments when pipe stdin use buffer * Apply suggestions from code review * fix: apply review comments in Lib/base64.py * fix: address comments * Reword comment and NEWS entry. --------- Signed-off-by: yihong0618 Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Peter Bierma Co-authored-by: Gregory P. Smith --- Lib/base64.py | 9 ++++++++- .../2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst diff --git a/Lib/base64.py b/Lib/base64.py index cfc57626c40ba9..f95132a4274051 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -604,7 +604,14 @@ def main(): with open(args[0], 'rb') as f: func(f, sys.stdout.buffer) else: - func(sys.stdin.buffer, sys.stdout.buffer) + if sys.stdin.isatty(): + # gh-138775: read terminal input data all at once to detect EOF + import io + data = sys.stdin.buffer.read() + buffer = io.BytesIO(data) + else: + buffer = sys.stdin.buffer + func(buffer, sys.stdout.buffer) if __name__ == '__main__': diff --git a/Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst b/Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst new file mode 100644 index 00000000000000..455c1a9925a5e1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst @@ -0,0 +1,2 @@ +Use of ``python -m`` with :mod:`base64` has been fixed to detect input from a +terminal so that it properly notices EOF. From 0d7b48a8f5de5c1c6d57e1cf7194b6fb222d92e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Wed, 12 Nov 2025 01:03:14 +0100 Subject: [PATCH 21/58] gh-137952: update `csv.Sniffer().has_header()` docs to describe the actual off-by-onish behavior (GH-137953) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * checks 21, not 20 * Say "header" instead of "first row" to disambiguate per review. --------- Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Co-authored-by: Maurycy Pawłowski-Wieroński --- Doc/library/csv.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 3ea7cd210f729d..4a033d823e6a7e 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -295,8 +295,8 @@ The :mod:`csv` module defines the following classes: - the second through n-th rows contain strings where at least one value's length differs from that of the putative header of that column. - Twenty rows after the first row are sampled; if more than half of columns + - rows meet the criteria, :const:`True` is returned. + Twenty-one rows after the header are sampled; if more than half of the + columns + rows meet the criteria, :const:`True` is returned. .. note:: From 0e88be6f55f35ab045e57f9f869b893c15dcc099 Mon Sep 17 00:00:00 2001 From: Jan-Eric Nitschke <47750513+JanEricNitschke@users.noreply.github.com> Date: Wed, 12 Nov 2025 01:32:26 +0100 Subject: [PATCH 22/58] gh-138621: Increase test coverage for csv.DictReader and csv.Sniffer (GH-138622) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Increase test coverage for csv.DictReader and csv.Sniffer Previously there were no tests for the DictReader fieldnames setter, the case where a StopIteration was encountered when trying to determine the fieldnames from the content or the case where Sniffer could not find a delimiter. * Revert whitespace change to comment * Add a test that csv.Sniffer.has_header checks up to 20 rows * Replace name and age with letter and offset Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> * Address review comment --------- Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com> --- Lib/test/test_csv.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 6be6a7ae222f02..df79840088abc3 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -918,6 +918,14 @@ def test_dict_reader_fieldnames_accepts_list(self): reader = csv.DictReader(f, fieldnames) self.assertEqual(reader.fieldnames, fieldnames) + def test_dict_reader_set_fieldnames(self): + fieldnames = ["a", "b", "c"] + f = StringIO() + reader = csv.DictReader(f) + self.assertIsNone(reader.fieldnames) + reader.fieldnames = fieldnames + self.assertEqual(reader.fieldnames, fieldnames) + def test_dict_writer_fieldnames_rejects_iter(self): fieldnames = ["a", "b", "c"] f = StringIO() @@ -933,6 +941,7 @@ def test_dict_writer_fieldnames_accepts_list(self): def test_dict_reader_fieldnames_is_optional(self): f = StringIO() reader = csv.DictReader(f, fieldnames=None) + self.assertIsNone(reader.fieldnames) def test_read_dict_fields(self): with TemporaryFile("w+", encoding="utf-8") as fileobj: @@ -1353,6 +1362,19 @@ class TestSniffer(unittest.TestCase): ghi\0jkl """ + sample15 = "\n\n\n" + sample16 = "abc\ndef\nghi" + + sample17 = ["letter,offset"] + sample17.extend(f"{chr(ord('a') + i)},{i}" for i in range(20)) + sample17.append("v,twenty_one") # 'u' was skipped + sample17 = '\n'.join(sample17) + + sample18 = ["letter,offset"] + sample18.extend(f"{chr(ord('a') + i)},{i}" for i in range(21)) + sample18.append("v,twenty_one") # 'u' was not skipped + sample18 = '\n'.join(sample18) + def test_issue43625(self): sniffer = csv.Sniffer() self.assertTrue(sniffer.has_header(self.sample12)) @@ -1374,6 +1396,11 @@ def test_has_header_regex_special_delimiter(self): self.assertIs(sniffer.has_header(self.sample8), False) self.assertIs(sniffer.has_header(self.header2 + self.sample8), True) + def test_has_header_checks_20_rows(self): + sniffer = csv.Sniffer() + self.assertFalse(sniffer.has_header(self.sample17)) + self.assertTrue(sniffer.has_header(self.sample18)) + def test_guess_quote_and_delimiter(self): sniffer = csv.Sniffer() for header in (";'123;4';", "'123;4';", ";'123;4'", "'123;4'"): @@ -1423,6 +1450,10 @@ def test_delimiters(self): self.assertEqual(dialect.quotechar, "'") dialect = sniffer.sniff(self.sample14) self.assertEqual(dialect.delimiter, '\0') + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, self.sample15) + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, self.sample16) def test_doublequote(self): sniffer = csv.Sniffer() From df6676549cd67c7b83111c6fce7c546270604aa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Cabello=20Jim=C3=A9nez?= <33024690+acabelloj@users.noreply.github.com> Date: Wed, 12 Nov 2025 01:36:43 +0100 Subject: [PATCH 23/58] gh-137928: remove redundant size validation in multiprocessing.heap (GH-137929) remove redundant size check, malloc does it --------- Co-authored-by: Gregory P. Smith --- Lib/multiprocessing/heap.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Lib/multiprocessing/heap.py b/Lib/multiprocessing/heap.py index 6217dfe12689b3..5c835648395f79 100644 --- a/Lib/multiprocessing/heap.py +++ b/Lib/multiprocessing/heap.py @@ -324,10 +324,6 @@ class BufferWrapper(object): _heap = Heap() def __init__(self, size): - if size < 0: - raise ValueError("Size {0:n} out of range".format(size)) - if sys.maxsize <= size: - raise OverflowError("Size {0:n} too large".format(size)) block = BufferWrapper._heap.malloc(size) self._state = (block, size) util.Finalize(self, BufferWrapper._heap.free, args=(block,)) From 9ce99c6c1901705238e4cb3ce81eb6f499e7b4f4 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 12 Nov 2025 00:53:21 +0000 Subject: [PATCH 24/58] GH-137618: Require Python 3.10 to Python 3.15 for PYTHON_FOR_REGEN (GH-137619) * Require Python 3.11 to Python 3.15 for PYTHON_FOR_REGEN * NEWS * keep allowing python 3.10 --------- Co-authored-by: Gregory P. Smith --- .../next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst | 2 ++ configure | 2 +- configure.ac | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst diff --git a/Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst b/Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst new file mode 100644 index 00000000000000..0b56c4c8f68566 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst @@ -0,0 +1,2 @@ +``PYTHON_FOR_REGEN`` now requires Python 3.10 to Python 3.15. +Patch by Adam Turner. diff --git a/configure b/configure index 8463b5b5e4a9d0..eeb24c1d844e86 100755 --- a/configure +++ b/configure @@ -3818,7 +3818,7 @@ fi -for ac_prog in python$PACKAGE_VERSION python3.13 python3.12 python3.11 python3.10 python3 python +for ac_prog in python$PACKAGE_VERSION python3.15 python3.14 python3.13 python3.12 python3.11 python3.10 python3 python do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 diff --git a/configure.ac b/configure.ac index df94ae25e63561..92adc44da0d6fe 100644 --- a/configure.ac +++ b/configure.ac @@ -205,7 +205,7 @@ AC_SUBST([FREEZE_MODULE_DEPS]) AC_SUBST([PYTHON_FOR_BUILD_DEPS]) AC_CHECK_PROGS([PYTHON_FOR_REGEN], - [python$PACKAGE_VERSION python3.13 python3.12 python3.11 python3.10 python3 python], + [python$PACKAGE_VERSION python3.15 python3.14 python3.13 python3.12 python3.11 python3.10 python3 python], [python3]) AC_SUBST([PYTHON_FOR_REGEN]) From fbebca289d811669fc1980e3a135325b8542a846 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 12 Nov 2025 09:59:48 +0500 Subject: [PATCH 25/58] GH-116946: eliminate the need for the GC in the `_thread.lock` and `_thread.RLock` (#141268) --- Modules/_threadmodule.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index cc8277c5783858..0e22c7bd386be1 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -41,6 +41,7 @@ typedef struct { typedef struct { PyObject_HEAD PyMutex lock; + PyObject *weakreflist; /* List of weak references */ } lockobject; #define lockobject_CAST(op) ((lockobject *)(op)) @@ -48,6 +49,7 @@ typedef struct { typedef struct { PyObject_HEAD _PyRecursiveMutex lock; + PyObject *weakreflist; /* List of weak references */ } rlockobject; #define rlockobject_CAST(op) ((rlockobject *)(op)) @@ -767,7 +769,6 @@ static PyType_Spec ThreadHandle_Type_spec = { static void lock_dealloc(PyObject *self) { - PyObject_GC_UnTrack(self); PyObject_ClearWeakRefs(self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); @@ -999,6 +1000,10 @@ lock_new_impl(PyTypeObject *type) return (PyObject *)self; } +static PyMemberDef lock_members[] = { + {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(lockobject, weakreflist), Py_READONLY}, + {NULL} +}; static PyMethodDef lock_methods[] = { _THREAD_LOCK_ACQUIRE_LOCK_METHODDEF @@ -1034,8 +1039,8 @@ static PyType_Slot lock_type_slots[] = { {Py_tp_dealloc, lock_dealloc}, {Py_tp_repr, lock_repr}, {Py_tp_doc, (void *)lock_doc}, + {Py_tp_members, lock_members}, {Py_tp_methods, lock_methods}, - {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_new, lock_new}, {0, 0} }; @@ -1043,8 +1048,7 @@ static PyType_Slot lock_type_slots[] = { static PyType_Spec lock_type_spec = { .name = "_thread.lock", .basicsize = sizeof(lockobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE), .slots = lock_type_slots, }; @@ -1059,7 +1063,6 @@ rlock_locked_impl(rlockobject *self) static void rlock_dealloc(PyObject *self) { - PyObject_GC_UnTrack(self); PyObject_ClearWeakRefs(self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); @@ -1319,6 +1322,11 @@ _thread_RLock__at_fork_reinit_impl(rlockobject *self) #endif /* HAVE_FORK */ +static PyMemberDef rlock_members[] = { + {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(rlockobject, weakreflist), Py_READONLY}, + {NULL} +}; + static PyMethodDef rlock_methods[] = { _THREAD_RLOCK_ACQUIRE_METHODDEF _THREAD_RLOCK_RELEASE_METHODDEF @@ -1339,10 +1347,10 @@ static PyMethodDef rlock_methods[] = { static PyType_Slot rlock_type_slots[] = { {Py_tp_dealloc, rlock_dealloc}, {Py_tp_repr, rlock_repr}, + {Py_tp_members, rlock_members}, {Py_tp_methods, rlock_methods}, {Py_tp_alloc, PyType_GenericAlloc}, {Py_tp_new, rlock_new}, - {Py_tp_traverse, _PyObject_VisitType}, {0, 0}, }; @@ -1350,7 +1358,7 @@ static PyType_Spec rlock_type_spec = { .name = "_thread.RLock", .basicsize = sizeof(rlockobject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), + Py_TPFLAGS_IMMUTABLETYPE), .slots = rlock_type_slots, }; From ef474cfafbdf3aa383fb1334a7ab95cef9834ced Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 12 Nov 2025 10:47:38 +0530 Subject: [PATCH 26/58] gh-103847: fix cancellation safety of `asyncio.create_subprocess_exec` (#140805) --- Lib/asyncio/base_subprocess.py | 11 +++++ Lib/test/test_asyncio/test_subprocess.py | 40 ++++++++++++++++++- ...-10-31-13-57-55.gh-issue-103847.VM7TnW.rst | 1 + 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index d40af422e614c1..321a4e5d5d18fb 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -26,6 +26,7 @@ def __init__(self, loop, protocol, args, shell, self._pending_calls = collections.deque() self._pipes = {} self._finished = False + self._pipes_connected = False if stdin == subprocess.PIPE: self._pipes[0] = None @@ -213,6 +214,7 @@ async def _connect_pipes(self, waiter): else: if waiter is not None and not waiter.cancelled(): waiter.set_result(None) + self._pipes_connected = True def _call(self, cb, *data): if self._pending_calls is not None: @@ -256,6 +258,15 @@ def _try_finish(self): assert not self._finished if self._returncode is None: return + if not self._pipes_connected: + # self._pipes_connected can be False if not all pipes were connected + # because either the process failed to start or the self._connect_pipes task + # got cancelled. In this broken state we consider all pipes disconnected and + # to avoid hanging forever in self._wait as otherwise _exit_waiters + # would never be woken up, we wake them up here. + for waiter in self._exit_waiters: + if not waiter.cancelled(): + waiter.set_result(self._returncode) if all(p is not None and p.disconnected for p in self._pipes.values()): self._finished = True diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 3a17c169c34f12..bf301740741ae7 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -11,7 +11,7 @@ from asyncio import subprocess from test.test_asyncio import utils as test_utils from test import support -from test.support import os_helper +from test.support import os_helper, warnings_helper, gc_collect if not support.has_subprocess_support: raise unittest.SkipTest("test module requires subprocess") @@ -879,6 +879,44 @@ async def main(): self.loop.run_until_complete(main()) + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_read_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_read_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, stderr=asyncio.subprocess.PIPE) + + asyncio.run(main()) + gc_collect() + + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_write_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_write_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, stdin=asyncio.subprocess.PIPE) + + asyncio.run(main()) + gc_collect() + + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_read_write_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_read_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + loop.connect_write_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec( + *PROGRAM_BLOCKED, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + asyncio.run(main()) + gc_collect() if sys.platform != 'win32': # Unix diff --git a/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst b/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst new file mode 100644 index 00000000000000..e14af7d97083d6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst @@ -0,0 +1 @@ +Fix hang when cancelling process created by :func:`asyncio.create_subprocess_exec` or :func:`asyncio.create_subprocess_shell`. Patch by Kumar Aditya. From f1b7961ccfa050e9c80622fff1b3cdada46f9aab Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 12 Nov 2025 12:51:43 +0530 Subject: [PATCH 27/58] GH-116946: revert eliminate the need for the GC in the `_thread.lock` and `_thread.RLock` (#141448) Revert "GH-116946: eliminate the need for the GC in the `_thread.lock` and `_thread.RLock` (#141268)" This reverts commit fbebca289d811669fc1980e3a135325b8542a846. --- Modules/_threadmodule.c | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 0e22c7bd386be1..cc8277c5783858 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -41,7 +41,6 @@ typedef struct { typedef struct { PyObject_HEAD PyMutex lock; - PyObject *weakreflist; /* List of weak references */ } lockobject; #define lockobject_CAST(op) ((lockobject *)(op)) @@ -49,7 +48,6 @@ typedef struct { typedef struct { PyObject_HEAD _PyRecursiveMutex lock; - PyObject *weakreflist; /* List of weak references */ } rlockobject; #define rlockobject_CAST(op) ((rlockobject *)(op)) @@ -769,6 +767,7 @@ static PyType_Spec ThreadHandle_Type_spec = { static void lock_dealloc(PyObject *self) { + PyObject_GC_UnTrack(self); PyObject_ClearWeakRefs(self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); @@ -1000,10 +999,6 @@ lock_new_impl(PyTypeObject *type) return (PyObject *)self; } -static PyMemberDef lock_members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(lockobject, weakreflist), Py_READONLY}, - {NULL} -}; static PyMethodDef lock_methods[] = { _THREAD_LOCK_ACQUIRE_LOCK_METHODDEF @@ -1039,8 +1034,8 @@ static PyType_Slot lock_type_slots[] = { {Py_tp_dealloc, lock_dealloc}, {Py_tp_repr, lock_repr}, {Py_tp_doc, (void *)lock_doc}, - {Py_tp_members, lock_members}, {Py_tp_methods, lock_methods}, + {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_new, lock_new}, {0, 0} }; @@ -1048,7 +1043,8 @@ static PyType_Slot lock_type_slots[] = { static PyType_Spec lock_type_spec = { .name = "_thread.lock", .basicsize = sizeof(lockobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), .slots = lock_type_slots, }; @@ -1063,6 +1059,7 @@ rlock_locked_impl(rlockobject *self) static void rlock_dealloc(PyObject *self) { + PyObject_GC_UnTrack(self); PyObject_ClearWeakRefs(self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); @@ -1322,11 +1319,6 @@ _thread_RLock__at_fork_reinit_impl(rlockobject *self) #endif /* HAVE_FORK */ -static PyMemberDef rlock_members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(rlockobject, weakreflist), Py_READONLY}, - {NULL} -}; - static PyMethodDef rlock_methods[] = { _THREAD_RLOCK_ACQUIRE_METHODDEF _THREAD_RLOCK_RELEASE_METHODDEF @@ -1347,10 +1339,10 @@ static PyMethodDef rlock_methods[] = { static PyType_Slot rlock_type_slots[] = { {Py_tp_dealloc, rlock_dealloc}, {Py_tp_repr, rlock_repr}, - {Py_tp_members, rlock_members}, {Py_tp_methods, rlock_methods}, {Py_tp_alloc, PyType_GenericAlloc}, {Py_tp_new, rlock_new}, + {Py_tp_traverse, _PyObject_VisitType}, {0, 0}, }; @@ -1358,7 +1350,7 @@ static PyType_Spec rlock_type_spec = { .name = "_thread.RLock", .basicsize = sizeof(rlockobject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), .slots = rlock_type_slots, }; From 35908265b09ac39b67116bfdfe8a053be09e6d8f Mon Sep 17 00:00:00 2001 From: Mark Byrne <31762852+mbyrnepr2@users.noreply.github.com> Date: Wed, 12 Nov 2025 09:20:55 +0100 Subject: [PATCH 28/58] gh-75593: Add support of bytes and path-like paths in wave.open() (GH-140951) --- Doc/library/wave.rst | 9 ++++++-- Lib/test/test_wave.py | 22 +++++++++++++++++++ Lib/wave.py | 5 +++-- ...5-11-04-12-16-13.gh-issue-75593.EFVhKR.rst | 1 + 4 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst diff --git a/Doc/library/wave.rst b/Doc/library/wave.rst index a3f5bfd5e2f99c..7ff2c97992c4e3 100644 --- a/Doc/library/wave.rst +++ b/Doc/library/wave.rst @@ -25,8 +25,9 @@ The :mod:`wave` module defines the following function and exception: .. function:: open(file, mode=None) - If *file* is a string, open the file by that name, otherwise treat it as a - file-like object. *mode* can be: + If *file* is a string, a :term:`path-like object` or a + :term:`bytes-like object` open the file by that name, otherwise treat it as + a file-like object. *mode* can be: ``'rb'`` Read only mode. @@ -52,6 +53,10 @@ The :mod:`wave` module defines the following function and exception: .. versionchanged:: 3.4 Added support for unseekable files. + .. versionchanged:: 3.15 + Added support for :term:`path-like objects ` + and :term:`bytes-like objects `. + .. exception:: Error An error raised when something is impossible because it violates the WAV diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py index 226b1aa84bd73c..4c21f16553775c 100644 --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -1,9 +1,11 @@ import unittest from test import audiotests from test import support +from test.support.os_helper import FakePath import io import os import struct +import tempfile import sys import wave @@ -206,5 +208,25 @@ def test_open_in_write_raises(self): self.assertIsNone(cm.unraisable) +class WaveOpen(unittest.TestCase): + def test_open_pathlike(self): + """It is possible to use `wave.read` and `wave.write` with a path-like object""" + with tempfile.NamedTemporaryFile(delete_on_close=False) as fp: + cases = ( + FakePath(fp.name), + FakePath(os.fsencode(fp.name)), + os.fsencode(fp.name), + ) + for fake_path in cases: + with self.subTest(fake_path): + with wave.open(fake_path, 'wb') as f: + f.setnchannels(1) + f.setsampwidth(2) + f.setframerate(44100) + + with wave.open(fake_path, 'rb') as f: + pass + + if __name__ == '__main__': unittest.main() diff --git a/Lib/wave.py b/Lib/wave.py index 5af745e2217ec3..056bd6aab7ffa3 100644 --- a/Lib/wave.py +++ b/Lib/wave.py @@ -69,6 +69,7 @@ from collections import namedtuple import builtins +import os import struct import sys @@ -274,7 +275,7 @@ def initfp(self, file): def __init__(self, f): self._i_opened_the_file = None - if isinstance(f, str): + if isinstance(f, (bytes, str, os.PathLike)): f = builtins.open(f, 'rb') self._i_opened_the_file = f # else, assume it is an open file object already @@ -431,7 +432,7 @@ class Wave_write: def __init__(self, f): self._i_opened_the_file = None - if isinstance(f, str): + if isinstance(f, (bytes, str, os.PathLike)): f = builtins.open(f, 'wb') self._i_opened_the_file = f try: diff --git a/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst b/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst new file mode 100644 index 00000000000000..9a31af9c110454 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst @@ -0,0 +1 @@ +Add support of :term:`path-like objects ` and :term:`bytes-like objects ` in :func:`wave.open`. From 909f76dab91f028edd2ae7bd589d3975996de9e1 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 12 Nov 2025 09:42:56 +0100 Subject: [PATCH 29/58] gh-141376: Rename _AsyncioDebug to _Py_AsyncioDebug (GH-141391) --- Modules/_asynciomodule.c | 4 ++-- Tools/c-analyzer/cpython/ignored.tsv | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 1f58b1fb3506c6..9b2b7011244d77 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -119,7 +119,7 @@ typedef struct _Py_AsyncioModuleDebugOffsets { } asyncio_thread_state; } Py_AsyncioModuleDebugOffsets; -GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets _AsyncioDebug) +GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets _Py_AsyncioDebug) = {.asyncio_task_object = { .size = sizeof(TaskObj), .task_name = offsetof(TaskObj, task_name), @@ -4338,7 +4338,7 @@ module_init(asyncio_state *state) goto fail; } - state->debug_offsets = &_AsyncioDebug; + state->debug_offsets = &_Py_AsyncioDebug; Py_DECREF(module); return 0; diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 8b73189fb07dc5..11a3cd794ff4d7 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -56,7 +56,7 @@ Python/pyhash.c - _Py_HashSecret - Python/parking_lot.c - buckets - ## data needed for introspecting asyncio state from debuggers and profilers -Modules/_asynciomodule.c - _AsyncioDebug - +Modules/_asynciomodule.c - _Py_AsyncioDebug - ################################## From 6f988b08d122e44848e89c04ad1e10c25d072cc7 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 12 Nov 2025 01:37:48 -0800 Subject: [PATCH 30/58] gh-85524: Raise "UnsupportedOperation" on FileIO.readall (#141214) io.UnsupportedOperation is a subclass of OSError and recommended by io.IOBase for this case; matches other read methods on io.FileIO. --- Lib/test/test_io/test_general.py | 1 + .../2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst | 3 +++ Modules/_io/clinic/fileio.c.h | 14 +++++++++----- Modules/_io/fileio.c | 13 ++++++++++--- 4 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst diff --git a/Lib/test/test_io/test_general.py b/Lib/test/test_io/test_general.py index a1cdd6876c2892..f0677b01ea5ce1 100644 --- a/Lib/test/test_io/test_general.py +++ b/Lib/test/test_io/test_general.py @@ -125,6 +125,7 @@ def test_invalid_operations(self): self.assertRaises(exc, fp.readline) with self.open(os_helper.TESTFN, "wb", buffering=0) as fp: self.assertRaises(exc, fp.read) + self.assertRaises(exc, fp.readall) self.assertRaises(exc, fp.readline) with self.open(os_helper.TESTFN, "rb", buffering=0) as fp: self.assertRaises(exc, fp.write, b"blah") diff --git a/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst b/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst new file mode 100644 index 00000000000000..3e4fd1a5897b04 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst @@ -0,0 +1,3 @@ +Update ``io.FileIO.readall``, an implementation of :meth:`io.RawIOBase.readall`, +to follow :class:`io.IOBase` guidelines and raise :exc:`io.UnsupportedOperation` +when a file is in "w" mode rather than :exc:`OSError` diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index 04870b1c890361..96c31ce8d6f415 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -277,15 +277,19 @@ PyDoc_STRVAR(_io_FileIO_readall__doc__, "data is available (EAGAIN is returned before bytes are read) returns None."); #define _IO_FILEIO_READALL_METHODDEF \ - {"readall", (PyCFunction)_io_FileIO_readall, METH_NOARGS, _io_FileIO_readall__doc__}, + {"readall", _PyCFunction_CAST(_io_FileIO_readall), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_readall__doc__}, static PyObject * -_io_FileIO_readall_impl(fileio *self); +_io_FileIO_readall_impl(fileio *self, PyTypeObject *cls); static PyObject * -_io_FileIO_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) +_io_FileIO_readall(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _io_FileIO_readall_impl((fileio *)self); + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "readall() takes no arguments"); + return NULL; + } + return _io_FileIO_readall_impl((fileio *)self, cls); } PyDoc_STRVAR(_io_FileIO_read__doc__, @@ -543,4 +547,4 @@ _io_FileIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=1902fac9e39358aa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2e48f3df2f189170 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 2544ff4ea91ec8..5d7741fdd830a5 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -728,6 +728,9 @@ new_buffersize(fileio *self, size_t currentsize) @permit_long_docstring_body _io.FileIO.readall + cls: defining_class + / + Read all data from the file, returned as bytes. Reads until either there is an error or read() returns size 0 (indicates EOF). @@ -738,8 +741,8 @@ data is available (EAGAIN is returned before bytes are read) returns None. [clinic start generated code]*/ static PyObject * -_io_FileIO_readall_impl(fileio *self) -/*[clinic end generated code: output=faa0292b213b4022 input=10d8b2ec403302dc]*/ +_io_FileIO_readall_impl(fileio *self, PyTypeObject *cls) +/*[clinic end generated code: output=d546737ec895c462 input=cecda40bf9961299]*/ { Py_off_t pos, end; PyBytesWriter *writer; @@ -750,6 +753,10 @@ _io_FileIO_readall_impl(fileio *self) if (self->fd < 0) { return err_closed(); } + if (!self->readable) { + _PyIO_State *state = get_io_state_by_cls(cls); + return err_mode(state, "reading"); + } if (self->stat_atopen != NULL && self->stat_atopen->st_size < _PY_READ_MAX) { end = (Py_off_t)self->stat_atopen->st_size; @@ -873,7 +880,7 @@ _io_FileIO_read_impl(fileio *self, PyTypeObject *cls, Py_ssize_t size) } if (size < 0) - return _io_FileIO_readall_impl(self); + return _io_FileIO_readall_impl(self, cls); if (size > _PY_READ_MAX) { size = _PY_READ_MAX; From 20f53df07d42c495a08c73a3d54b8dd9098a62f0 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 12 Nov 2025 12:50:44 +0300 Subject: [PATCH 31/58] gh-141370: document undefined behavior of Py_ABS() (GH-141439) --- Doc/c-api/intro.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 6e1a9dcb35543b..c76cc2f70ecccf 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -121,6 +121,10 @@ complete listing. Return the absolute value of ``x``. + If the result cannot be represented (for example, if ``x`` has + :c:macro:`!INT_MIN` value for :c:expr:`int` type), the behavior is + undefined. + .. versionadded:: 3.3 .. c:macro:: Py_ALWAYS_INLINE From 7d54374f9c7d91e0ef90c4ad84baf10073cf1d8a Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 12 Nov 2025 01:57:05 -0800 Subject: [PATCH 32/58] gh-141311: Avoid assertion in BytesIO.readinto() (GH-141333) Fix error in assertion which causes failure if pos is equal to PY_SSIZE_T_MAX. Fix undefined behavior in read() and readinto() if pos is larger that the size of the underlying buffer. --- Lib/test/test_io/test_memoryio.py | 14 ++++++++++++++ ...025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst | 2 ++ Modules/_io/bytesio.c | 16 +++++++++++++--- 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst diff --git a/Lib/test/test_io/test_memoryio.py b/Lib/test/test_io/test_memoryio.py index 63998a86c45b53..bb023735e21398 100644 --- a/Lib/test/test_io/test_memoryio.py +++ b/Lib/test/test_io/test_memoryio.py @@ -54,6 +54,12 @@ def testSeek(self): self.assertEqual(buf[3:], bytesIo.read()) self.assertRaises(TypeError, bytesIo.seek, 0.0) + self.assertEqual(sys.maxsize, bytesIo.seek(sys.maxsize)) + self.assertEqual(self.EOF, bytesIo.read(4)) + + self.assertEqual(sys.maxsize - 2, bytesIo.seek(sys.maxsize - 2)) + self.assertEqual(self.EOF, bytesIo.read(4)) + def testTell(self): buf = self.buftype("1234567890") bytesIo = self.ioclass(buf) @@ -552,6 +558,14 @@ def test_relative_seek(self): memio.seek(1, 1) self.assertEqual(memio.read(), buf[1:]) + def test_issue141311(self): + memio = self.ioclass() + # Seek allows PY_SSIZE_T_MAX, read should handle that. + # Past end of buffer read should always return 0 (EOF). + self.assertEqual(sys.maxsize, memio.seek(sys.maxsize)) + buf = bytearray(2) + self.assertEqual(0, memio.readinto(buf)) + def test_unicode(self): memio = self.ioclass() diff --git a/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst b/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst new file mode 100644 index 00000000000000..bb425ce5df309d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst @@ -0,0 +1,2 @@ +Fix assertion failure in :func:`!io.BytesIO.readinto` and undefined behavior +arising when read position is above capcity in :class:`io.BytesIO`. diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index d6bfb93177c9ee..96611823ab6b45 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -436,6 +436,13 @@ read_bytes_lock_held(bytesio *self, Py_ssize_t size) return Py_NewRef(self->buf); } + /* gh-141311: Avoid undefined behavior when self->pos (limit PY_SSIZE_T_MAX) + is beyond the size of self->buf. Assert above validates size is always in + bounds. When self->pos is out of bounds calling code sets size to 0. */ + if (size == 0) { + return PyBytes_FromStringAndSize(NULL, 0); + } + output = PyBytes_AS_STRING(self->buf) + self->pos; self->pos += size; return PyBytes_FromStringAndSize(output, size); @@ -609,11 +616,14 @@ _io_BytesIO_readinto_impl(bytesio *self, Py_buffer *buffer) n = self->string_size - self->pos; if (len > n) { len = n; - if (len < 0) - len = 0; + if (len < 0) { + /* gh-141311: Avoid undefined behavior when self->pos (limit + PY_SSIZE_T_MAX) points beyond the size of self->buf. */ + return PyLong_FromSsize_t(0); + } } - assert(self->pos + len < PY_SSIZE_T_MAX); + assert(self->pos + len <= PY_SSIZE_T_MAX); assert(len >= 0); memcpy(buffer->buf, PyBytes_AS_STRING(self->buf) + self->pos, len); self->pos += len; From 23d85a2a3fb029172ea15c6e596f64f8c2868ed3 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 12 Nov 2025 13:06:29 +0300 Subject: [PATCH 33/58] gh-141042: fix sNaN's packing for mixed floating-point formats (#141107) --- Lib/test/test_capi/test_float.py | 54 +++++++++++++++---- ...-11-06-06-28-14.gh-issue-141042.brOioJ.rst | 3 ++ Objects/floatobject.c | 16 ++++-- 3 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst diff --git a/Lib/test/test_capi/test_float.py b/Lib/test/test_capi/test_float.py index 983b991b4f163d..df7017e6436a69 100644 --- a/Lib/test/test_capi/test_float.py +++ b/Lib/test/test_capi/test_float.py @@ -29,6 +29,23 @@ NAN = float("nan") +def make_nan(size, sign, quiet, payload=None): + if size == 8: + payload_mask = 0x7ffffffffffff + i = (sign << 63) + (0x7ff << 52) + (quiet << 51) + elif size == 4: + payload_mask = 0x3fffff + i = (sign << 31) + (0xff << 23) + (quiet << 22) + elif size == 2: + payload_mask = 0x1ff + i = (sign << 15) + (0x1f << 10) + (quiet << 9) + else: + raise ValueError("size must be either 2, 4, or 8") + if payload is None: + payload = random.randint(not quiet, payload_mask) + return i + payload + + class CAPIFloatTest(unittest.TestCase): def test_check(self): # Test PyFloat_Check() @@ -202,16 +219,7 @@ def test_pack_unpack_roundtrip_for_nans(self): # HP PA RISC uses 0 for quiet, see: # https://en.wikipedia.org/wiki/NaN#Encoding signaling = 1 - quiet = int(not signaling) - if size == 8: - payload = random.randint(signaling, 0x7ffffffffffff) - i = (sign << 63) + (0x7ff << 52) + (quiet << 51) + payload - elif size == 4: - payload = random.randint(signaling, 0x3fffff) - i = (sign << 31) + (0xff << 23) + (quiet << 22) + payload - elif size == 2: - payload = random.randint(signaling, 0x1ff) - i = (sign << 15) + (0x1f << 10) + (quiet << 9) + payload + i = make_nan(size, sign, not signaling) data = bytes.fromhex(f'{i:x}') for endian in (BIG_ENDIAN, LITTLE_ENDIAN): with self.subTest(data=data, size=size, endian=endian): @@ -221,6 +229,32 @@ def test_pack_unpack_roundtrip_for_nans(self): self.assertTrue(math.isnan(value)) self.assertEqual(data1, data2) + @unittest.skipUnless(HAVE_IEEE_754, "requires IEEE 754") + @unittest.skipUnless(sys.maxsize != 2147483647, "requires 64-bit mode") + def test_pack_unpack_nans_for_different_formats(self): + pack = _testcapi.float_pack + unpack = _testcapi.float_unpack + + for endian in (BIG_ENDIAN, LITTLE_ENDIAN): + with self.subTest(endian=endian): + byteorder = "big" if endian == BIG_ENDIAN else "little" + + # Convert sNaN to qNaN, if payload got truncated + data = make_nan(8, 0, False, 0x80001).to_bytes(8, byteorder) + snan_low = unpack(data, endian) + qnan4 = make_nan(4, 0, True, 0).to_bytes(4, byteorder) + qnan2 = make_nan(2, 0, True, 0).to_bytes(2, byteorder) + self.assertEqual(pack(4, snan_low, endian), qnan4) + self.assertEqual(pack(2, snan_low, endian), qnan2) + + # Preserve NaN type, if payload not truncated + data = make_nan(8, 0, False, 0x80000000001).to_bytes(8, byteorder) + snan_high = unpack(data, endian) + snan4 = make_nan(4, 0, False, 16384).to_bytes(4, byteorder) + snan2 = make_nan(2, 0, False, 2).to_bytes(2, byteorder) + self.assertEqual(pack(4, snan_high, endian), snan4) + self.assertEqual(pack(2, snan_high, endian), snan2) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst b/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst new file mode 100644 index 00000000000000..22a1aa1f405318 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst @@ -0,0 +1,3 @@ +Make qNaN in :c:func:`PyFloat_Pack2` and :c:func:`PyFloat_Pack4`, if while +conversion to a narrower precision floating-point format --- the remaining +after truncation payload will be zero. Patch by Sergey B Kirpichev. diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 1fefb12803ec19..ef613efe4e7f44 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -2030,6 +2030,10 @@ PyFloat_Pack2(double x, char *data, int le) memcpy(&v, &x, sizeof(v)); v &= 0xffc0000000000ULL; bits = (unsigned short)(v >> 42); /* NaN's type & payload */ + /* set qNaN if no payload */ + if (!bits) { + bits |= (1<<9); + } } else { sign = (x < 0.0); @@ -2202,16 +2206,16 @@ PyFloat_Pack4(double x, char *data, int le) if ((v & (1ULL << 51)) == 0) { uint32_t u32; memcpy(&u32, &y, 4); - u32 &= ~(1 << 22); /* make sNaN */ + /* if have payload, make sNaN */ + if (u32 & 0x3fffff) { + u32 &= ~(1 << 22); + } memcpy(&y, &u32, 4); } #else uint32_t u32; memcpy(&u32, &y, 4); - if ((v & (1ULL << 51)) == 0) { - u32 &= ~(1 << 22); - } /* Workaround RISC-V: "If a NaN value is converted to a * different floating-point type, the result is the * canonical NaN of the new type". The canonical NaN here @@ -2222,6 +2226,10 @@ PyFloat_Pack4(double x, char *data, int le) /* add payload */ u32 -= (u32 & 0x3fffff); u32 += (uint32_t)((v & 0x7ffffffffffffULL) >> 29); + /* if have payload, make sNaN */ + if ((v & (1ULL << 51)) == 0 && (u32 & 0x3fffff)) { + u32 &= ~(1 << 22); + } memcpy(&y, &u32, 4); #endif From 70748bdbea872a84dd8eadad9b48c73e218d2e1f Mon Sep 17 00:00:00 2001 From: Jacob Austin Lincoln <99031153+lincolnj1@users.noreply.github.com> Date: Wed, 12 Nov 2025 02:07:21 -0800 Subject: [PATCH 34/58] gh-131116: Fix inspect.getdoc() to work with cached_property objects (GH-131165) --- Doc/library/inspect.rst | 3 ++ Lib/inspect.py | 6 +++ Lib/test/test_inspect/inspect_fodder3.py | 39 +++++++++++++++++++ Lib/test/test_inspect/test_inspect.py | 20 ++++++++++ ...-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst | 2 + 5 files changed, 70 insertions(+) create mode 100644 Lib/test/test_inspect/inspect_fodder3.py create mode 100644 Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index aff53b78c4a774..13a352cbdb2cdc 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -639,6 +639,9 @@ Retrieving source code .. versionchanged:: next Added parameters *inherit_class_doc* and *fallback_to_class_doc*. + Documentation strings on :class:`~functools.cached_property` + objects are now inherited if not overriden. + .. function:: getcomments(object) diff --git a/Lib/inspect.py b/Lib/inspect.py index bb17848b444b67..8e7511b3af015f 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -747,6 +747,12 @@ def _finddoc(obj, *, search_in_class=True): cls = _findclass(obj.fget) if cls is None or getattr(cls, name) is not obj: return None + # Should be tested before ismethoddescriptor() + elif isinstance(obj, functools.cached_property): + name = obj.attrname + cls = _findclass(obj.func) + if cls is None or getattr(cls, name) is not obj: + return None elif ismethoddescriptor(obj) or isdatadescriptor(obj): name = obj.__name__ cls = obj.__objclass__ diff --git a/Lib/test/test_inspect/inspect_fodder3.py b/Lib/test/test_inspect/inspect_fodder3.py new file mode 100644 index 00000000000000..ea2481edf938c2 --- /dev/null +++ b/Lib/test/test_inspect/inspect_fodder3.py @@ -0,0 +1,39 @@ +from functools import cached_property + +# docstring in parent, inherited in child +class ParentInheritDoc: + @cached_property + def foo(self): + """docstring for foo defined in parent""" + +class ChildInheritDoc(ParentInheritDoc): + pass + +class ChildInheritDefineDoc(ParentInheritDoc): + @cached_property + def foo(self): + pass + +# Redefine foo as something other than cached_property +class ChildPropertyFoo(ParentInheritDoc): + @property + def foo(self): + """docstring for the property foo""" + +class ChildMethodFoo(ParentInheritDoc): + def foo(self): + """docstring for the method foo""" + +# docstring in child but not parent +class ParentNoDoc: + @cached_property + def foo(self): + pass + +class ChildNoDoc(ParentNoDoc): + pass + +class ChildDefineDoc(ParentNoDoc): + @cached_property + def foo(self): + """docstring for foo defined in child""" diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 24fd4a2fa626d4..dd3b7d9c5b4b5b 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -46,6 +46,7 @@ from test.test_inspect import inspect_fodder as mod from test.test_inspect import inspect_fodder2 as mod2 +from test.test_inspect import inspect_fodder3 as mod3 from test.test_inspect import inspect_stringized_annotations from test.test_inspect import inspect_deferred_annotations @@ -714,6 +715,25 @@ class B(A): b.__doc__ = 'Instance' self.assertEqual(inspect.getdoc(b, fallback_to_class_doc=False), 'Instance') + def test_getdoc_inherited_cached_property(self): + doc = inspect.getdoc(mod3.ParentInheritDoc.foo) + self.assertEqual(doc, 'docstring for foo defined in parent') + self.assertEqual(inspect.getdoc(mod3.ChildInheritDoc.foo), doc) + self.assertEqual(inspect.getdoc(mod3.ChildInheritDefineDoc.foo), doc) + + def test_getdoc_redefine_cached_property_as_other(self): + self.assertEqual(inspect.getdoc(mod3.ChildPropertyFoo.foo), + 'docstring for the property foo') + self.assertEqual(inspect.getdoc(mod3.ChildMethodFoo.foo), + 'docstring for the method foo') + + def test_getdoc_define_cached_property(self): + self.assertEqual(inspect.getdoc(mod3.ChildDefineDoc.foo), + 'docstring for foo defined in child') + + def test_getdoc_nodoc_inherited(self): + self.assertIsNone(inspect.getdoc(mod3.ChildNoDoc.foo)) + @unittest.skipIf(MISSING_C_DOCSTRINGS, "test requires docstrings") def test_finddoc(self): finddoc = inspect._finddoc diff --git a/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst b/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst new file mode 100644 index 00000000000000..f5e60ab6e8c4cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst @@ -0,0 +1,2 @@ +:func:`inspect.getdoc` now correctly returns an inherited docstring on +:class:`~functools.cached_property` objects if none is given in a subclass. From c6f3dd6a506a9bb1808c070e5ef5cf345a3bedc8 Mon Sep 17 00:00:00 2001 From: Rani Pinchuk <33353578+rani-pinchuk@users.noreply.github.com> Date: Wed, 12 Nov 2025 13:35:01 +0100 Subject: [PATCH 35/58] gh-98896: resource_tracker: use json&base64 to allow arbitrary shared memory names (GH-138473) --- Lib/multiprocessing/resource_tracker.py | 60 ++++++++++++++++--- Lib/test/_test_multiprocessing.py | 43 +++++++++++++ ...5-09-03-20-18-39.gh-issue-98896.tjez89.rst | 2 + 3 files changed, 97 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 38fcaed48fa9fb..b0f9099f4a59f3 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -15,6 +15,7 @@ # this resource tracker process, "killall python" would probably leave unlinked # resources. +import base64 import os import signal import sys @@ -22,6 +23,8 @@ import warnings from collections import deque +import json + from . import spawn from . import util @@ -196,6 +199,17 @@ def _launch(self): finally: os.close(r) + def _make_probe_message(self): + """Return a JSON-encoded probe message.""" + return ( + json.dumps( + {"cmd": "PROBE", "rtype": "noop"}, + ensure_ascii=True, + separators=(",", ":"), + ) + + "\n" + ).encode("ascii") + def _ensure_running_and_write(self, msg=None): with self._lock: if self._lock._recursion_count() > 1: @@ -207,7 +221,7 @@ def _ensure_running_and_write(self, msg=None): if self._fd is not None: # resource tracker was launched before, is it still running? if msg is None: - to_send = b'PROBE:0:noop\n' + to_send = self._make_probe_message() else: to_send = msg try: @@ -234,7 +248,7 @@ def _check_alive(self): try: # We cannot use send here as it calls ensure_running, creating # a cycle. - os.write(self._fd, b'PROBE:0:noop\n') + os.write(self._fd, self._make_probe_message()) except OSError: return False else: @@ -253,11 +267,25 @@ def _write(self, msg): assert nbytes == len(msg), f"{nbytes=} != {len(msg)=}" def _send(self, cmd, name, rtype): - msg = f"{cmd}:{name}:{rtype}\n".encode("ascii") - if len(msg) > 512: - # posix guarantees that writes to a pipe of less than PIPE_BUF - # bytes are atomic, and that PIPE_BUF >= 512 - raise ValueError('msg too long') + # POSIX guarantees that writes to a pipe of less than PIPE_BUF (512 on Linux) + # bytes are atomic. Therefore, we want the message to be shorter than 512 bytes. + # POSIX shm_open() and sem_open() require the name, including its leading slash, + # to be at most NAME_MAX bytes (255 on Linux) + # With json.dump(..., ensure_ascii=True) every non-ASCII byte becomes a 6-char + # escape like \uDC80. + # As we want the overall message to be kept atomic and therefore smaller than 512, + # we encode encode the raw name bytes with URL-safe Base64 - so a 255 long name + # will not exceed 340 bytes. + b = name.encode('utf-8', 'surrogateescape') + if len(b) > 255: + raise ValueError('shared memory name too long (max 255 bytes)') + b64 = base64.urlsafe_b64encode(b).decode('ascii') + + payload = {"cmd": cmd, "rtype": rtype, "base64_name": b64} + msg = (json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + "\n").encode("ascii") + + # The entire JSON message is guaranteed < PIPE_BUF (512 bytes) by construction. + assert len(msg) <= 512, f"internal error: message too long ({len(msg)} bytes)" self._ensure_running_and_write(msg) @@ -290,7 +318,23 @@ def main(fd): with open(fd, 'rb') as f: for line in f: try: - cmd, name, rtype = line.strip().decode('ascii').split(':') + try: + obj = json.loads(line.decode('ascii')) + except Exception as e: + raise ValueError("malformed resource_tracker message: %r" % (line,)) from e + + cmd = obj["cmd"] + rtype = obj["rtype"] + b64 = obj.get("base64_name", "") + + if not isinstance(cmd, str) or not isinstance(rtype, str) or not isinstance(b64, str): + raise ValueError("malformed resource_tracker fields: %r" % (obj,)) + + try: + name = base64.urlsafe_b64decode(b64).decode('utf-8', 'surrogateescape') + except ValueError as e: + raise ValueError("malformed resource_tracker base64_name: %r" % (b64,)) from e + cleanup_func = _CLEANUP_FUNCS.get(rtype, None) if cleanup_func is None: raise ValueError( diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 850744e47d0e0b..0f9c5c222250ae 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -7364,3 +7364,46 @@ def test_forkpty(self): res = assert_python_failure("-c", code, PYTHONWARNINGS='error') self.assertIn(b'DeprecationWarning', res.err) self.assertIn(b'is multi-threaded, use of forkpty() may lead to deadlocks in the child', res.err) + +@unittest.skipUnless(HAS_SHMEM, "requires multiprocessing.shared_memory") +class TestSharedMemoryNames(unittest.TestCase): + def test_that_shared_memory_name_with_colons_has_no_resource_tracker_errors(self): + # Test script that creates and cleans up shared memory with colon in name + test_script = textwrap.dedent(""" + import sys + from multiprocessing import shared_memory + import time + + # Test various patterns of colons in names + test_names = [ + "a:b", + "a:b:c", + "test:name:with:many:colons", + ":starts:with:colon", + "ends:with:colon:", + "::double::colons::", + "name\\nwithnewline", + "name-with-trailing-newline\\n", + "\\nname-starts-with-newline", + "colons:and\\nnewlines:mix", + "multi\\nline\\nname", + ] + + for name in test_names: + try: + shm = shared_memory.SharedMemory(create=True, size=100, name=name) + shm.buf[:5] = b'hello' # Write something to the shared memory + shm.close() + shm.unlink() + + except Exception as e: + print(f"Error with name '{name}': {e}", file=sys.stderr) + sys.exit(1) + + print("SUCCESS") + """) + + rc, out, err = assert_python_ok("-c", test_script) + self.assertIn(b"SUCCESS", out) + self.assertNotIn(b"traceback", err.lower(), err) + self.assertNotIn(b"resource_tracker.py", err, err) diff --git a/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst b/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst new file mode 100644 index 00000000000000..6831499c0afb43 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst @@ -0,0 +1,2 @@ +Fix a failure in multiprocessing resource_tracker when SharedMemory names contain colons. +Patch by Rani Pinchuk. From e2026731f5680022bd016b8b5ca5841c82e9574c Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 12 Nov 2025 15:44:49 +0300 Subject: [PATCH 36/58] gh-141004: soft-deprecate Py_INFINITY macro (#141033) Co-authored-by: Victor Stinner --- Doc/c-api/conversion.rst | 2 +- Doc/c-api/float.rst | 7 +++++-- Doc/whatsnew/3.14.rst | 2 +- Doc/whatsnew/3.15.rst | 4 ++++ Include/floatobject.h | 16 +++++++-------- Include/internal/pycore_pymath.h | 6 +++--- Include/pymath.h | 3 ++- ...-11-05-04-38-16.gh-issue-141004.rJL43P.rst | 1 + Modules/cmathmodule.c | 6 +++--- Modules/mathmodule.c | 20 +++++++++---------- Objects/complexobject.c | 8 ++++---- Objects/floatobject.c | 2 +- Python/pystrtod.c | 4 ++-- 13 files changed, 45 insertions(+), 36 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index 533e5460da8952..a18bbf4e0e37d7 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -105,7 +105,7 @@ The following functions provide locale-independent string to number conversions. If ``s`` represents a value that is too large to store in a float (for example, ``"1e500"`` is such a string on many platforms) then - if ``overflow_exception`` is ``NULL`` return ``Py_INFINITY`` (with + if ``overflow_exception`` is ``NULL`` return :c:macro:`!INFINITY` (with an appropriate sign) and don't set any exception. Otherwise, ``overflow_exception`` must point to a Python exception object; raise that exception and return ``-1.0``. In both cases, set diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index eae4792af7d299..b6020533a2b9d9 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -83,8 +83,11 @@ Floating-Point Objects This macro expands a to constant expression of type :c:expr:`double`, that represents the positive infinity. - On most platforms, this is equivalent to the :c:macro:`!INFINITY` macro from - the C11 standard ```` header. + It is equivalent to the :c:macro:`!INFINITY` macro from the C11 standard + ```` header. + + .. deprecated:: 3.15 + The macro is soft deprecated. .. c:macro:: Py_NAN diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 1a2fbda0c4ce81..9459b73bcb502f 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -3045,7 +3045,7 @@ Deprecated C APIs ----------------- * The :c:macro:`!Py_HUGE_VAL` macro is now :term:`soft deprecated`. - Use :c:macro:`!Py_INFINITY` instead. + Use :c:macro:`!INFINITY` instead. (Contributed by Sergey B Kirpichev in :gh:`120026`.) * The :c:macro:`!Py_IS_NAN`, :c:macro:`!Py_IS_INFINITY`, diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index c543b6e6c2a779..f0fd49c9033bc1 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1095,6 +1095,10 @@ Deprecated C APIs since 3.15 and will be removed in 3.17. (Contributed by Nikita Sobolev in :gh:`136355`.) +* :c:macro:`!Py_INFINITY` macro is :term:`soft deprecated`, + use the C11 standard ```` :c:macro:`!INFINITY` instead. + (Contributed by Sergey B Kirpichev in :gh:`141004`.) + * :c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated since 3.15 and will be removed in 3.20. (Contributed by Sergey B Kirpichev in :gh:`141004`.) diff --git a/Include/floatobject.h b/Include/floatobject.h index 4d24a76edd5de1..814337b070ab50 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -18,14 +18,14 @@ PyAPI_DATA(PyTypeObject) PyFloat_Type; #define Py_RETURN_NAN return PyFloat_FromDouble(Py_NAN) -#define Py_RETURN_INF(sign) \ - do { \ - if (copysign(1., sign) == 1.) { \ - return PyFloat_FromDouble(Py_INFINITY); \ - } \ - else { \ - return PyFloat_FromDouble(-Py_INFINITY); \ - } \ +#define Py_RETURN_INF(sign) \ + do { \ + if (copysign(1., sign) == 1.) { \ + return PyFloat_FromDouble(INFINITY); \ + } \ + else { \ + return PyFloat_FromDouble(-INFINITY); \ + } \ } while(0) PyAPI_FUNC(double) PyFloat_GetMax(void); diff --git a/Include/internal/pycore_pymath.h b/Include/internal/pycore_pymath.h index eea8996ba68ca0..4fcac3aab8bf51 100644 --- a/Include/internal/pycore_pymath.h +++ b/Include/internal/pycore_pymath.h @@ -33,7 +33,7 @@ extern "C" { static inline void _Py_ADJUST_ERANGE1(double x) { if (errno == 0) { - if (x == Py_INFINITY || x == -Py_INFINITY) { + if (x == INFINITY || x == -INFINITY) { errno = ERANGE; } } @@ -44,8 +44,8 @@ static inline void _Py_ADJUST_ERANGE1(double x) static inline void _Py_ADJUST_ERANGE2(double x, double y) { - if (x == Py_INFINITY || x == -Py_INFINITY || - y == Py_INFINITY || y == -Py_INFINITY) + if (x == INFINITY || x == -INFINITY || + y == INFINITY || y == -INFINITY) { if (errno == 0) { errno = ERANGE; diff --git a/Include/pymath.h b/Include/pymath.h index 0f9f0f3b2990fe..7cfe441365df78 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -45,13 +45,14 @@ #define Py_IS_FINITE(X) isfinite(X) // Py_INFINITY: Value that evaluates to a positive double infinity. +// Soft deprecated since Python 3.15, use INFINITY instead. #ifndef Py_INFINITY # define Py_INFINITY ((double)INFINITY) #endif /* Py_HUGE_VAL should always be the same as Py_INFINITY. But historically * this was not reliable and Python did not require IEEE floats and C99 - * conformity. The macro was soft deprecated in Python 3.14, use Py_INFINITY instead. + * conformity. The macro was soft deprecated in Python 3.14, use INFINITY instead. */ #ifndef Py_HUGE_VAL # define Py_HUGE_VAL HUGE_VAL diff --git a/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst b/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst new file mode 100644 index 00000000000000..a054f8eda6fb0b --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst @@ -0,0 +1 @@ +The :c:macro:`!Py_INFINITY` macro is :term:`soft deprecated`. diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index a4ea5557a6a415..aee3e4f343d8be 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -150,7 +150,7 @@ special_type(double d) #define P14 0.25*Py_MATH_PI #define P12 0.5*Py_MATH_PI #define P34 0.75*Py_MATH_PI -#define INF Py_INFINITY +#define INF INFINITY #define N Py_NAN #define U -9.5426319407711027e33 /* unlikely value, used as placeholder */ @@ -1186,11 +1186,11 @@ cmath_exec(PyObject *mod) if (PyModule_Add(mod, "tau", PyFloat_FromDouble(Py_MATH_TAU)) < 0) { return -1; } - if (PyModule_Add(mod, "inf", PyFloat_FromDouble(Py_INFINITY)) < 0) { + if (PyModule_Add(mod, "inf", PyFloat_FromDouble(INFINITY)) < 0) { return -1; } - Py_complex infj = {0.0, Py_INFINITY}; + Py_complex infj = {0.0, INFINITY}; if (PyModule_Add(mod, "infj", PyComplex_FromCComplex(infj)) < 0) { return -1; } diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index de1886451eda8f..11c46c987e146a 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -395,7 +395,7 @@ m_tgamma(double x) if (x == 0.0) { errno = EDOM; /* tgamma(+-0.0) = +-inf, divide-by-zero */ - return copysign(Py_INFINITY, x); + return copysign(INFINITY, x); } /* integer arguments */ @@ -426,7 +426,7 @@ m_tgamma(double x) } else { errno = ERANGE; - return Py_INFINITY; + return INFINITY; } } @@ -490,14 +490,14 @@ m_lgamma(double x) if (isnan(x)) return x; /* lgamma(nan) = nan */ else - return Py_INFINITY; /* lgamma(+-inf) = +inf */ + return INFINITY; /* lgamma(+-inf) = +inf */ } /* integer arguments */ if (x == floor(x) && x <= 2.0) { if (x <= 0.0) { errno = EDOM; /* lgamma(n) = inf, divide-by-zero for */ - return Py_INFINITY; /* integers n <= 0 */ + return INFINITY; /* integers n <= 0 */ } else { return 0.0; /* lgamma(1) = lgamma(2) = 0.0 */ @@ -633,7 +633,7 @@ m_log(double x) return log(x); errno = EDOM; if (x == 0.0) - return -Py_INFINITY; /* log(0) = -inf */ + return -INFINITY; /* log(0) = -inf */ else return Py_NAN; /* log(-ve) = nan */ } @@ -676,7 +676,7 @@ m_log2(double x) } else if (x == 0.0) { errno = EDOM; - return -Py_INFINITY; /* log2(0) = -inf, divide-by-zero */ + return -INFINITY; /* log2(0) = -inf, divide-by-zero */ } else { errno = EDOM; @@ -692,7 +692,7 @@ m_log10(double x) return log10(x); errno = EDOM; if (x == 0.0) - return -Py_INFINITY; /* log10(0) = -inf */ + return -INFINITY; /* log10(0) = -inf */ else return Py_NAN; /* log10(-ve) = nan */ } @@ -1500,7 +1500,7 @@ math_ldexp_impl(PyObject *module, double x, PyObject *i) errno = 0; } else if (exp > INT_MAX) { /* overflow */ - r = copysign(Py_INFINITY, x); + r = copysign(INFINITY, x); errno = ERANGE; } else if (exp < INT_MIN) { /* underflow to +-0 */ @@ -2983,7 +2983,7 @@ math_ulp_impl(PyObject *module, double x) if (isinf(x)) { return x; } - double inf = Py_INFINITY; + double inf = INFINITY; double x2 = nextafter(x, inf); if (isinf(x2)) { /* special case: x is the largest positive representable float */ @@ -3007,7 +3007,7 @@ math_exec(PyObject *module) if (PyModule_Add(module, "tau", PyFloat_FromDouble(Py_MATH_TAU)) < 0) { return -1; } - if (PyModule_Add(module, "inf", PyFloat_FromDouble(Py_INFINITY)) < 0) { + if (PyModule_Add(module, "inf", PyFloat_FromDouble(INFINITY)) < 0) { return -1; } if (PyModule_Add(module, "nan", PyFloat_FromDouble(fabs(Py_NAN))) < 0) { diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 6247376a0e68f5..3612c2699a557d 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -139,8 +139,8 @@ _Py_c_prod(Py_complex z, Py_complex w) recalc = 1; } if (recalc) { - r.real = Py_INFINITY*(a*c - b*d); - r.imag = Py_INFINITY*(a*d + b*c); + r.real = INFINITY*(a*c - b*d); + r.imag = INFINITY*(a*d + b*c); } } @@ -229,8 +229,8 @@ _Py_c_quot(Py_complex a, Py_complex b) { const double x = copysign(isinf(a.real) ? 1.0 : 0.0, a.real); const double y = copysign(isinf(a.imag) ? 1.0 : 0.0, a.imag); - r.real = Py_INFINITY * (x*b.real + y*b.imag); - r.imag = Py_INFINITY * (y*b.real - x*b.imag); + r.real = INFINITY * (x*b.real + y*b.imag); + r.imag = INFINITY * (y*b.real - x*b.imag); } else if ((isinf(abs_breal) || isinf(abs_bimag)) && isfinite(a.real) && isfinite(a.imag)) diff --git a/Objects/floatobject.c b/Objects/floatobject.c index ef613efe4e7f44..78006783c6ec78 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -2415,7 +2415,7 @@ PyFloat_Unpack2(const char *data, int le) if (e == 0x1f) { if (f == 0) { /* Infinity */ - return sign ? -Py_INFINITY : Py_INFINITY; + return sign ? -INFINITY : INFINITY; } else { /* NaN */ diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 7b74f613ed563b..e8aca939d1fb98 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -43,7 +43,7 @@ _Py_parse_inf_or_nan(const char *p, char **endptr) s += 3; if (case_insensitive_match(s, "inity")) s += 5; - retval = negate ? -Py_INFINITY : Py_INFINITY; + retval = negate ? -INFINITY : INFINITY; } else if (case_insensitive_match(s, "nan")) { s += 3; @@ -286,7 +286,7 @@ _PyOS_ascii_strtod(const char *nptr, char **endptr) string, -1.0 is returned and again ValueError is raised. On overflow (e.g., when trying to convert '1e500' on an IEEE 754 machine), - if overflow_exception is NULL then +-Py_INFINITY is returned, and no Python + if overflow_exception is NULL then +-INFINITY is returned, and no Python exception is raised. Otherwise, overflow_exception should point to a Python exception, this exception will be raised, -1.0 will be returned, and *endptr will point just past the end of the converted value. From f963864cb54c2e7364b2c850485c6bf25479f6f2 Mon Sep 17 00:00:00 2001 From: yihong Date: Wed, 12 Nov 2025 20:45:43 +0800 Subject: [PATCH 37/58] gh-141464: a typo in profiling sampling when can not run warning in linux (#141465) --- Lib/profiling/sampling/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/profiling/sampling/__main__.py b/Lib/profiling/sampling/__main__.py index a76ca62e2cda50..cd1425b8b9c7d3 100644 --- a/Lib/profiling/sampling/__main__.py +++ b/Lib/profiling/sampling/__main__.py @@ -15,7 +15,7 @@ """ LINUX_PERMISSION_ERROR = """ -🔒 Tachyon was unable to acess process memory. This could be because tachyon +🔒 Tachyon was unable to access process memory. This could be because tachyon has insufficient privileges (the required capability is CAP_SYS_PTRACE). Unprivileged processes cannot trace processes that they cannot send signals to or those running set-user-ID/set-group-ID programs, for security reasons. From 88aeff8eabefdc13b6fb29edb3cde618f743a034 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Wed, 12 Nov 2025 14:22:01 +0000 Subject: [PATCH 38/58] gh-87710: Update mime type for ``.ai`` (#141239) --- Doc/whatsnew/3.15.rst | 4 +++- Lib/mimetypes.py | 2 +- Lib/test/test_mimetypes.py | 1 + .../Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index f0fd49c9033bc1..c6089f63dee2cb 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -472,7 +472,9 @@ mimetypes * Add ``application/node`` MIME type for ``.cjs`` extension. (Contributed by John Franey in :gh:`140937`.) * Add ``application/toml``. (Contributed by Gil Forcada in :gh:`139959`.) * Rename ``application/x-texinfo`` to ``application/texinfo``. - (Contributed by Charlie Lin in :gh:`140165`) + (Contributed by Charlie Lin in :gh:`140165`.) +* Changed the MIME type for ``.ai`` files to ``application/pdf``. + (Contributed by Stan Ulbrych in :gh:`141239`.) mmap diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index d6896fc4042cb4..42477713c78418 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -497,9 +497,9 @@ def _default_mime_types(): '.oda' : 'application/oda', '.ogx' : 'application/ogg', '.pdf' : 'application/pdf', + '.ai' : 'application/pdf', '.p7c' : 'application/pkcs7-mime', '.ps' : 'application/postscript', - '.ai' : 'application/postscript', '.eps' : 'application/postscript', '.texi' : 'application/texinfo', '.texinfo': 'application/texinfo', diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 746984ec0ca9df..734144983591b4 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -229,6 +229,7 @@ def check_extensions(): ("application/octet-stream", ".bin"), ("application/gzip", ".gz"), ("application/ogg", ".ogx"), + ("application/pdf", ".pdf"), ("application/postscript", ".ps"), ("application/texinfo", ".texi"), ("application/toml", ".toml"), diff --git a/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst b/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst new file mode 100644 index 00000000000000..62073280e32b81 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst @@ -0,0 +1 @@ +:mod:`mimetypes`: Update mime type for ``.ai`` files to ``application/pdf``. From 2ac738d325a6934e39fecb097f43d4d4ed97a2b9 Mon Sep 17 00:00:00 2001 From: M Bussonnier Date: Wed, 12 Nov 2025 16:20:08 +0100 Subject: [PATCH 39/58] gh-132657: add regression test for `PySet_Contains` with unhashable type (#141411) --- Modules/_testlimitedcapi/set.c | 47 ++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/Modules/_testlimitedcapi/set.c b/Modules/_testlimitedcapi/set.c index 35da5fa5f008e1..34ed6b1d60b5a4 100644 --- a/Modules/_testlimitedcapi/set.c +++ b/Modules/_testlimitedcapi/set.c @@ -155,6 +155,51 @@ test_frozenset_add_in_capi(PyObject *self, PyObject *Py_UNUSED(obj)) return NULL; } +static PyObject * +test_set_contains_does_not_convert_unhashable_key(PyObject *self, PyObject *Py_UNUSED(obj)) +{ + // See https://docs.python.org/3/c-api/set.html#c.PySet_Contains + PyObject *outer_set = PySet_New(NULL); + + PyObject *needle = PySet_New(NULL); + if (needle == NULL) { + Py_DECREF(outer_set); + return NULL; + } + + PyObject *num = PyLong_FromLong(42); + if (num == NULL) { + Py_DECREF(outer_set); + Py_DECREF(needle); + return NULL; + } + + if (PySet_Add(needle, num) < 0) { + Py_DECREF(outer_set); + Py_DECREF(needle); + Py_DECREF(num); + return NULL; + } + + int result = PySet_Contains(outer_set, needle); + + Py_DECREF(num); + Py_DECREF(needle); + Py_DECREF(outer_set); + + if (result < 0) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Clear(); + Py_RETURN_NONE; + } + return NULL; + } + + PyErr_SetString(PyExc_AssertionError, + "PySet_Contains should have raised TypeError for unhashable key"); + return NULL; +} + static PyMethodDef test_methods[] = { {"set_check", set_check, METH_O}, {"set_checkexact", set_checkexact, METH_O}, @@ -174,6 +219,8 @@ static PyMethodDef test_methods[] = { {"set_clear", set_clear, METH_O}, {"test_frozenset_add_in_capi", test_frozenset_add_in_capi, METH_NOARGS}, + {"test_set_contains_does_not_convert_unhashable_key", + test_set_contains_does_not_convert_unhashable_key, METH_NOARGS}, {NULL}, }; From f1330b35b8eb43904dfed0656acde80c08d63176 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Wed, 12 Nov 2025 16:37:54 +0000 Subject: [PATCH 40/58] gh-141004: Document `Py_MATH_{E, PI, TAU}` constants (#141373) --- Doc/c-api/float.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index b6020533a2b9d9..79de5daaa90d8f 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -99,6 +99,11 @@ Floating-Point Objects the C11 standard ```` header. +.. c:macro:: Py_MATH_E + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.e` constant. + + .. c:macro:: Py_MATH_El High precision (long double) definition of :data:`~math.e` constant. @@ -106,6 +111,11 @@ Floating-Point Objects .. deprecated-removed:: 3.15 3.20 +.. c:macro:: Py_MATH_PI + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.pi` constant. + + .. c:macro:: Py_MATH_PIl High precision (long double) definition of :data:`~math.pi` constant. @@ -113,6 +123,13 @@ Floating-Point Objects .. deprecated-removed:: 3.15 3.20 +.. c:macro:: Py_MATH_TAU + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.tau` constant. + + .. versionadded:: 3.6 + + .. c:macro:: Py_RETURN_NAN Return :data:`math.nan` from a function. From 9cd5427d9619b96db20d0347a136b3d331af71ae Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 12 Nov 2025 11:38:17 -0500 Subject: [PATCH 41/58] gh-141004: Document `PyType_SUPPORTS_WEAKREFS` (GH-141408) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/type.rst | 26 ++++++++++++++++++++++++++ Doc/c-api/weakref.rst | 8 ++++++++ 2 files changed, 34 insertions(+) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 29ffeb7c483dce..b608f815160f76 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -195,12 +195,14 @@ Type Objects before initialization) and should be paired with :c:func:`PyObject_Free` in :c:member:`~PyTypeObject.tp_free`. + .. c:function:: PyObject* PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) Generic handler for the :c:member:`~PyTypeObject.tp_new` slot of a type object. Creates a new instance using the type's :c:member:`~PyTypeObject.tp_alloc` slot and returns the resulting object. + .. c:function:: int PyType_Ready(PyTypeObject *type) Finalize a type object. This should be called on all type objects to finish @@ -217,6 +219,7 @@ Type Objects GC protocol itself by at least implementing the :c:member:`~PyTypeObject.tp_traverse` handle. + .. c:function:: PyObject* PyType_GetName(PyTypeObject *type) Return the type's name. Equivalent to getting the type's @@ -224,6 +227,7 @@ Type Objects .. versionadded:: 3.11 + .. c:function:: PyObject* PyType_GetQualName(PyTypeObject *type) Return the type's qualified name. Equivalent to getting the @@ -239,6 +243,7 @@ Type Objects .. versionadded:: 3.13 + .. c:function:: PyObject* PyType_GetModuleName(PyTypeObject *type) Return the type's module name. Equivalent to getting the @@ -246,6 +251,7 @@ Type Objects .. versionadded:: 3.13 + .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the @@ -262,6 +268,7 @@ Type Objects :c:func:`PyType_GetSlot` can now accept all types. Previously, it was limited to :ref:`heap types `. + .. c:function:: PyObject* PyType_GetModule(PyTypeObject *type) Return the module object associated with the given type when the type was @@ -281,6 +288,7 @@ Type Objects .. versionadded:: 3.9 + .. c:function:: void* PyType_GetModuleState(PyTypeObject *type) Return the state of the module object associated with the given type. @@ -295,6 +303,7 @@ Type Objects .. versionadded:: 3.9 + .. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) Find the first superclass whose module was created from @@ -314,6 +323,7 @@ Type Objects .. versionadded:: 3.11 + .. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) Find the first superclass in *type*'s :term:`method resolution order` whose @@ -332,6 +342,7 @@ Type Objects .. versionadded:: 3.14 + .. c:function:: int PyUnstable_Type_AssignVersionTag(PyTypeObject *type) Attempt to assign a version tag to the given type. @@ -342,6 +353,16 @@ Type Objects .. versionadded:: 3.12 +.. c:function:: int PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) + + Return true if instances of *type* support creating weak references, false + otherwise. This function always succeeds. *type* must not be ``NULL``. + + .. seealso:: + * :ref:`weakrefobjects` + * :py:mod:`weakref` + + Creating Heap-Allocated Types ............................. @@ -390,6 +411,7 @@ The following functions and structs are used to create .. versionadded:: 3.12 + .. c:function:: PyObject* PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromMetaclass(NULL, module, spec, bases)``. @@ -416,6 +438,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, bases)``. @@ -437,6 +460,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, NULL)``. @@ -457,6 +481,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: int PyType_Freeze(PyTypeObject *type) Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag. @@ -628,6 +653,7 @@ The following functions and structs are used to create * :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` rather than ``NULL``) + .. c:macro:: Py_tp_token A :c:member:`~PyType_Slot.slot` that records a static memory layout ID diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 39e4febd3ef0f2..db6ae0a9d4ea3d 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -45,6 +45,10 @@ as much as it can. weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + .. seealso:: + :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly + referenceable. + .. c:function:: PyObject* PyWeakref_NewProxy(PyObject *ob, PyObject *callback) @@ -57,6 +61,10 @@ as much as it can. is not a weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + .. seealso:: + :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly + referenceable. + .. c:function:: int PyWeakref_GetRef(PyObject *ref, PyObject **pobj) From d162c427904e232fec52d8da759caa1bfa4c01b5 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 12 Nov 2025 10:09:25 -0800 Subject: [PATCH 42/58] GH-140479: Update JIT builds to use LLVM 21 (#140973) --- .github/workflows/jit.yml | 8 ++++---- ...-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst | 1 + PCbuild/get_externals.bat | 4 ++-- Tools/jit/README.md | 20 +++++++++---------- Tools/jit/_llvm.py | 4 ++-- 5 files changed, 19 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 69d900091a3bd1..62325250bd368e 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -68,7 +68,7 @@ jobs: - true - false llvm: - - 20 + - 21 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 @@ -138,7 +138,7 @@ jobs: fail-fast: false matrix: llvm: - - 20 + - 21 steps: - uses: actions/checkout@v4 with: @@ -166,7 +166,7 @@ jobs: fail-fast: false matrix: llvm: - - 20 + - 21 steps: - uses: actions/checkout@v4 with: @@ -193,7 +193,7 @@ jobs: fail-fast: false matrix: llvm: - - 20 + - 21 steps: - uses: actions/checkout@v4 with: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst new file mode 100644 index 00000000000000..0a615ed131127f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst @@ -0,0 +1 @@ +Update JIT compilation to use LLVM 21 at build time. diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 115203cecc8e48..9d02e2121cc623 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -82,7 +82,7 @@ if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.18 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 -if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-20.1.8.0 +if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-21.1.4.0 for %%b in (%binaries%) do ( if exist "%EXTERNALS_DIR%\%%b" ( @@ -92,7 +92,7 @@ for %%b in (%binaries%) do ( git clone --depth 1 https://github.com/%ORG%/cpython-bin-deps --branch %%b "%EXTERNALS_DIR%\%%b" ) else ( echo.Fetching %%b... - if "%%b"=="llvm-20.1.8.0" ( + if "%%b"=="llvm-21.1.4.0" ( %PYTHON% -E "%PCBUILD%\get_external.py" --release --organization %ORG% --externals-dir "%EXTERNALS_DIR%" %%b ) else ( %PYTHON% -E "%PCBUILD%\get_external.py" --binary --organization %ORG% --externals-dir "%EXTERNALS_DIR%" %%b diff --git a/Tools/jit/README.md b/Tools/jit/README.md index d83b09aab59f8c..dd7deb7b256449 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -9,32 +9,32 @@ Python 3.11 or newer is required to build the JIT. The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 20 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 21 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. It's easy to install all of the required tools: ### Linux -Install LLVM 20 on Ubuntu/Debian: +Install LLVM 21 on Ubuntu/Debian: ```sh wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh -sudo ./llvm.sh 20 +sudo ./llvm.sh 21 ``` -Install LLVM 20 on Fedora Linux 40 or newer: +Install LLVM 21 on Fedora Linux 40 or newer: ```sh -sudo dnf install 'clang(major) = 20' 'llvm(major) = 20' +sudo dnf install 'clang(major) = 21' 'llvm(major) = 21' ``` ### macOS -Install LLVM 20 with [Homebrew](https://brew.sh): +Install LLVM 21 with [Homebrew](https://brew.sh): ```sh -brew install llvm@20 +brew install llvm@21 ``` Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. @@ -43,18 +43,18 @@ Homebrew won't add any of the tools to your `$PATH`. That's okay; the build scri LLVM is downloaded automatically (along with other external binary dependencies) by `PCbuild\build.bat`. -Otherwise, you can install LLVM 20 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=20), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** +Otherwise, you can install LLVM 21 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=21), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** Alternatively, you can use [chocolatey](https://chocolatey.org): ```sh -choco install llvm --version=20.1.8 +choco install llvm --version=21.1.0 ``` ### Dev Containers If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no -need to install LLVM as the Fedora 42 base image includes LLVM 20 out of the box. +need to install LLVM as the Fedora 43 base image includes LLVM 21 out of the box. ## Building diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index f1b0ad3f5dbc43..0b9cb5192f1b75 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -11,8 +11,8 @@ import _targets -_LLVM_VERSION = "20" -_EXTERNALS_LLVM_TAG = "llvm-20.1.8.0" +_LLVM_VERSION = "21" +_EXTERNALS_LLVM_TAG = "llvm-21.1.4.0" _P = typing.ParamSpec("_P") _R = typing.TypeVar("_R") From fbcac799518e0cb29fcf5f84ed1fa001010b9073 Mon Sep 17 00:00:00 2001 From: Bob Kline Date: Wed, 12 Nov 2025 13:25:23 -0500 Subject: [PATCH 43/58] gh-141412: Use reliable target URL for urllib example (GH-141428) The endpoint used for demonstrating reading URLs is no longer stable. This change substitutes a target over which we have more control. --- Doc/tutorial/stdlib.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 49a3e370a4c018..342c1a00193959 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -183,13 +183,13 @@ protocols. Two of the simplest are :mod:`urllib.request` for retrieving data from URLs and :mod:`smtplib` for sending mail:: >>> from urllib.request import urlopen - >>> with urlopen('http://worldtimeapi.org/api/timezone/etc/UTC.txt') as response: + >>> with urlopen('https://docs.python.org/3/') as response: ... for line in response: ... line = line.decode() # Convert bytes to a str - ... if line.startswith('datetime'): + ... if 'updated' in line: ... print(line.rstrip()) # Remove trailing newline ... - datetime: 2022-01-01T01:36:47.689215+00:00 + Last updated on Nov 11, 2025 (20:11 UTC). >>> import smtplib >>> server = smtplib.SMTP('localhost') From 1f381a579cc50aa82838de84c2294b4979586bd9 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 12 Nov 2025 10:26:50 -0800 Subject: [PATCH 44/58] Add details about JIT build infrastructure and updating dependencies to `Tools/jit` (#141167) --- Tools/jit/README.md | 3 +++ Tools/jit/jit_infra.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 Tools/jit/jit_infra.md diff --git a/Tools/jit/README.md b/Tools/jit/README.md index dd7deb7b256449..c70c0c47d94ad2 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -66,6 +66,9 @@ Otherwise, just configure and build as you normally would. Cross-compiling "just The JIT can also be enabled or disabled using the `PYTHON_JIT` environment variable, even on builds where it is enabled or disabled by default. More details about configuring CPython with the JIT and optional values for `--enable-experimental-jit` can be found [here](https://docs.python.org/dev/using/configure.html#cmdoption-enable-experimental-jit). +## Miscellaneous +If you're looking for information on how to update the JIT build dependencies, see [JIT Build Infrastructure](jit_infra.md). + [^pep-744]: [PEP 744](https://peps.python.org/pep-0744/) [^why-llvm]: Clang is specifically needed because it's the only C compiler with support for guaranteed tail calls (`musttail`), which are required by CPython's continuation-passing-style approach to JIT compilation. Since LLVM also includes other functionalities we need (namely, object file parsing and disassembly), it's convenient to only support one toolchain at this time. diff --git a/Tools/jit/jit_infra.md b/Tools/jit/jit_infra.md new file mode 100644 index 00000000000000..1a954755611d19 --- /dev/null +++ b/Tools/jit/jit_infra.md @@ -0,0 +1,28 @@ +# JIT Build Infrastructure + +This document includes details about the intricacies of the JIT build infrastructure. + +## Updating LLVM + +When we update LLVM, we need to also update the LLVM release artifact for Windows builds. This is because Windows builds automatically pull prebuilt LLVM binaries in our pipelines (e.g. notice that `.github/workflows/jit.yml` does not explicitly download LLVM or build it from source). + +To update the LLVM release artifact for Windows builds, follow these steps: +1. Go to the [LLVM releases page](https://github.com/llvm/llvm-project/releases). +1. Download x86_64 Windows artifact for the desired LLVM version (e.g. `clang+llvm-21.1.4-x86_64-pc-windows-msvc.tar.xz`). +1. Extract and repackage the tarball with the correct directory structure. For example: + ```bash + tar -xf clang+llvm-21.1.4-x86_64-pc-windows-msvc.tar.xz + mv clang+llvm-21.1.4-x86_64-pc-windows-msvc llvm-21.1.4.0 + tar -cf - llvm-21.1.4.0 | pv | xz > llvm-21.1.4.0.tar.xz + ``` + The tarball must contain a top-level directory named `llvm-{version}.0/`. +1. Go to [cpython-bin-deps](https://github.com/python/cpython-bin-deps). +1. Create a new release with the updated LLVM artifact. + - Create a new tag to match the LLVM version (e.g. `llvm-21.1.4.0`). + - Specify the release title (e.g. `LLVM 21.1.4 for x86_64 Windows`). + - Upload the asset (you can leave all other fields the same). + +### Other notes +- You must make sure that the name of the artifact matches exactly what is expected in `Tools/jit/_llvm.py` and `PCbuild/get_externals.py`. +- We don't need multiple release artifacts for each architecture because LLVM can cross-compile for different architectures on Windows; x86_64 is sufficient. +- You must have permissions to create releases in the `cpython-bin-deps` repository. If you don't have permissions, you should contact one of the organization admins. \ No newline at end of file From 35ed3e4cedc8aef3936da81a6b64e90374532b13 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Wed, 12 Nov 2025 22:04:02 +0300 Subject: [PATCH 45/58] gh-140936: Fix JIT assertion crash at finalization if some generator is alive (GH-140969) --- Lib/test/test_capi/test_opt.py | 19 +++++++++++++++++++ Python/optimizer.c | 8 +++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 4e94f62d35eba2..e65556fb28f92d 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -2660,6 +2660,25 @@ def f(): f() + def test_interpreter_finalization_with_generator_alive(self): + script_helper.assert_python_ok("-c", textwrap.dedent(""" + import sys + t = tuple(range(%d)) + def simple_for(): + for x in t: + x + + def gen(): + try: + yield + except: + simple_for() + + sys.settrace(lambda *args: None) + simple_for() + g = gen() + next(g) + """ % _testinternalcapi.SPECIALIZATION_THRESHOLD)) def global_identity(x): diff --git a/Python/optimizer.c b/Python/optimizer.c index f44f8a9614b846..3b7e2dafab85bb 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -118,7 +118,13 @@ _PyOptimizer_Optimize( { _PyStackRef *stack_pointer = frame->stackpointer; PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(interp->jit); + if (!interp->jit) { + // gh-140936: It is possible that interp->jit will become false during + // interpreter finalization. However, the specialized JUMP_BACKWARD_JIT + // instruction may still be present. In this case, we should + // return immediately without optimization. + return 0; + } assert(!interp->compiling); #ifndef Py_GIL_DISABLED interp->compiling = true; From 558936bec1f1e0f8346063a8cb2b2782d085178e Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Thu, 13 Nov 2025 05:41:26 +0800 Subject: [PATCH 46/58] gh-141442: Add escaping to iOS testbed arguments (#141443) Xcode concatenates the test argument array, losing quoting in the process. --- Apple/testbed/__main__.py | 3 ++- .../Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst diff --git a/Apple/testbed/__main__.py b/Apple/testbed/__main__.py index 42eb60a4c8dc02..49974cb142853c 100644 --- a/Apple/testbed/__main__.py +++ b/Apple/testbed/__main__.py @@ -2,6 +2,7 @@ import json import os import re +import shlex import shutil import subprocess import sys @@ -252,7 +253,7 @@ def update_test_plan(testbed_path, platform, args): test_plan = json.load(f) test_plan["defaultOptions"]["commandLineArgumentEntries"] = [ - {"argument": arg} for arg in args + {"argument": shlex.quote(arg)} for arg in args ] with test_plan_path.open("w", encoding="utf-8") as f: diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst b/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst new file mode 100644 index 00000000000000..073c070413f7e0 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst @@ -0,0 +1 @@ +The iOS testbed now correctly handles test arguments that contain spaces. From dc0987080ed66c662e8e0b24cdb8c179817bd697 Mon Sep 17 00:00:00 2001 From: Michael Cho Date: Wed, 12 Nov 2025 17:16:58 -0500 Subject: [PATCH 47/58] gh-124111: Fix TCL 9 thread detection (GH-128103) --- .../Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst | 2 ++ Modules/_tkinter.c | 4 ++++ 2 files changed, 6 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst diff --git a/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst b/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst new file mode 100644 index 00000000000000..8436cd2415dbd6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst @@ -0,0 +1,2 @@ +Updated Tcl threading configuration in :mod:`_tkinter` to assume that +threads are always available in Tcl 9 and later. diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index c0ed8977d8fd6f..8cea7b59fe730e 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -575,8 +575,12 @@ Tkapp_New(const char *screenName, const char *className, v->interp = Tcl_CreateInterp(); v->wantobjects = wantobjects; +#if TCL_MAJOR_VERSION >= 9 + v->threaded = 1; +#else v->threaded = Tcl_GetVar2Ex(v->interp, "tcl_platform", "threaded", TCL_GLOBAL_ONLY) != NULL; +#endif v->thread_id = Tcl_GetCurrentThread(); v->dispatching = 0; v->trace = NULL; From 26b7df2430cd5a9ee772bfa6ee03a73bd0b11619 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 12 Nov 2025 17:52:56 -0500 Subject: [PATCH 48/58] gh-141004: Document `PyRun_InteractiveOneObject` (GH-141405) --- Doc/c-api/veryhigh.rst | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 916c616dfee589..3b07b5fbed5959 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -100,18 +100,12 @@ the same library that the Python runtime is using. Otherwise, Python may not handle script file with LF line ending correctly. -.. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) - - This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, - leaving *flags* set to ``NULL``. - - -.. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) +.. c:function:: int PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) Read and execute a single statement from a file associated with an interactive device according to the *flags* argument. The user will be - prompted using ``sys.ps1`` and ``sys.ps2``. *filename* is decoded from the - :term:`filesystem encoding and error handler`. + prompted using ``sys.ps1`` and ``sys.ps2``. *filename* must be a Python + :class:`str` object. Returns ``0`` when the input was executed successfully, ``-1`` if there was an exception, or an error code @@ -120,6 +114,19 @@ the same library that the Python runtime is using. :file:`Python.h`, so must be included specifically if needed.) +.. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) + + This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, + leaving *flags* set to ``NULL``. + + +.. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) + + Similar to :c:func:`PyRun_InteractiveOneObject`, but *filename* is a + :c:expr:`const char*`, which is decoded from the + :term:`filesystem encoding and error handler`. + + .. c:function:: int PyRun_InteractiveLoop(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_InteractiveLoopFlags` below, From 781cc68c3c814e46e6a74c3a6a32e0f9f8f7eb11 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Wed, 12 Nov 2025 18:15:16 -0800 Subject: [PATCH 49/58] gh-137109: refactor warning about threads when forking (#141438) * gh-137109: refactor warning about threads when forking This splits the OS API specific functionality to get the number of threads out from the fallback Python method and warning raising code itself. This way the OS APIs can be queried before we've run `os.register_at_fork(after_in_parent=...)` registered functions which themselves may (re)start threads that would otherwise be detected. This is best effort. If the OS APIs are either unavailable or fail, the warning generating code still falls back to looking at the Python threading state after the CPython interpreter world has been restarted and the after_in_parent calls have been made. The common case for most Linux and macOS environments should work today. This also lines up with the existing TODO refactoring, we may choose to expose this API to get the number of OS threads in the `os` module in the future. * NEWS entry * avoid "function-prototype" compiler warning? --- ...-11-12-01-49-03.gh-issue-137109.D6sq2B.rst | 5 + Modules/posixmodule.c | 103 ++++++++++-------- 2 files changed, 65 insertions(+), 43 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst diff --git a/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst b/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst new file mode 100644 index 00000000000000..32f4e39f6d5f4c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst @@ -0,0 +1,5 @@ +The :mod:`os.fork` and related forking APIs will no longer warn in the +common case where Linux or macOS platform APIs return the number of threads +in a process and find the answer to be 1 even when a +:func:`os.register_at_fork` ``after_in_parent=`` callback (re)starts a +thread. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 6390f1fc5fe24f..fc609b2707c6c6 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8431,53 +8431,19 @@ os_register_at_fork_impl(PyObject *module, PyObject *before, // running in the process. Best effort, silent if unable to count threads. // Constraint: Quick. Never overcounts. Never leaves an error set. // -// This should only be called from the parent process after +// This MUST only be called from the parent process after // PyOS_AfterFork_Parent(). static int -warn_about_fork_with_threads(const char* name) +warn_about_fork_with_threads( + const char* name, // Name of the API to use in the warning message. + const Py_ssize_t num_os_threads // Only trusted when >= 1. +) { // It's not safe to issue the warning while the world is stopped, because // other threads might be holding locks that we need, which would deadlock. assert(!_PyRuntime.stoptheworld.world_stopped); - // TODO: Consider making an `os` module API to return the current number - // of threads in the process. That'd presumably use this platform code but - // raise an error rather than using the inaccurate fallback. - Py_ssize_t num_python_threads = 0; -#if defined(__APPLE__) && defined(HAVE_GETPID) - mach_port_t macos_self = mach_task_self(); - mach_port_t macos_task; - if (task_for_pid(macos_self, getpid(), &macos_task) == KERN_SUCCESS) { - thread_array_t macos_threads; - mach_msg_type_number_t macos_n_threads; - if (task_threads(macos_task, &macos_threads, - &macos_n_threads) == KERN_SUCCESS) { - num_python_threads = macos_n_threads; - } - } -#elif defined(__linux__) - // Linux /proc/self/stat 20th field is the number of threads. - FILE* proc_stat = fopen("/proc/self/stat", "r"); - if (proc_stat) { - size_t n; - // Size chosen arbitrarily. ~60% more bytes than a 20th column index - // observed on the author's workstation. - char stat_line[160]; - n = fread(&stat_line, 1, 159, proc_stat); - stat_line[n] = '\0'; - fclose(proc_stat); - - char *saveptr = NULL; - char *field = strtok_r(stat_line, " ", &saveptr); - unsigned int idx; - for (idx = 19; idx && field; --idx) { - field = strtok_r(NULL, " ", &saveptr); - } - if (idx == 0 && field) { // found the 20th field - num_python_threads = atoi(field); // 0 on error - } - } -#endif + Py_ssize_t num_python_threads = num_os_threads; if (num_python_threads <= 0) { // Fall back to just the number our threading module knows about. // An incomplete view of the world, but better than nothing. @@ -8530,6 +8496,51 @@ warn_about_fork_with_threads(const char* name) } return 0; } + +// If this returns <= 0, we were unable to successfully use any OS APIs. +// Returns a positive number of threads otherwise. +static Py_ssize_t get_number_of_os_threads(void) +{ + // TODO: Consider making an `os` module API to return the current number + // of threads in the process. That'd presumably use this platform code but + // raise an error rather than using the inaccurate fallback. + Py_ssize_t num_python_threads = 0; +#if defined(__APPLE__) && defined(HAVE_GETPID) + mach_port_t macos_self = mach_task_self(); + mach_port_t macos_task; + if (task_for_pid(macos_self, getpid(), &macos_task) == KERN_SUCCESS) { + thread_array_t macos_threads; + mach_msg_type_number_t macos_n_threads; + if (task_threads(macos_task, &macos_threads, + &macos_n_threads) == KERN_SUCCESS) { + num_python_threads = macos_n_threads; + } + } +#elif defined(__linux__) + // Linux /proc/self/stat 20th field is the number of threads. + FILE* proc_stat = fopen("/proc/self/stat", "r"); + if (proc_stat) { + size_t n; + // Size chosen arbitrarily. ~60% more bytes than a 20th column index + // observed on the author's workstation. + char stat_line[160]; + n = fread(&stat_line, 1, 159, proc_stat); + stat_line[n] = '\0'; + fclose(proc_stat); + + char *saveptr = NULL; + char *field = strtok_r(stat_line, " ", &saveptr); + unsigned int idx; + for (idx = 19; idx && field; --idx) { + field = strtok_r(NULL, " ", &saveptr); + } + if (idx == 0 && field) { // found the 20th field + num_python_threads = atoi(field); // 0 on error + } + } +#endif + return num_python_threads; +} #endif // HAVE_FORK1 || HAVE_FORKPTY || HAVE_FORK #ifdef HAVE_FORK1 @@ -8564,10 +8575,12 @@ os_fork1_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + // Called before AfterFork_Parent in case those hooks start threads. + Py_ssize_t num_os_threads = get_number_of_os_threads(); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - if (warn_about_fork_with_threads("fork1") < 0) { + if (warn_about_fork_with_threads("fork1", num_os_threads) < 0) { return NULL; } } @@ -8615,10 +8628,12 @@ os_fork_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + // Called before AfterFork_Parent in case those hooks start threads. + Py_ssize_t num_os_threads = get_number_of_os_threads(); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - if (warn_about_fork_with_threads("fork") < 0) + if (warn_about_fork_with_threads("fork", num_os_threads) < 0) return NULL; } if (pid == -1) { @@ -9476,6 +9491,8 @@ os_forkpty_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + // Called before AfterFork_Parent in case those hooks start threads. + Py_ssize_t num_os_threads = get_number_of_os_threads(); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); /* set O_CLOEXEC on master_fd */ @@ -9485,7 +9502,7 @@ os_forkpty_impl(PyObject *module) } // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - if (warn_about_fork_with_threads("forkpty") < 0) + if (warn_about_fork_with_threads("forkpty", num_os_threads) < 0) return NULL; } if (pid == -1) { From 63548b36998e7f7cd5c7c28b53b348a93f836737 Mon Sep 17 00:00:00 2001 From: Shamil Date: Thu, 13 Nov 2025 14:01:31 +0300 Subject: [PATCH 50/58] gh-140260: fix data race in `_struct` module initialization with subinterpreters (#140909) --- Lib/test/test_struct.py | 17 ++++ ...-11-02-15-28-33.gh-issue-140260.JNzlGz.rst | 2 + Modules/_struct.c | 91 ++++++++++--------- Tools/c-analyzer/cpython/ignored.tsv | 1 + 4 files changed, 70 insertions(+), 41 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 75c76a36ee92f5..cceecdd526c006 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -800,6 +800,23 @@ def test_c_complex_round_trip(self): round_trip = struct.unpack(f, struct.pack(f, z))[0] self.assertComplexesAreIdentical(z, round_trip) + @unittest.skipIf( + support.is_android or support.is_apple_mobile, + "Subinterpreters are not supported on Android and iOS" + ) + def test_endian_table_init_subinterpreters(self): + # Verify that the _struct extension module can be initialized + # concurrently in subinterpreters (gh-140260). + try: + from concurrent.futures import InterpreterPoolExecutor + except ImportError: + raise unittest.SkipTest("InterpreterPoolExecutor not available") + + code = "import struct" + with InterpreterPoolExecutor(max_workers=5) as executor: + results = executor.map(exec, [code] * 5) + self.assertListEqual(list(results), [None] * 5) + class UnpackIteratorTest(unittest.TestCase): """ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst new file mode 100644 index 00000000000000..96bf9b51e4862c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst @@ -0,0 +1,2 @@ +Fix :mod:`struct` data race in endian table initialization with +subinterpreters. Patch by Shamil Abdulaev. diff --git a/Modules/_struct.c b/Modules/_struct.c index f09252e82c3915..2acb3df3a30395 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -9,6 +9,7 @@ #include "Python.h" #include "pycore_bytesobject.h" // _PyBytesWriter +#include "pycore_lock.h" // _PyOnceFlag_CallOnce() #include "pycore_long.h" // _PyLong_AsByteArray() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() @@ -1505,6 +1506,53 @@ static formatdef lilendian_table[] = { {0} }; +/* Ensure endian table optimization happens exactly once across all interpreters */ +static _PyOnceFlag endian_tables_init_once = {0}; + +static int +init_endian_tables(void *Py_UNUSED(arg)) +{ + const formatdef *native = native_table; + formatdef *other, *ptr; +#if PY_LITTLE_ENDIAN + other = lilendian_table; +#else + other = bigendian_table; +#endif + /* Scan through the native table, find a matching + entry in the endian table and swap in the + native implementations whenever possible + (64-bit platforms may not have "standard" sizes) */ + while (native->format != '\0' && other->format != '\0') { + ptr = other; + while (ptr->format != '\0') { + if (ptr->format == native->format) { + /* Match faster when formats are + listed in the same order */ + if (ptr == other) + other++; + /* Only use the trick if the + size matches */ + if (ptr->size != native->size) + break; + /* Skip float and double, could be + "unknown" float format */ + if (ptr->format == 'd' || ptr->format == 'f') + break; + /* Skip _Bool, semantics are different for standard size */ + if (ptr->format == '?') + break; + ptr->pack = native->pack; + ptr->unpack = native->unpack; + break; + } + ptr++; + } + native++; + } + return 0; +} + static const formatdef * whichtable(const char **pfmt) @@ -2710,47 +2758,8 @@ _structmodule_exec(PyObject *m) return -1; } - /* Check endian and swap in faster functions */ - { - const formatdef *native = native_table; - formatdef *other, *ptr; -#if PY_LITTLE_ENDIAN - other = lilendian_table; -#else - other = bigendian_table; -#endif - /* Scan through the native table, find a matching - entry in the endian table and swap in the - native implementations whenever possible - (64-bit platforms may not have "standard" sizes) */ - while (native->format != '\0' && other->format != '\0') { - ptr = other; - while (ptr->format != '\0') { - if (ptr->format == native->format) { - /* Match faster when formats are - listed in the same order */ - if (ptr == other) - other++; - /* Only use the trick if the - size matches */ - if (ptr->size != native->size) - break; - /* Skip float and double, could be - "unknown" float format */ - if (ptr->format == 'd' || ptr->format == 'f') - break; - /* Skip _Bool, semantics are different for standard size */ - if (ptr->format == '?') - break; - ptr->pack = native->pack; - ptr->unpack = native->unpack; - break; - } - ptr++; - } - native++; - } - } + /* init cannot fail */ + (void)_PyOnceFlag_CallOnce(&endian_tables_init_once, init_endian_tables, NULL); /* Add some symbolic constants to the module */ state->StructError = PyErr_NewException("struct.error", NULL, NULL); diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 11a3cd794ff4d7..4621ad250f4633 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -24,6 +24,7 @@ Modules/posixmodule.c os_dup2_impl dup3_works - ## guards around resource init Python/thread_pthread.h PyThread__init_thread lib_initialized - +Modules/_struct.c - endian_tables_init_once - ##----------------------- ## other values (not Python-specific) From d8e6bdc0d083f4e76ac49574544555ad91257592 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Nov 2025 13:21:32 +0200 Subject: [PATCH 51/58] gh-135801: Add the module parameter to compile() etc (GH-139652) Many functions related to compiling or parsing Python code, such as compile(), ast.parse(), symtable.symtable(), and importlib.abc.InspectLoader.source_to_code() now allow to pass the module name used when filtering syntax warnings. --- Doc/library/ast.rst | 7 ++- Doc/library/functions.rst | 11 +++- Doc/library/importlib.rst | 10 +++- Doc/library/symtable.rst | 8 ++- Doc/whatsnew/3.15.rst | 7 +++ Include/internal/pycore_compile.h | 9 ++- Include/internal/pycore_parser.h | 3 +- Include/internal/pycore_pyerrors.h | 3 +- Include/internal/pycore_pythonrun.h | 6 ++ Include/internal/pycore_symtable.h | 3 +- Lib/ast.py | 5 +- Lib/importlib/_bootstrap_external.py | 9 +-- Lib/importlib/abc.py | 11 ++-- Lib/modulefinder.py | 2 +- Lib/profiling/sampling/_sync_coordinator.py | 2 +- Lib/profiling/tracing/__init__.py | 2 +- Lib/runpy.py | 6 +- Lib/symtable.py | 4 +- Lib/test/test_ast/test_ast.py | 10 ++++ Lib/test/test_builtin.py | 3 +- Lib/test/test_cmd_line_script.py | 23 ++++++-- Lib/test/test_compile.py | 10 ++++ Lib/test/test_import/__init__.py | 15 +---- Lib/test/test_runpy.py | 43 ++++++++++++++ Lib/test/test_symtable.py | 10 ++++ Lib/test/test_zipimport_support.py | 23 ++++++++ Lib/zipimport.py | 6 +- ...-10-06-14-19-47.gh-issue-135801.OhxEZS.rst | 6 ++ Modules/clinic/symtablemodule.c.h | 58 ++++++++++++++++--- Modules/symtablemodule.c | 19 +++++- Parser/lexer/state.c | 2 + Parser/lexer/state.h | 1 + Parser/peg_api.c | 6 +- Parser/pegen.c | 8 ++- Parser/pegen.h | 2 +- Parser/string_parser.c | 2 +- Parser/tokenizer/helpers.c | 4 +- Programs/_freeze_module.py | 2 +- Programs/freeze_test_frozenmain.py | 2 +- Python/ast_preprocess.c | 8 ++- Python/bltinmodule.c | 25 ++++++-- Python/clinic/bltinmodule.c.h | 26 ++++++--- Python/compile.c | 30 ++++++---- Python/errors.c | 7 ++- Python/pythonrun.c | 38 ++++++++++-- Python/symtable.c | 4 +- .../peg_extension/peg_extension.c | 4 +- 47 files changed, 390 insertions(+), 115 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 494621672171f2..0ea3c3c59a660d 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2205,10 +2205,10 @@ Async and await Apart from the node classes, the :mod:`ast` module defines these utility functions and classes for traversing abstract syntax trees: -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1) +.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1, module=None) Parse the source into an AST node. Equivalent to ``compile(source, - filename, mode, flags=FLAGS_VALUE, optimize=optimize)``, + filename, mode, flags=FLAGS_VALUE, optimize=optimize, module=module)``, where ``FLAGS_VALUE`` is ``ast.PyCF_ONLY_AST`` if ``optimize <= 0`` and ``ast.PyCF_OPTIMIZED_AST`` otherwise. @@ -2261,6 +2261,9 @@ and classes for traversing abstract syntax trees: The minimum supported version for ``feature_version`` is now ``(3, 7)``. The ``optimize`` argument was added. + .. versionadded:: next + Added the *module* parameter. + .. function:: unparse(ast_obj) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index e98793975556ef..3257daf89d327b 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -292,7 +292,9 @@ are always available. They are listed here in alphabetical order. :func:`property`. -.. function:: compile(source, filename, mode, flags=0, dont_inherit=False, optimize=-1) +.. function:: compile(source, filename, mode, flags=0, \ + dont_inherit=False, optimize=-1, \ + *, module=None) Compile the *source* into a code or AST object. Code objects can be executed by :func:`exec` or :func:`eval`. *source* can either be a normal string, a @@ -334,6 +336,10 @@ are always available. They are listed here in alphabetical order. ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false) or ``2`` (docstrings are removed too). + The optional argument *module* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax warnings + by module name. + This function raises :exc:`SyntaxError` if the compiled source is invalid, and :exc:`ValueError` if the source contains null bytes. @@ -371,6 +377,9 @@ are always available. They are listed here in alphabetical order. ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable support for top-level ``await``, ``async for``, and ``async with``. + .. versionadded:: next + Added the *module* parameter. + .. class:: complex(number=0, /) complex(string, /) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 602a7100a12350..03ba23b6216cbf 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -459,7 +459,7 @@ ABC hierarchy:: .. versionchanged:: 3.4 Raises :exc:`ImportError` instead of :exc:`NotImplementedError`. - .. staticmethod:: source_to_code(data, path='') + .. staticmethod:: source_to_code(data, path='', fullname=None) Create a code object from Python source. @@ -471,11 +471,19 @@ ABC hierarchy:: With the subsequent code object one can execute it in a module by running ``exec(code, module.__dict__)``. + The optional argument *fullname* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax + warnings by module name. + .. versionadded:: 3.4 .. versionchanged:: 3.5 Made the method static. + .. versionadded:: next + Added the *fullname* parameter. + + .. method:: exec_module(module) Implementation of :meth:`Loader.exec_module`. diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 54e19af4bd69a6..c0d9e79197de7c 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -21,11 +21,17 @@ tables. Generating Symbol Tables ------------------------ -.. function:: symtable(code, filename, compile_type) +.. function:: symtable(code, filename, compile_type, *, module=None) Return the toplevel :class:`SymbolTable` for the Python source *code*. *filename* is the name of the file containing the code. *compile_type* is like the *mode* argument to :func:`compile`. + The optional argument *module* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax warnings + by module name. + + .. versionadded:: next + Added the *module* parameter. Examining Symbol Tables diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index c6089f63dee2cb..3cb766978a7217 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -307,6 +307,13 @@ Other language changes not only integers or floats, although this does not improve precision. (Contributed by Serhiy Storchaka in :gh:`67795`.) +* Many functions related to compiling or parsing Python code, such as + :func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, + and :func:`importlib.abc.InspectLoader.source_to_code`, now allow to pass + the module name. It is needed to unambiguous :ref:`filter ` + syntax warnings by module name. + (Contributed by Serhiy Storchaka in :gh:`135801`.) + New modules =========== diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 1c60834fa2058c..527141b54d0dca 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -32,7 +32,8 @@ PyAPI_FUNC(PyCodeObject*) _PyAST_Compile( PyObject *filename, PyCompilerFlags *flags, int optimize, - struct _arena *arena); + struct _arena *arena, + PyObject *module); /* AST preprocessing */ extern int _PyCompile_AstPreprocess( @@ -41,7 +42,8 @@ extern int _PyCompile_AstPreprocess( PyCompilerFlags *flags, int optimize, struct _arena *arena, - int syntax_check_only); + int syntax_check_only, + PyObject *module); extern int _PyAST_Preprocess( struct _mod *, @@ -50,7 +52,8 @@ extern int _PyAST_Preprocess( int optimize, int ff_features, int syntax_check_only, - int enable_warnings); + int enable_warnings, + PyObject *module); typedef struct { diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index 2885dee63dcf94..2c46f59ab7da9f 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -48,7 +48,8 @@ extern struct _mod* _PyParser_ASTFromString( PyObject* filename, int mode, PyCompilerFlags *flags, - PyArena *arena); + PyArena *arena, + PyObject *module); extern struct _mod* _PyParser_ASTFromFile( FILE *fp, diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 2c2048f7e1272a..f80808fcc8c4d7 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -123,7 +123,8 @@ extern void _PyErr_SetNone(PyThreadState *tstate, PyObject *exception); extern PyObject* _PyErr_NoMemory(PyThreadState *tstate); extern int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, - int end_lineno, int end_col_offset); + int end_lineno, int end_col_offset, + PyObject *module); extern void _PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset); diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index c2832098ddb3e7..f954f1b63ef67c 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -33,6 +33,12 @@ extern const char* _Py_SourceAsString( PyCompilerFlags *cf, PyObject **cmd_copy); +extern PyObject * _Py_CompileStringObjectWithModule( + const char *str, + PyObject *filename, int start, + PyCompilerFlags *flags, int optimize, + PyObject *module); + /* Stack size, in "pointers". This must be large enough, so * no two calls to check recursion depth are more than this far diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 98099b4a497b01..9dbfa913219afa 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -188,7 +188,8 @@ extern struct symtable* _Py_SymtableStringObjectFlags( const char *str, PyObject *filename, int start, - PyCompilerFlags *flags); + PyCompilerFlags *flags, + PyObject *module); int _PyFuture_FromAST( struct _mod * mod, diff --git a/Lib/ast.py b/Lib/ast.py index 983ac1710d0205..d9743ba7ab40b1 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -24,7 +24,7 @@ def parse(source, filename='', mode='exec', *, - type_comments=False, feature_version=None, optimize=-1): + type_comments=False, feature_version=None, optimize=-1, module=None): """ Parse the source into an AST node. Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). @@ -44,7 +44,8 @@ def parse(source, filename='', mode='exec', *, feature_version = minor # Else it should be an int giving the minor version for 3.x. return compile(source, filename, mode, flags, - _feature_version=feature_version, optimize=optimize) + _feature_version=feature_version, optimize=optimize, + module=module) def literal_eval(node_or_string): diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 035ae0fcae14e8..4ab0e79ea6efeb 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -819,13 +819,14 @@ def get_source(self, fullname): name=fullname) from exc return decode_source(source_bytes) - def source_to_code(self, data, path, *, _optimize=-1): + def source_to_code(self, data, path, fullname=None, *, _optimize=-1): """Return the code object compiled from source. The 'data' argument can be any object type that compile() supports. """ return _bootstrap._call_with_frames_removed(compile, data, path, 'exec', - dont_inherit=True, optimize=_optimize) + dont_inherit=True, optimize=_optimize, + module=fullname) def get_code(self, fullname): """Concrete implementation of InspectLoader.get_code. @@ -894,7 +895,7 @@ def get_code(self, fullname): source_path=source_path) if source_bytes is None: source_bytes = self.get_data(source_path) - code_object = self.source_to_code(source_bytes, source_path) + code_object = self.source_to_code(source_bytes, source_path, fullname) _bootstrap._verbose_message('code object from {}', source_path) if (not sys.dont_write_bytecode and bytecode_path is not None and source_mtime is not None): @@ -1186,7 +1187,7 @@ def get_source(self, fullname): return '' def get_code(self, fullname): - return compile('', '', 'exec', dont_inherit=True) + return compile('', '', 'exec', dont_inherit=True, module=fullname) def create_module(self, spec): """Use default semantics for module creation.""" diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 1e47495f65fa02..5c13432b5bda8c 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -108,7 +108,7 @@ def get_code(self, fullname): source = self.get_source(fullname) if source is None: return None - return self.source_to_code(source) + return self.source_to_code(source, '', fullname) @abc.abstractmethod def get_source(self, fullname): @@ -120,12 +120,12 @@ def get_source(self, fullname): raise ImportError @staticmethod - def source_to_code(data, path=''): + def source_to_code(data, path='', fullname=None): """Compile 'data' into a code object. The 'data' argument can be anything that compile() can handle. The'path' argument should be where the data was retrieved (when applicable).""" - return compile(data, path, 'exec', dont_inherit=True) + return compile(data, path, 'exec', dont_inherit=True, module=fullname) exec_module = _bootstrap_external._LoaderBasics.exec_module load_module = _bootstrap_external._LoaderBasics.load_module @@ -163,9 +163,8 @@ def get_code(self, fullname): try: path = self.get_filename(fullname) except ImportError: - return self.source_to_code(source) - else: - return self.source_to_code(source, path) + path = '' + return self.source_to_code(source, path, fullname) _register( ExecutionLoader, diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index ac478ee7f51722..b115d99ab30ff1 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -334,7 +334,7 @@ def load_module(self, fqname, fp, pathname, file_info): self.msgout(2, "load_module ->", m) return m if type == _PY_SOURCE: - co = compile(fp.read(), pathname, 'exec') + co = compile(fp.read(), pathname, 'exec', module=fqname) elif type == _PY_COMPILED: try: data = fp.read() diff --git a/Lib/profiling/sampling/_sync_coordinator.py b/Lib/profiling/sampling/_sync_coordinator.py index 8716e654104791..adb040e89cc7b1 100644 --- a/Lib/profiling/sampling/_sync_coordinator.py +++ b/Lib/profiling/sampling/_sync_coordinator.py @@ -182,7 +182,7 @@ def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None: try: # Compile and execute the script - code = compile(source_code, script_path, 'exec') + code = compile(source_code, script_path, 'exec', module='__main__') exec(code, {'__name__': '__main__', '__file__': script_path}) except SyntaxError as e: raise TargetError(f"Syntax error in script {script_path}: {e}") from e diff --git a/Lib/profiling/tracing/__init__.py b/Lib/profiling/tracing/__init__.py index 2dc7ea92c8ca4d..a6b8edf721611f 100644 --- a/Lib/profiling/tracing/__init__.py +++ b/Lib/profiling/tracing/__init__.py @@ -185,7 +185,7 @@ def main(): progname = args[0] sys.path.insert(0, os.path.dirname(progname)) with io.open_code(progname) as fp: - code = compile(fp.read(), progname, 'exec') + code = compile(fp.read(), progname, 'exec', module='__main__') spec = importlib.machinery.ModuleSpec(name='__main__', loader=None, origin=progname) module = importlib.util.module_from_spec(spec) diff --git a/Lib/runpy.py b/Lib/runpy.py index ef54d3282eee06..f072498f6cb405 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -247,7 +247,7 @@ def _get_main_module_details(error=ImportError): sys.modules[main_name] = saved_main -def _get_code_from_file(fname): +def _get_code_from_file(fname, module): # Check for a compiled file first from pkgutil import read_code code_path = os.path.abspath(fname) @@ -256,7 +256,7 @@ def _get_code_from_file(fname): if code is None: # That didn't work, so try it as normal source code with io.open_code(code_path) as f: - code = compile(f.read(), fname, 'exec') + code = compile(f.read(), fname, 'exec', module=module) return code def run_path(path_name, init_globals=None, run_name=None): @@ -283,7 +283,7 @@ def run_path(path_name, init_globals=None, run_name=None): if isinstance(importer, type(None)): # Not a valid sys.path entry, so run the code directly # execfile() doesn't help as we want to allow compiled files - code = _get_code_from_file(path_name) + code = _get_code_from_file(path_name, run_name) return _run_module_code(code, init_globals, run_name, pkg_name=pkg_name, script_name=path_name) else: diff --git a/Lib/symtable.py b/Lib/symtable.py index 77475c3ffd9224..4c832e68f94cbd 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -17,13 +17,13 @@ __all__ = ["symtable", "SymbolTableType", "SymbolTable", "Class", "Function", "Symbol"] -def symtable(code, filename, compile_type): +def symtable(code, filename, compile_type, *, module=None): """ Return the toplevel *SymbolTable* for the source code. *filename* is the name of the file with the code and *compile_type* is the *compile()* mode argument. """ - top = _symtable.symtable(code, filename, compile_type) + top = _symtable.symtable(code, filename, compile_type, module=module) return _newSymbolTable(top, filename) class SymbolTableFactory: diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 551de5851daace..fb4a441ca64772 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -1083,6 +1083,16 @@ def test_filter_syntax_warnings_by_module(self): self.assertEqual(wm.filename, '') self.assertIs(wm.category, SyntaxWarning) + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.module\z') + warnings.filterwarnings('error', module=r'') + ast.parse(source, filename, module='package.module') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + class CopyTests(unittest.TestCase): """Test copying and pickling AST nodes.""" diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index fba46af6617640..ce60a5d095dd52 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1103,7 +1103,8 @@ def test_exec_filter_syntax_warnings_by_module(self): with warnings.catch_warnings(record=True) as wlog: warnings.simplefilter('error') - warnings.filterwarnings('always', module=r'\z') + warnings.filterwarnings('always', module=r'package.module\z') + warnings.filterwarnings('error', module=r'') exec(source, {'__name__': 'package.module', '__file__': filename}) self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) for wm in wlog: diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index f8115cc8300df7..cc1a625a5097d8 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -814,15 +814,26 @@ def test_filter_syntax_warnings_by_module(self): filename = support.findfile('test_import/data/syntax_warnings.py') rc, out, err = assert_python_ok( '-Werror', - '-Walways:::test.test_import.data.syntax_warnings', + '-Walways:::__main__', + '-Werror:::test.test_import.data.syntax_warnings', + '-Werror:::syntax_warnings', filename) self.assertEqual(err.count(b': SyntaxWarning: '), 6) - rc, out, err = assert_python_ok( - '-Werror', - '-Walways:::syntax_warnings', - filename) - self.assertEqual(err.count(b': SyntaxWarning: '), 6) + def test_zipfile_run_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with open(filename, 'rb') as f: + source = f.read() + with os_helper.temp_dir() as script_dir: + zip_name, _ = make_zip_pkg( + script_dir, 'test_zip', 'test_pkg', '__main__', source) + rc, out, err = assert_python_ok( + '-Werror', + '-Walways:::__main__', + '-Werror:::test_pkg.__main__', + os.path.join(zip_name, 'test_pkg') + ) + self.assertEqual(err.count(b': SyntaxWarning: '), 12) def tearDownModule(): diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 9c2364491fe08d..30f21875b22ab3 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1759,6 +1759,16 @@ def test_filter_syntax_warnings_by_module(self): self.assertEqual(wm.filename, filename) self.assertIs(wm.category, SyntaxWarning) + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.module\z') + warnings.filterwarnings('error', module=module_re) + compile(source, filename, 'exec', module='package.module') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + @support.subTests('src', [ textwrap.dedent(""" def f(): diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index e87d8b7e7bbb1f..fe669bb04df02a 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1259,20 +1259,7 @@ def test_filter_syntax_warnings_by_module(self): warnings.catch_warnings(record=True) as wlog): warnings.simplefilter('error') warnings.filterwarnings('always', module=module_re) - import test.test_import.data.syntax_warnings - self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) - filename = test.test_import.data.syntax_warnings.__file__ - for wm in wlog: - self.assertEqual(wm.filename, filename) - self.assertIs(wm.category, SyntaxWarning) - - module_re = r'syntax_warnings\z' - unload('test.test_import.data.syntax_warnings') - with (os_helper.temp_dir() as tmpdir, - temporary_pycache_prefix(tmpdir), - warnings.catch_warnings(record=True) as wlog): - warnings.simplefilter('error') - warnings.filterwarnings('always', module=module_re) + warnings.filterwarnings('error', module='syntax_warnings') import test.test_import.data.syntax_warnings self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) filename = test.test_import.data.syntax_warnings.__file__ diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index a2a07c04f58ef2..cc76b72b9639eb 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -20,9 +20,11 @@ requires_subprocess, verbose, ) +from test import support from test.support.import_helper import forget, make_legacy_pyc, unload from test.support.os_helper import create_empty_file, temp_dir, FakePath from test.support.script_helper import make_script, make_zip_script +from test.test_importlib.util import temporary_pycache_prefix import runpy @@ -763,6 +765,47 @@ def test_encoding(self): result = run_path(filename) self.assertEqual(result['s'], "non-ASCII: h\xe9") + def test_run_module_filter_syntax_warnings_by_module(self): + module_re = r'test\.test_import\.data\.syntax_warnings\z' + with (temp_dir() as tmpdir, + temporary_pycache_prefix(tmpdir), + warnings.catch_warnings(record=True) as wlog): + warnings.simplefilter('error') + warnings.filterwarnings('always', module=module_re) + warnings.filterwarnings('error', module='syntax_warnings') + ns = run_module('test.test_import.data.syntax_warnings') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + filename = ns['__file__'] + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + + def test_run_path_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'\z') + warnings.filterwarnings('error', module='test') + warnings.filterwarnings('error', module='syntax_warnings') + warnings.filterwarnings('error', + module=r'test\.test_import\.data\.syntax_warnings') + run_path(filename) + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.script\z') + warnings.filterwarnings('error', module='') + warnings.filterwarnings('error', module='test') + warnings.filterwarnings('error', module='syntax_warnings') + warnings.filterwarnings('error', + module=r'test\.test_import\.data\.syntax_warnings') + run_path(filename, run_name='package.script') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + @force_not_colorized_test_class class TestExit(unittest.TestCase): diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index ef2c00e04b820c..094ab8f573e7ba 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -601,6 +601,16 @@ def test_filter_syntax_warnings_by_module(self): self.assertEqual(wm.filename, filename) self.assertIs(wm.category, SyntaxWarning) + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.module\z') + warnings.filterwarnings('error', module=module_re) + symtable.symtable(source, filename, 'exec', module='package.module') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + class ComprehensionTests(unittest.TestCase): def get_identifiers_recursive(self, st, res): diff --git a/Lib/test/test_zipimport_support.py b/Lib/test/test_zipimport_support.py index ae8a8c99762313..2b28f46149b4ff 100644 --- a/Lib/test/test_zipimport_support.py +++ b/Lib/test/test_zipimport_support.py @@ -13,9 +13,12 @@ import inspect import linecache import unittest +import warnings +from test import support from test.support import os_helper from test.support.script_helper import (spawn_python, kill_python, assert_python_ok, make_script, make_zip_script) +from test.support import import_helper verbose = test.support.verbose @@ -236,6 +239,26 @@ def f(): # bdb/pdb applies normcase to its filename before displaying self.assertIn(os.path.normcase(run_name.encode('utf-8')), data) + def test_import_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with (os_helper.temp_dir() as tmpdir, + import_helper.DirsOnSysPath()): + zip_name, _ = make_zip_script(tmpdir, "test_zip", + filename, 'test_pkg/test_mod.py') + sys.path.insert(0, zip_name) + import_helper.unload('test_pkg.test_mod') + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'test_pkg\.test_mod\z') + warnings.filterwarnings('error', module='test_mod') + import test_pkg.test_mod + self.assertEqual(sorted(wm.lineno for wm in wlog), + sorted([4, 7, 10, 13, 14, 21]*2)) + filename = test_pkg.test_mod.__file__ + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + def tearDownModule(): test.support.reap_children() diff --git a/Lib/zipimport.py b/Lib/zipimport.py index 340a7e07112504..19279d1c2bea36 100644 --- a/Lib/zipimport.py +++ b/Lib/zipimport.py @@ -742,9 +742,9 @@ def _normalize_line_endings(source): # Given a string buffer containing Python source code, compile it # and return a code object. -def _compile_source(pathname, source): +def _compile_source(pathname, source, module): source = _normalize_line_endings(source) - return compile(source, pathname, 'exec', dont_inherit=True) + return compile(source, pathname, 'exec', dont_inherit=True, module=module) # Convert the date/time values found in the Zip archive to a value # that's compatible with the time stamp stored in .pyc files. @@ -815,7 +815,7 @@ def _get_module_code(self, fullname): except ImportError as exc: import_error = exc else: - code = _compile_source(modpath, data) + code = _compile_source(modpath, data, fullname) if code is None: # bad magic number or non-matching mtime # in byte code, try next diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst new file mode 100644 index 00000000000000..96226a7c525e80 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst @@ -0,0 +1,6 @@ +Many functions related to compiling or parsing Python code, such as +:func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, and +:func:`importlib.abc.InspectLoader.source_to_code` now allow to specify +the module name. +It is needed to unambiguous :ref:`filter ` syntax warnings +by module name. diff --git a/Modules/clinic/symtablemodule.c.h b/Modules/clinic/symtablemodule.c.h index bd55d77c5409e9..65352593f94802 100644 --- a/Modules/clinic/symtablemodule.c.h +++ b/Modules/clinic/symtablemodule.c.h @@ -2,30 +2,67 @@ preserve [clinic start generated code]*/ -#include "pycore_modsupport.h" // _PyArg_CheckPositional() +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_symtable_symtable__doc__, -"symtable($module, source, filename, startstr, /)\n" +"symtable($module, source, filename, startstr, /, *, module=None)\n" "--\n" "\n" "Return symbol and scope dictionaries used internally by compiler."); #define _SYMTABLE_SYMTABLE_METHODDEF \ - {"symtable", _PyCFunction_CAST(_symtable_symtable), METH_FASTCALL, _symtable_symtable__doc__}, + {"symtable", _PyCFunction_CAST(_symtable_symtable), METH_FASTCALL|METH_KEYWORDS, _symtable_symtable__doc__}, static PyObject * _symtable_symtable_impl(PyObject *module, PyObject *source, - PyObject *filename, const char *startstr); + PyObject *filename, const char *startstr, + PyObject *modname); static PyObject * -_symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +_symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(module), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "", "", "module", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "symtable", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; PyObject *source; PyObject *filename = NULL; const char *startstr; + PyObject *modname = Py_None; - if (!_PyArg_CheckPositional("symtable", nargs, 3, 3)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { goto exit; } source = args[0]; @@ -45,7 +82,12 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = _symtable_symtable_impl(module, source, filename, startstr); + if (!noptargs) { + goto skip_optional_kwonly; + } + modname = args[3]; +skip_optional_kwonly: + return_value = _symtable_symtable_impl(module, source, filename, startstr, modname); exit: /* Cleanup for filename */ @@ -53,4 +95,4 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -/*[clinic end generated code: output=7a8545d9a1efe837 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0137be60c487c841 input=a9049054013a1b77]*/ diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index d353f406831ecd..a24927a9db64db 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -16,14 +16,17 @@ _symtable.symtable filename: unicode_fs_decoded startstr: str / + * + module as modname: object = None Return symbol and scope dictionaries used internally by compiler. [clinic start generated code]*/ static PyObject * _symtable_symtable_impl(PyObject *module, PyObject *source, - PyObject *filename, const char *startstr) -/*[clinic end generated code: output=59eb0d5fc7285ac4 input=436ffff90d02e4f6]*/ + PyObject *filename, const char *startstr, + PyObject *modname) +/*[clinic end generated code: output=235ec5a87a9ce178 input=fbf9adaa33c7070d]*/ { struct symtable *st; PyObject *t; @@ -50,7 +53,17 @@ _symtable_symtable_impl(PyObject *module, PyObject *source, Py_XDECREF(source_copy); return NULL; } - st = _Py_SymtableStringObjectFlags(str, filename, start, &cf); + if (modname == Py_None) { + modname = NULL; + } + else if (!PyUnicode_Check(modname)) { + PyErr_Format(PyExc_TypeError, + "symtable() argument 'module' must be str or None, not %T", + modname); + Py_XDECREF(source_copy); + return NULL; + } + st = _Py_SymtableStringObjectFlags(str, filename, start, &cf, modname); Py_XDECREF(source_copy); if (st == NULL) { return NULL; diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 2de9004fe084f2..3663dc3eb7f9f6 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -43,6 +43,7 @@ _PyTokenizer_tok_new(void) tok->encoding = NULL; tok->cont_line = 0; tok->filename = NULL; + tok->module = NULL; tok->decoding_readline = NULL; tok->decoding_buffer = NULL; tok->readline = NULL; @@ -91,6 +92,7 @@ _PyTokenizer_Free(struct tok_state *tok) Py_XDECREF(tok->decoding_buffer); Py_XDECREF(tok->readline); Py_XDECREF(tok->filename); + Py_XDECREF(tok->module); if ((tok->readline != NULL || tok->fp != NULL ) && tok->buf != NULL) { PyMem_Free(tok->buf); } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 877127125a7652..9cd196a114c7cb 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -102,6 +102,7 @@ struct tok_state { int parenlinenostack[MAXLEVEL]; int parencolstack[MAXLEVEL]; PyObject *filename; + PyObject *module; /* Stuff for checking on different tab sizes */ int altindstack[MAXINDENT]; /* Stack of alternate indents */ /* Stuff for PEP 0263 */ diff --git a/Parser/peg_api.c b/Parser/peg_api.c index d4acc3e4935d10..e30ca0453bd3e1 100644 --- a/Parser/peg_api.c +++ b/Parser/peg_api.c @@ -4,13 +4,15 @@ mod_ty _PyParser_ASTFromString(const char *str, PyObject* filename, int mode, - PyCompilerFlags *flags, PyArena *arena) + PyCompilerFlags *flags, PyArena *arena, + PyObject *module) { if (PySys_Audit("compile", "yO", str, filename) < 0) { return NULL; } - mod_ty result = _PyPegen_run_parser_from_string(str, mode, filename, flags, arena); + mod_ty result = _PyPegen_run_parser_from_string(str, mode, filename, flags, + arena, module); return result; } diff --git a/Parser/pegen.c b/Parser/pegen.c index 70493031656028..a38e973b3f64c6 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1010,6 +1010,11 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena // From here on we need to clean up even if there's an error mod_ty result = NULL; + tok->module = PyUnicode_FromString("__main__"); + if (tok->module == NULL) { + goto error; + } + int parser_flags = compute_parser_flags(flags); Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION, errcode, NULL, arena); @@ -1036,7 +1041,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena mod_ty _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filename_ob, - PyCompilerFlags *flags, PyArena *arena) + PyCompilerFlags *flags, PyArena *arena, PyObject *module) { int exec_input = start_rule == Py_file_input; @@ -1054,6 +1059,7 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen } // This transfers the ownership to the tokenizer tok->filename = Py_NewRef(filename_ob); + tok->module = Py_XNewRef(module); // We need to clear up from here on mod_ty result = NULL; diff --git a/Parser/pegen.h b/Parser/pegen.h index 6b49b3537a04b2..b8f887608b104e 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -378,7 +378,7 @@ mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char const char *, const char *, PyCompilerFlags *, int *, PyObject **, PyArena *); void *_PyPegen_run_parser(Parser *); -mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *); +mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *, PyObject *); asdl_stmt_seq *_PyPegen_interactive_exit(Parser *); // Generated function in parse.c - function definition in python.gram diff --git a/Parser/string_parser.c b/Parser/string_parser.c index ebe68989d1af58..b164dfbc81a933 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -88,7 +88,7 @@ warn_invalid_escape_sequence(Parser *p, const char* buffer, const char *first_in } if (PyErr_WarnExplicitObject(category, msg, p->tok->filename, - lineno, NULL, NULL) < 0) { + lineno, p->tok->module, NULL) < 0) { if (PyErr_ExceptionMatches(category)) { /* Replace the Syntax/DeprecationWarning exception with a SyntaxError to get a more accurate error report */ diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index e5e2eed2d34aee..a03531a744136d 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -127,7 +127,7 @@ _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_inval } if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, tok->filename, - tok->lineno, NULL, NULL) < 0) { + tok->lineno, tok->module, NULL) < 0) { Py_DECREF(msg); if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { @@ -166,7 +166,7 @@ _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char * } if (PyErr_WarnExplicitObject(category, errmsg, tok->filename, - tok->lineno, NULL, NULL) < 0) { + tok->lineno, tok->module, NULL) < 0) { if (PyErr_ExceptionMatches(category)) { /* Replace the DeprecationWarning exception with a SyntaxError to get a more accurate error report */ diff --git a/Programs/_freeze_module.py b/Programs/_freeze_module.py index ba638eef6c4cd6..62274e4aa9ce11 100644 --- a/Programs/_freeze_module.py +++ b/Programs/_freeze_module.py @@ -23,7 +23,7 @@ def read_text(inpath: str) -> bytes: def compile_and_marshal(name: str, text: bytes) -> bytes: filename = f"" # exec == Py_file_input - code = compile(text, filename, "exec", optimize=0, dont_inherit=True) + code = compile(text, filename, "exec", optimize=0, dont_inherit=True, module=name) return marshal.dumps(code) diff --git a/Programs/freeze_test_frozenmain.py b/Programs/freeze_test_frozenmain.py index 848fc31b3d6f44..1a986bbac2afc7 100644 --- a/Programs/freeze_test_frozenmain.py +++ b/Programs/freeze_test_frozenmain.py @@ -24,7 +24,7 @@ def dump(fp, filename, name): with tokenize.open(filename) as source_fp: source = source_fp.read() - code = compile(source, code_filename, 'exec') + code = compile(source, code_filename, 'exec', module=name) data = marshal.dumps(code) writecode(fp, name, data) diff --git a/Python/ast_preprocess.c b/Python/ast_preprocess.c index fe6fd9479d1531..d45435257cc8ac 100644 --- a/Python/ast_preprocess.c +++ b/Python/ast_preprocess.c @@ -16,6 +16,7 @@ typedef struct { typedef struct { PyObject *filename; + PyObject *module; int optimize; int ff_features; int syntax_check_only; @@ -71,7 +72,8 @@ control_flow_in_finally_warning(const char *kw, stmt_ty n, _PyASTPreprocessState } int ret = _PyErr_EmitSyntaxWarning(msg, state->filename, n->lineno, n->col_offset + 1, n->end_lineno, - n->end_col_offset + 1); + n->end_col_offset + 1, + state->module); Py_DECREF(msg); return ret < 0 ? 0 : 1; } @@ -969,11 +971,13 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTPreprocessState *st int _PyAST_Preprocess(mod_ty mod, PyArena *arena, PyObject *filename, int optimize, - int ff_features, int syntax_check_only, int enable_warnings) + int ff_features, int syntax_check_only, int enable_warnings, + PyObject *module) { _PyASTPreprocessState state; memset(&state, 0, sizeof(_PyASTPreprocessState)); state.filename = filename; + state.module = module; state.optimize = optimize; state.ff_features = ff_features; state.syntax_check_only = syntax_check_only; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index f6fadd936bb8ff..c2d780ac9b9270 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -751,6 +751,7 @@ compile as builtin_compile dont_inherit: bool = False optimize: int = -1 * + module as modname: object = None _feature_version as feature_version: int = -1 Compile source into a code object that can be executed by exec() or eval(). @@ -770,8 +771,8 @@ in addition to any features explicitly specified. static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, - int optimize, int feature_version) -/*[clinic end generated code: output=b0c09c84f116d3d7 input=8f0069edbdac381b]*/ + int optimize, PyObject *modname, int feature_version) +/*[clinic end generated code: output=9a0dce1945917a86 input=ddeae1e0253459dc]*/ { PyObject *source_copy; const char *str; @@ -800,6 +801,15 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, "compile(): invalid optimize value"); goto error; } + if (modname == Py_None) { + modname = NULL; + } + else if (!PyUnicode_Check(modname)) { + PyErr_Format(PyExc_TypeError, + "compile() argument 'module' must be str or None, not %T", + modname); + goto error; + } if (!dont_inherit) { PyEval_MergeCompilerFlags(&cf); @@ -845,8 +855,9 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, goto error; } int syntax_check_only = ((flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST); /* unoptiomized AST */ - if (_PyCompile_AstPreprocess(mod, filename, &cf, optimize, - arena, syntax_check_only) < 0) { + if (_PyCompile_AstPreprocess(mod, filename, &cf, optimize, arena, + syntax_check_only, modname) < 0) + { _PyArena_Free(arena); goto error; } @@ -859,7 +870,7 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, goto error; } result = (PyObject*)_PyAST_Compile(mod, filename, - &cf, optimize, arena); + &cf, optimize, arena, modname); } _PyArena_Free(arena); goto finally; @@ -877,7 +888,9 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, tstate->suppress_co_const_immortalization++; #endif - result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize); + result = _Py_CompileStringObjectWithModule(str, filename, + start[compile_mode], &cf, + optimize, modname); #ifdef Py_GIL_DISABLED tstate->suppress_co_const_immortalization--; diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index adb82f45c25b5d..f08e5847abe32a 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -238,7 +238,8 @@ PyDoc_STRVAR(builtin_chr__doc__, PyDoc_STRVAR(builtin_compile__doc__, "compile($module, /, source, filename, mode, flags=0,\n" -" dont_inherit=False, optimize=-1, *, _feature_version=-1)\n" +" dont_inherit=False, optimize=-1, *, module=None,\n" +" _feature_version=-1)\n" "--\n" "\n" "Compile source into a code object that can be executed by exec() or eval().\n" @@ -260,7 +261,7 @@ PyDoc_STRVAR(builtin_compile__doc__, static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, - int optimize, int feature_version); + int optimize, PyObject *modname, int feature_version); static PyObject * builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -268,7 +269,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 7 + #define NUM_KEYWORDS 8 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -277,7 +278,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(source), &_Py_ID(filename), &_Py_ID(mode), &_Py_ID(flags), &_Py_ID(dont_inherit), &_Py_ID(optimize), &_Py_ID(_feature_version), }, + .ob_item = { &_Py_ID(source), &_Py_ID(filename), &_Py_ID(mode), &_Py_ID(flags), &_Py_ID(dont_inherit), &_Py_ID(optimize), &_Py_ID(module), &_Py_ID(_feature_version), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -286,14 +287,14 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"source", "filename", "mode", "flags", "dont_inherit", "optimize", "_feature_version", NULL}; + static const char * const _keywords[] = {"source", "filename", "mode", "flags", "dont_inherit", "optimize", "module", "_feature_version", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "compile", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[7]; + PyObject *argsbuf[8]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; PyObject *source; PyObject *filename = NULL; @@ -301,6 +302,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj int flags = 0; int dont_inherit = 0; int optimize = -1; + PyObject *modname = Py_None; int feature_version = -1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, @@ -359,12 +361,18 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (!noptargs) { goto skip_optional_kwonly; } - feature_version = PyLong_AsInt(args[6]); + if (args[6]) { + modname = args[6]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + feature_version = PyLong_AsInt(args[7]); if (feature_version == -1 && PyErr_Occurred()) { goto exit; } skip_optional_kwonly: - return_value = builtin_compile_impl(module, source, filename, mode, flags, dont_inherit, optimize, feature_version); + return_value = builtin_compile_impl(module, source, filename, mode, flags, dont_inherit, optimize, modname, feature_version); exit: /* Cleanup for filename */ @@ -1277,4 +1285,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=7eada753dc2e046f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=06500bcc9a341e68 input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index e2f1c7e8eb5bce..6951c98500dfec 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -104,11 +104,13 @@ typedef struct _PyCompiler { * (including instructions for nested code objects) */ int c_disable_warning; + PyObject *c_module; } compiler; static int compiler_setup(compiler *c, mod_ty mod, PyObject *filename, - PyCompilerFlags *flags, int optimize, PyArena *arena) + PyCompilerFlags *flags, int optimize, PyArena *arena, + PyObject *module) { PyCompilerFlags local_flags = _PyCompilerFlags_INIT; @@ -126,6 +128,7 @@ compiler_setup(compiler *c, mod_ty mod, PyObject *filename, if (!_PyFuture_FromAST(mod, filename, &c->c_future)) { return ERROR; } + c->c_module = Py_XNewRef(module); if (!flags) { flags = &local_flags; } @@ -136,7 +139,9 @@ compiler_setup(compiler *c, mod_ty mod, PyObject *filename, c->c_optimize = (optimize == -1) ? _Py_GetConfig()->optimization_level : optimize; c->c_save_nested_seqs = false; - if (!_PyAST_Preprocess(mod, arena, filename, c->c_optimize, merged, 0, 1)) { + if (!_PyAST_Preprocess(mod, arena, filename, c->c_optimize, merged, + 0, 1, module)) + { return ERROR; } c->c_st = _PySymtable_Build(mod, filename, &c->c_future); @@ -156,6 +161,7 @@ compiler_free(compiler *c) _PySymtable_Free(c->c_st); } Py_XDECREF(c->c_filename); + Py_XDECREF(c->c_module); Py_XDECREF(c->c_const_cache); Py_XDECREF(c->c_stack); PyMem_Free(c); @@ -163,13 +169,13 @@ compiler_free(compiler *c) static compiler* new_compiler(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, - int optimize, PyArena *arena) + int optimize, PyArena *arena, PyObject *module) { compiler *c = PyMem_Calloc(1, sizeof(compiler)); if (c == NULL) { return NULL; } - if (compiler_setup(c, mod, filename, pflags, optimize, arena) < 0) { + if (compiler_setup(c, mod, filename, pflags, optimize, arena, module) < 0) { compiler_free(c); return NULL; } @@ -1221,7 +1227,8 @@ _PyCompile_Warn(compiler *c, location loc, const char *format, ...) return ERROR; } int ret = _PyErr_EmitSyntaxWarning(msg, c->c_filename, loc.lineno, loc.col_offset + 1, - loc.end_lineno, loc.end_col_offset + 1); + loc.end_lineno, loc.end_col_offset + 1, + c->c_module); Py_DECREF(msg); return ret; } @@ -1476,10 +1483,10 @@ _PyCompile_OptimizeAndAssemble(compiler *c, int addNone) PyCodeObject * _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, - int optimize, PyArena *arena) + int optimize, PyArena *arena, PyObject *module) { assert(!PyErr_Occurred()); - compiler *c = new_compiler(mod, filename, pflags, optimize, arena); + compiler *c = new_compiler(mod, filename, pflags, optimize, arena, module); if (c == NULL) { return NULL; } @@ -1492,7 +1499,8 @@ _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, int _PyCompile_AstPreprocess(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, - int optimize, PyArena *arena, int no_const_folding) + int optimize, PyArena *arena, int no_const_folding, + PyObject *module) { _PyFutureFeatures future; if (!_PyFuture_FromAST(mod, filename, &future)) { @@ -1502,7 +1510,9 @@ _PyCompile_AstPreprocess(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, if (optimize == -1) { optimize = _Py_GetConfig()->optimization_level; } - if (!_PyAST_Preprocess(mod, arena, filename, optimize, flags, no_const_folding, 0)) { + if (!_PyAST_Preprocess(mod, arena, filename, optimize, flags, + no_const_folding, 0, module)) + { return -1; } return 0; @@ -1627,7 +1637,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, return NULL; } - compiler *c = new_compiler(mod, filename, pflags, optimize, arena); + compiler *c = new_compiler(mod, filename, pflags, optimize, arena, NULL); if (c == NULL) { _PyArena_Free(arena); return NULL; diff --git a/Python/errors.c b/Python/errors.c index 9fe95cec0ab794..5c6ac48371a0ff 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1960,10 +1960,11 @@ _PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_o */ int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, - int end_lineno, int end_col_offset) + int end_lineno, int end_col_offset, + PyObject *module) { - if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, - filename, lineno, NULL, NULL) < 0) + if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, filename, lineno, + module, NULL) < 0) { if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { /* Replace the SyntaxWarning exception with a SyntaxError diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 45211e1b075042..49ce0a97d4742f 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1252,12 +1252,19 @@ _PyRun_StringFlagsWithName(const char *str, PyObject* name, int start, } else { name = &_Py_STR(anon_string); } + PyObject *module = NULL; + if (globals && PyDict_GetItemStringRef(globals, "__name__", &module) < 0) { + goto done; + } - mod = _PyParser_ASTFromString(str, name, start, flags, arena); + mod = _PyParser_ASTFromString(str, name, start, flags, arena, module); + Py_XDECREF(module); - if (mod != NULL) { + if (mod != NULL) { ret = run_mod(mod, name, globals, locals, flags, arena, source, generate_new_source); } + +done: Py_XDECREF(source); _PyArena_Free(arena); return ret; @@ -1407,8 +1414,17 @@ run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, return NULL; } } + PyObject *module = NULL; + if (globals && PyDict_GetItemStringRef(globals, "__name__", &module) < 0) { + if (interactive_src) { + Py_DECREF(interactive_filename); + } + return NULL; + } - PyCodeObject *co = _PyAST_Compile(mod, interactive_filename, flags, -1, arena); + PyCodeObject *co = _PyAST_Compile(mod, interactive_filename, flags, -1, + arena, module); + Py_XDECREF(module); if (co == NULL) { if (interactive_src) { Py_DECREF(interactive_filename); @@ -1507,6 +1523,14 @@ run_pyc_file(FILE *fp, PyObject *globals, PyObject *locals, PyObject * Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) +{ + return _Py_CompileStringObjectWithModule(str, filename, start, + flags, optimize, NULL); +} + +PyObject * +_Py_CompileStringObjectWithModule(const char *str, PyObject *filename, int start, + PyCompilerFlags *flags, int optimize, PyObject *module) { PyCodeObject *co; mod_ty mod; @@ -1514,14 +1538,16 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, if (arena == NULL) return NULL; - mod = _PyParser_ASTFromString(str, filename, start, flags, arena); + mod = _PyParser_ASTFromString(str, filename, start, flags, arena, module); if (mod == NULL) { _PyArena_Free(arena); return NULL; } if (flags && (flags->cf_flags & PyCF_ONLY_AST)) { int syntax_check_only = ((flags->cf_flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST); /* unoptiomized AST */ - if (_PyCompile_AstPreprocess(mod, filename, flags, optimize, arena, syntax_check_only) < 0) { + if (_PyCompile_AstPreprocess(mod, filename, flags, optimize, arena, + syntax_check_only, module) < 0) + { _PyArena_Free(arena); return NULL; } @@ -1529,7 +1555,7 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, _PyArena_Free(arena); return result; } - co = _PyAST_Compile(mod, filename, flags, optimize, arena); + co = _PyAST_Compile(mod, filename, flags, optimize, arena, module); _PyArena_Free(arena); return (PyObject *)co; } diff --git a/Python/symtable.c b/Python/symtable.c index bcd7365f8e1f14..29cf9190a4e95b 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -3137,7 +3137,7 @@ symtable_raise_if_not_coroutine(struct symtable *st, const char *msg, _Py_Source struct symtable * _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, - int start, PyCompilerFlags *flags) + int start, PyCompilerFlags *flags, PyObject *module) { struct symtable *st; mod_ty mod; @@ -3147,7 +3147,7 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, if (arena == NULL) return NULL; - mod = _PyParser_ASTFromString(str, filename, start, flags, arena); + mod = _PyParser_ASTFromString(str, filename, start, flags, arena, module); if (mod == NULL) { _PyArena_Free(arena); return NULL; diff --git a/Tools/peg_generator/peg_extension/peg_extension.c b/Tools/peg_generator/peg_extension/peg_extension.c index 1587d53d59472e..2fec5b0512940f 100644 --- a/Tools/peg_generator/peg_extension/peg_extension.c +++ b/Tools/peg_generator/peg_extension/peg_extension.c @@ -8,7 +8,7 @@ _build_return_object(mod_ty module, int mode, PyObject *filename_ob, PyArena *ar PyObject *result = NULL; if (mode == 2) { - result = (PyObject *)_PyAST_Compile(module, filename_ob, NULL, -1, arena); + result = (PyObject *)_PyAST_Compile(module, filename_ob, NULL, -1, arena, NULL); } else if (mode == 1) { result = PyAST_mod2obj(module); } else { @@ -93,7 +93,7 @@ parse_string(PyObject *self, PyObject *args, PyObject *kwds) PyCompilerFlags flags = _PyCompilerFlags_INIT; mod_ty res = _PyPegen_run_parser_from_string(the_string, Py_file_input, filename_ob, - &flags, arena); + &flags, arena, NULL); if (res == NULL) { goto error; } From 2fbd39666663cb5ca1c0e3021ce2e7bc72331020 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Nov 2025 13:37:01 +0200 Subject: [PATCH 52/58] gh-140601: Refactor ElementTree.iterparse() tests (GH-141499) Split existing tests on smaller methods and move them to separate class. Rename variable "content" to "it". Use BytesIO instead of StringIO. Add few more tests. --- Lib/test/test_xml_etree.py | 430 ++++++++++++++++++++----------------- 1 file changed, 228 insertions(+), 202 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index f65baa0cfae2ad..25c084c8b9c9eb 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -574,208 +574,6 @@ def test_parseliteral(self): self.assertEqual(len(ids), 1) self.assertEqual(ids["body"].tag, 'body') - def test_iterparse(self): - # Test iterparse interface. - - iterparse = ET.iterparse - - context = iterparse(SIMPLE_XMLFILE) - self.assertIsNone(context.root) - action, elem = next(context) - self.assertIsNone(context.root) - self.assertEqual((action, elem.tag), ('end', 'element')) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', 'element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - self.assertEqual(context.root.tag, 'root') - - context = iterparse(SIMPLE_NS_XMLFILE) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', '{namespace}element'), - ('end', '{namespace}element'), - ('end', '{namespace}empty-element'), - ('end', '{namespace}root'), - ]) - - with open(SIMPLE_XMLFILE, 'rb') as source: - context = iterparse(source) - action, elem = next(context) - self.assertEqual((action, elem.tag), ('end', 'element')) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', 'element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - self.assertEqual(context.root.tag, 'root') - - events = () - context = iterparse(SIMPLE_XMLFILE, events) - self.assertEqual([(action, elem.tag) for action, elem in context], []) - - events = () - context = iterparse(SIMPLE_XMLFILE, events=events) - self.assertEqual([(action, elem.tag) for action, elem in context], []) - - events = ("start", "end") - context = iterparse(SIMPLE_XMLFILE, events) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('start', 'root'), - ('start', 'element'), - ('end', 'element'), - ('start', 'element'), - ('end', 'element'), - ('start', 'empty-element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - - events = ("start", "end", "start-ns", "end-ns") - context = iterparse(SIMPLE_NS_XMLFILE, events) - self.assertEqual([(action, elem.tag) if action in ("start", "end") - else (action, elem) - for action, elem in context], [ - ('start-ns', ('', 'namespace')), - ('start', '{namespace}root'), - ('start', '{namespace}element'), - ('end', '{namespace}element'), - ('start', '{namespace}element'), - ('end', '{namespace}element'), - ('start', '{namespace}empty-element'), - ('end', '{namespace}empty-element'), - ('end', '{namespace}root'), - ('end-ns', None), - ]) - - events = ('start-ns', 'end-ns') - context = iterparse(io.StringIO(r""), events) - res = [action for action, elem in context] - self.assertEqual(res, ['start-ns', 'end-ns']) - - events = ("start", "end", "bogus") - with open(SIMPLE_XMLFILE, "rb") as f: - with self.assertRaises(ValueError) as cm: - iterparse(f, events) - self.assertFalse(f.closed) - self.assertEqual(str(cm.exception), "unknown event 'bogus'") - - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(ValueError) as cm: - iterparse(SIMPLE_XMLFILE, events) - self.assertEqual(str(cm.exception), "unknown event 'bogus'") - del cm - - source = io.BytesIO( - b"\n" - b"text\n") - events = ("start-ns",) - context = iterparse(source, events) - self.assertEqual([(action, elem) for action, elem in context], [ - ('start-ns', ('', 'http://\xe9ffbot.org/ns')), - ('start-ns', ('cl\xe9', 'http://effbot.org/ns')), - ]) - - source = io.StringIO("junk") - it = iterparse(source) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'document')) - with self.assertRaises(ET.ParseError) as cm: - next(it) - self.assertEqual(str(cm.exception), - 'junk after document element: line 1, column 12') - - self.addCleanup(os_helper.unlink, TESTFN) - with open(TESTFN, "wb") as f: - f.write(b"junk") - it = iterparse(TESTFN) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'document')) - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(ET.ParseError) as cm: - next(it) - self.assertEqual(str(cm.exception), - 'junk after document element: line 1, column 12') - del cm, it - - # Not exhausting the iterator still closes the resource (bpo-43292) - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - del it - - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - it.close() - del it - - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'element')) - del it, elem - - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - action, elem = next(it) - it.close() - self.assertEqual((action, elem.tag), ('end', 'element')) - del it, elem - - with self.assertRaises(FileNotFoundError): - iterparse("nonexistent") - - def test_iterparse_close(self): - iterparse = ET.iterparse - - it = iterparse(SIMPLE_XMLFILE) - it.close() - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - with open(SIMPLE_XMLFILE, 'rb') as source: - it = iterparse(source) - it.close() - self.assertFalse(source.closed) - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - it = iterparse(SIMPLE_XMLFILE) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'element')) - it.close() - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - with open(SIMPLE_XMLFILE, 'rb') as source: - it = iterparse(source) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'element')) - it.close() - self.assertFalse(source.closed) - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - it = iterparse(SIMPLE_XMLFILE) - list(it) - it.close() - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - with open(SIMPLE_XMLFILE, 'rb') as source: - it = iterparse(source) - list(it) - it.close() - self.assertFalse(source.closed) - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - def test_writefile(self): elem = ET.Element("tag") elem.text = "text" @@ -1499,6 +1297,234 @@ def test_attlist_default(self): {'{http://www.w3.org/XML/1998/namespace}lang': 'eng'}) +class IterparseTest(unittest.TestCase): + # Test iterparse interface. + + def test_basic(self): + iterparse = ET.iterparse + + it = iterparse(SIMPLE_XMLFILE) + self.assertIsNone(it.root) + action, elem = next(it) + self.assertIsNone(it.root) + self.assertEqual((action, elem.tag), ('end', 'element')) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', 'element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertEqual(it.root.tag, 'root') + it.close() + + it = iterparse(SIMPLE_NS_XMLFILE) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', '{namespace}element'), + ('end', '{namespace}element'), + ('end', '{namespace}empty-element'), + ('end', '{namespace}root'), + ]) + it.close() + + def test_external_file(self): + with open(SIMPLE_XMLFILE, 'rb') as source: + it = ET.iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', 'element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertEqual(it.root.tag, 'root') + + def test_events(self): + iterparse = ET.iterparse + + events = () + it = iterparse(SIMPLE_XMLFILE, events) + self.assertEqual([(action, elem.tag) for action, elem in it], []) + it.close() + + events = () + it = iterparse(SIMPLE_XMLFILE, events=events) + self.assertEqual([(action, elem.tag) for action, elem in it], []) + it.close() + + events = ("start", "end") + it = iterparse(SIMPLE_XMLFILE, events) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('start', 'root'), + ('start', 'element'), + ('end', 'element'), + ('start', 'element'), + ('end', 'element'), + ('start', 'empty-element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + it.close() + + def test_namespace_events(self): + iterparse = ET.iterparse + + events = ("start", "end", "start-ns", "end-ns") + it = iterparse(SIMPLE_NS_XMLFILE, events) + self.assertEqual([(action, elem.tag) if action in ("start", "end") + else (action, elem) + for action, elem in it], [ + ('start-ns', ('', 'namespace')), + ('start', '{namespace}root'), + ('start', '{namespace}element'), + ('end', '{namespace}element'), + ('start', '{namespace}element'), + ('end', '{namespace}element'), + ('start', '{namespace}empty-element'), + ('end', '{namespace}empty-element'), + ('end', '{namespace}root'), + ('end-ns', None), + ]) + it.close() + + events = ('start-ns', 'end-ns') + it = iterparse(io.BytesIO(br""), events) + res = [action for action, elem in it] + self.assertEqual(res, ['start-ns', 'end-ns']) + it.close() + + def test_unknown_events(self): + iterparse = ET.iterparse + + events = ("start", "end", "bogus") + with open(SIMPLE_XMLFILE, "rb") as f: + with self.assertRaises(ValueError) as cm: + iterparse(f, events) + self.assertFalse(f.closed) + self.assertEqual(str(cm.exception), "unknown event 'bogus'") + + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(ValueError) as cm: + iterparse(SIMPLE_XMLFILE, events) + self.assertEqual(str(cm.exception), "unknown event 'bogus'") + del cm + gc_collect() + + def test_non_utf8(self): + source = io.BytesIO( + b"\n" + b"text\n") + events = ("start-ns",) + it = ET.iterparse(source, events) + self.assertEqual([(action, elem) for action, elem in it], [ + ('start-ns', ('', 'http://\xe9ffbot.org/ns')), + ('start-ns', ('cl\xe9', 'http://effbot.org/ns')), + ]) + + def test_parsing_error(self): + source = io.BytesIO(b"junk") + it = ET.iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'document')) + with self.assertRaises(ET.ParseError) as cm: + next(it) + self.assertEqual(str(cm.exception), + 'junk after document element: line 1, column 12') + + def test_nonexistent_file(self): + with self.assertRaises(FileNotFoundError): + ET.iterparse("nonexistent") + + def test_resource_warnings_not_exhausted(self): + # Not exhausting the iterator still closes the underlying file (bpo-43292) + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + del it + gc_collect() + + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + gc_collect() + + def test_resource_warnings_failed_iteration(self): + self.addCleanup(os_helper.unlink, TESTFN) + with open(TESTFN, "wb") as f: + f.write(b"junk") + + it = ET.iterparse(TESTFN) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'document')) + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(ET.ParseError) as cm: + next(it) + self.assertEqual(str(cm.exception), + 'junk after document element: line 1, column 12') + del cm, it + gc_collect() + + def test_resource_warnings_exhausted(self): + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + list(it) + del it + gc_collect() + + def test_close_not_exhausted(self): + iterparse = ET.iterparse + + it = iterparse(SIMPLE_XMLFILE) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + def test_close_exhausted(self): + iterparse = ET.iterparse + it = iterparse(SIMPLE_XMLFILE) + list(it) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + list(it) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + class XMLPullParserTest(unittest.TestCase): def _feed(self, parser, data, chunk_size=None, flush=False): From 1637c046fc5ad7f0a375fd17792ad7e49899d2a3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 2 Nov 2025 22:22:58 +0100 Subject: [PATCH 53/58] [WIP] PEP 814: Add built-in frozendict type * Basic tests * Bare minimum documentation * Support frozendict in marshal, pickle, json * Replace dict with frozendict in many stdlib modules Co-Authored-by: Donghee Na Co-Authored-by: Marco Sulla --- Doc/library/stdtypes.rst | 19 +- Include/cpython/dictobject.h | 15 +- Include/internal/pycore_dict.h | 9 + Include/internal/pycore_object.h | 3 +- Lib/_collections_abc.py | 1 + Lib/_compat_pickle.py | 2 + Lib/_opcode_metadata.py | 516 +++++++++--------- Lib/_pydatetime.py | 14 +- Lib/_pydecimal.py | 9 +- Lib/bdb.py | 4 +- Lib/dataclasses.py | 5 +- Lib/dis.py | 8 +- Lib/email/headerregistry.py | 4 +- Lib/enum.py | 4 +- Lib/functools.py | 4 +- Lib/gettext.py | 5 +- Lib/imaplib.py | 6 +- Lib/json/decoder.py | 8 +- Lib/json/encoder.py | 12 +- Lib/json/tool.py | 8 +- Lib/locale.py | 9 +- Lib/opcode.py | 46 +- Lib/optparse.py | 9 +- Lib/pickle.py | 33 ++ Lib/pickletools.py | 22 + Lib/platform.py | 8 +- Lib/plistlib.py | 2 +- Lib/pydoc_data/topics.py | 5 +- Lib/ssl.py | 3 +- Lib/stringprep.py | 4 +- Lib/symtable.py | 2 +- Lib/tarfile.py | 8 +- Lib/test/mapping_tests.py | 129 ++--- Lib/test/test_dict.py | 3 + Lib/test/test_doctest/test_doctest.py | 2 +- Lib/test/test_inspect/test_inspect.py | 3 +- Lib/test/test_pickletools.py | 2 +- Lib/token.py | 11 +- Lib/tomllib/_parser.py | 4 +- Lib/typing.py | 4 +- Modules/_json.c | 2 +- Modules/_pickle.c | 124 ++++- Modules/errnomodule.c | 22 +- Objects/dictobject.c | 258 ++++++++- Objects/moduleobject.c | 1 + Objects/typeobject.c | 4 +- Python/bltinmodule.c | 1 + Python/marshal.c | 22 +- .../cases_generator/py_metadata_generator.py | 18 +- 49 files changed, 935 insertions(+), 482 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 97e7e08364e0bd..d74345416f3644 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4914,8 +4914,8 @@ The constructors for both classes work the same: .. _typesmapping: -Mapping Types --- :class:`dict` -=============================== +Mapping Types --- :class:`dict`, :class:`frozendict` +==================================================== .. index:: pair: object; mapping @@ -4926,8 +4926,9 @@ Mapping Types --- :class:`dict` pair: built-in function; len A :term:`mapping` object maps :term:`hashable` values to arbitrary objects. -Mappings are mutable objects. There is currently only one standard mapping -type, the :dfn:`dictionary`. (For other containers see the built-in +Mappings are mutable objects. There is currently two standard mapping +types, the :dfn:`dictionary` and :class:`frozendict`. +(For other containers see the built-in :class:`list`, :class:`set`, and :class:`tuple` classes, and the :mod:`collections` module.) @@ -5199,6 +5200,15 @@ can be used interchangeably to index the same dictionary entry. .. versionchanged:: 3.8 Dictionaries are now reversible. +.. class:: frozendict(**kwargs) + frozendict(mapping, /, **kwargs) + frozendict(iterable, /, **kwargs) + + Return a new frozen dictionary initialized from an optional positional + argument and a possibly empty set of keyword arguments. + + .. versionadded:: next + .. seealso:: :class:`types.MappingProxyType` can be used to create a read-only view @@ -5532,6 +5542,7 @@ list is non-exhaustive. * :class:`list` * :class:`dict` * :class:`set` +* :class:`frozendict` * :class:`frozenset` * :class:`type` * :class:`asyncio.Future` diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index df9ec7050fca1a..17bbed74cda862 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -32,6 +32,16 @@ typedef struct { PyDictValues *ma_values; } PyDictObject; +// frozendict +PyAPI_DATA(PyTypeObject) PyFrozenDict_Type; +#define PyFrozenDict_Check(op) PyObject_TypeCheck((op), &PyFrozenDict_Type) +#define PyFrozenDict_CheckExact(op) Py_IS_TYPE((op), &PyFrozenDict_Type) + +#define _PyAnyDict_CheckExact(ob) \ + (PyDict_CheckExact(ob) || PyFrozenDict_CheckExact(ob)) +#define _PyAnyDict_Check(ob) \ + (PyDict_Check(ob) || PyFrozenDict_Check(ob)) + PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, Py_hash_t hash); // PyDict_GetItemStringRef() can be used instead @@ -52,7 +62,7 @@ PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *defa /* Get the number of items of a dictionary. */ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { PyDictObject *mp; - assert(PyDict_Check(op)); + assert(_PyAnyDict_Check(op)); mp = _Py_CAST(PyDictObject*, op); #ifdef Py_GIL_DISABLED return _Py_atomic_load_ssize_relaxed(&mp->ma_used); @@ -103,3 +113,6 @@ PyAPI_FUNC(int) PyDict_ClearWatcher(int watcher_id); // Mark given dictionary as "watched" (callback will be called if it is modified) PyAPI_FUNC(int) PyDict_Watch(int watcher_id, PyObject* dict); PyAPI_FUNC(int) PyDict_Unwatch(int watcher_id, PyObject* dict); + +// Create a frozendict. Create an empty dictionary if iterable is NULL. +PyAPI_FUNC(PyObject*) PyFrozenDict_New(PyObject *iterable); diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index b8fe360321d14b..757329d491cd4d 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -406,6 +406,15 @@ _Py_DECREF_BUILTINS(PyObject *op) } #endif +/* frozendict */ +typedef struct { + PyDictObject ob_base; + Py_hash_t ma_hash; +} PyFrozenDictObject; + +#define _PyFrozenDictObject_CAST(op) \ + (assert(PyFrozenDict_Check(op)), _Py_CAST(PyFrozenDictObject*, (op))) + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 980d6d7764bd2c..fb50acd62da5eb 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -863,8 +863,7 @@ static inline Py_hash_t _PyObject_HashFast(PyObject *op) { if (PyUnicode_CheckExact(op)) { - Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED( - _PyASCIIObject_CAST(op)->hash); + Py_hash_t hash = PyUnstable_Unicode_GET_CACHED_HASH(op); if (hash != -1) { return hash; } diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 60b471317ce97c..23cc6d8faae2da 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -823,6 +823,7 @@ def __eq__(self, other): __reversed__ = None +Mapping.register(frozendict) Mapping.register(mappingproxy) Mapping.register(framelocalsproxy) diff --git a/Lib/_compat_pickle.py b/Lib/_compat_pickle.py index a981326432429b..6f868c9890e7c0 100644 --- a/Lib/_compat_pickle.py +++ b/Lib/_compat_pickle.py @@ -183,6 +183,7 @@ 'StringIO': 'io', 'cStringIO': 'io', }) +IMPORT_MAPPING = frozendict(IMPORT_MAPPING) REVERSE_IMPORT_MAPPING.update({ '_bz2': 'bz2', @@ -198,6 +199,7 @@ ('UserDict', 'UserDict'): ('collections', 'UserDict'), ('socket', '_socketobject'): ('socket', 'SocketType'), }) +NAME_MAPPING = frozendict(NAME_MAPPING) REVERSE_NAME_MAPPING.update({ ('_functools', 'reduce'): ('__builtin__', 'reduce'), diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index f168d169a32948..c0bb805f69ceba 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -2,11 +2,11 @@ # from: # Python/bytecodes.c # Do not edit! -_specializations = { - "RESUME": [ +_specializations = frozendict( + RESUME= [ "RESUME_CHECK", ], - "TO_BOOL": [ + TO_BOOL= [ "TO_BOOL_ALWAYS_TRUE", "TO_BOOL_BOOL", "TO_BOOL_INT", @@ -14,7 +14,7 @@ "TO_BOOL_NONE", "TO_BOOL_STR", ], - "BINARY_OP": [ + BINARY_OP= [ "BINARY_OP_MULTIPLY_INT", "BINARY_OP_ADD_INT", "BINARY_OP_SUBTRACT_INT", @@ -31,32 +31,32 @@ "BINARY_OP_EXTEND", "BINARY_OP_INPLACE_ADD_UNICODE", ], - "STORE_SUBSCR": [ + STORE_SUBSCR= [ "STORE_SUBSCR_DICT", "STORE_SUBSCR_LIST_INT", ], - "SEND": [ + SEND= [ "SEND_GEN", ], - "UNPACK_SEQUENCE": [ + UNPACK_SEQUENCE= [ "UNPACK_SEQUENCE_TWO_TUPLE", "UNPACK_SEQUENCE_TUPLE", "UNPACK_SEQUENCE_LIST", ], - "STORE_ATTR": [ + STORE_ATTR= [ "STORE_ATTR_INSTANCE_VALUE", "STORE_ATTR_SLOT", "STORE_ATTR_WITH_HINT", ], - "LOAD_GLOBAL": [ + LOAD_GLOBAL= [ "LOAD_GLOBAL_MODULE", "LOAD_GLOBAL_BUILTIN", ], - "LOAD_SUPER_ATTR": [ + LOAD_SUPER_ATTR= [ "LOAD_SUPER_ATTR_ATTR", "LOAD_SUPER_ATTR_METHOD", ], - "LOAD_ATTR": [ + LOAD_ATTR= [ "LOAD_ATTR_INSTANCE_VALUE", "LOAD_ATTR_MODULE", "LOAD_ATTR_WITH_HINT", @@ -71,26 +71,26 @@ "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", ], - "COMPARE_OP": [ + COMPARE_OP= [ "COMPARE_OP_FLOAT", "COMPARE_OP_INT", "COMPARE_OP_STR", ], - "CONTAINS_OP": [ + CONTAINS_OP= [ "CONTAINS_OP_SET", "CONTAINS_OP_DICT", ], - "JUMP_BACKWARD": [ + JUMP_BACKWARD= [ "JUMP_BACKWARD_NO_JIT", "JUMP_BACKWARD_JIT", ], - "FOR_ITER": [ + FOR_ITER= [ "FOR_ITER_LIST", "FOR_ITER_TUPLE", "FOR_ITER_RANGE", "FOR_ITER_GEN", ], - "CALL": [ + CALL= [ "CALL_BOUND_METHOD_EXACT_ARGS", "CALL_PY_EXACT_ARGS", "CALL_TYPE_1", @@ -112,254 +112,254 @@ "CALL_BOUND_METHOD_GENERAL", "CALL_NON_PY_GENERAL", ], - "CALL_KW": [ + CALL_KW= [ "CALL_KW_BOUND_METHOD", "CALL_KW_PY", "CALL_KW_NON_PY", ], -} +) -_specialized_opmap = { - 'BINARY_OP_ADD_FLOAT': 129, - 'BINARY_OP_ADD_INT': 130, - 'BINARY_OP_ADD_UNICODE': 131, - 'BINARY_OP_EXTEND': 132, - 'BINARY_OP_INPLACE_ADD_UNICODE': 3, - 'BINARY_OP_MULTIPLY_FLOAT': 133, - 'BINARY_OP_MULTIPLY_INT': 134, - 'BINARY_OP_SUBSCR_DICT': 135, - 'BINARY_OP_SUBSCR_GETITEM': 136, - 'BINARY_OP_SUBSCR_LIST_INT': 137, - 'BINARY_OP_SUBSCR_LIST_SLICE': 138, - 'BINARY_OP_SUBSCR_STR_INT': 139, - 'BINARY_OP_SUBSCR_TUPLE_INT': 140, - 'BINARY_OP_SUBTRACT_FLOAT': 141, - 'BINARY_OP_SUBTRACT_INT': 142, - 'CALL_ALLOC_AND_ENTER_INIT': 143, - 'CALL_BOUND_METHOD_EXACT_ARGS': 144, - 'CALL_BOUND_METHOD_GENERAL': 145, - 'CALL_BUILTIN_CLASS': 146, - 'CALL_BUILTIN_FAST': 147, - 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 148, - 'CALL_BUILTIN_O': 149, - 'CALL_ISINSTANCE': 150, - 'CALL_KW_BOUND_METHOD': 151, - 'CALL_KW_NON_PY': 152, - 'CALL_KW_PY': 153, - 'CALL_LEN': 154, - 'CALL_LIST_APPEND': 155, - 'CALL_METHOD_DESCRIPTOR_FAST': 156, - 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 157, - 'CALL_METHOD_DESCRIPTOR_NOARGS': 158, - 'CALL_METHOD_DESCRIPTOR_O': 159, - 'CALL_NON_PY_GENERAL': 160, - 'CALL_PY_EXACT_ARGS': 161, - 'CALL_PY_GENERAL': 162, - 'CALL_STR_1': 163, - 'CALL_TUPLE_1': 164, - 'CALL_TYPE_1': 165, - 'COMPARE_OP_FLOAT': 166, - 'COMPARE_OP_INT': 167, - 'COMPARE_OP_STR': 168, - 'CONTAINS_OP_DICT': 169, - 'CONTAINS_OP_SET': 170, - 'FOR_ITER_GEN': 171, - 'FOR_ITER_LIST': 172, - 'FOR_ITER_RANGE': 173, - 'FOR_ITER_TUPLE': 174, - 'JUMP_BACKWARD_JIT': 175, - 'JUMP_BACKWARD_NO_JIT': 176, - 'LOAD_ATTR_CLASS': 177, - 'LOAD_ATTR_CLASS_WITH_METACLASS_CHECK': 178, - 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 179, - 'LOAD_ATTR_INSTANCE_VALUE': 180, - 'LOAD_ATTR_METHOD_LAZY_DICT': 181, - 'LOAD_ATTR_METHOD_NO_DICT': 182, - 'LOAD_ATTR_METHOD_WITH_VALUES': 183, - 'LOAD_ATTR_MODULE': 184, - 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 185, - 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 186, - 'LOAD_ATTR_PROPERTY': 187, - 'LOAD_ATTR_SLOT': 188, - 'LOAD_ATTR_WITH_HINT': 189, - 'LOAD_GLOBAL_BUILTIN': 190, - 'LOAD_GLOBAL_MODULE': 191, - 'LOAD_SUPER_ATTR_ATTR': 192, - 'LOAD_SUPER_ATTR_METHOD': 193, - 'RESUME_CHECK': 194, - 'SEND_GEN': 195, - 'STORE_ATTR_INSTANCE_VALUE': 196, - 'STORE_ATTR_SLOT': 197, - 'STORE_ATTR_WITH_HINT': 198, - 'STORE_SUBSCR_DICT': 199, - 'STORE_SUBSCR_LIST_INT': 200, - 'TO_BOOL_ALWAYS_TRUE': 201, - 'TO_BOOL_BOOL': 202, - 'TO_BOOL_INT': 203, - 'TO_BOOL_LIST': 204, - 'TO_BOOL_NONE': 205, - 'TO_BOOL_STR': 206, - 'UNPACK_SEQUENCE_LIST': 207, - 'UNPACK_SEQUENCE_TUPLE': 208, - 'UNPACK_SEQUENCE_TWO_TUPLE': 209, -} +_specialized_opmap = frozendict( + BINARY_OP_ADD_FLOAT= 129, + BINARY_OP_ADD_INT= 130, + BINARY_OP_ADD_UNICODE= 131, + BINARY_OP_EXTEND= 132, + BINARY_OP_INPLACE_ADD_UNICODE= 3, + BINARY_OP_MULTIPLY_FLOAT= 133, + BINARY_OP_MULTIPLY_INT= 134, + BINARY_OP_SUBSCR_DICT= 135, + BINARY_OP_SUBSCR_GETITEM= 136, + BINARY_OP_SUBSCR_LIST_INT= 137, + BINARY_OP_SUBSCR_LIST_SLICE= 138, + BINARY_OP_SUBSCR_STR_INT= 139, + BINARY_OP_SUBSCR_TUPLE_INT= 140, + BINARY_OP_SUBTRACT_FLOAT= 141, + BINARY_OP_SUBTRACT_INT= 142, + CALL_ALLOC_AND_ENTER_INIT= 143, + CALL_BOUND_METHOD_EXACT_ARGS= 144, + CALL_BOUND_METHOD_GENERAL= 145, + CALL_BUILTIN_CLASS= 146, + CALL_BUILTIN_FAST= 147, + CALL_BUILTIN_FAST_WITH_KEYWORDS= 148, + CALL_BUILTIN_O= 149, + CALL_ISINSTANCE= 150, + CALL_KW_BOUND_METHOD= 151, + CALL_KW_NON_PY= 152, + CALL_KW_PY= 153, + CALL_LEN= 154, + CALL_LIST_APPEND= 155, + CALL_METHOD_DESCRIPTOR_FAST= 156, + CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS= 157, + CALL_METHOD_DESCRIPTOR_NOARGS= 158, + CALL_METHOD_DESCRIPTOR_O= 159, + CALL_NON_PY_GENERAL= 160, + CALL_PY_EXACT_ARGS= 161, + CALL_PY_GENERAL= 162, + CALL_STR_1= 163, + CALL_TUPLE_1= 164, + CALL_TYPE_1= 165, + COMPARE_OP_FLOAT= 166, + COMPARE_OP_INT= 167, + COMPARE_OP_STR= 168, + CONTAINS_OP_DICT= 169, + CONTAINS_OP_SET= 170, + FOR_ITER_GEN= 171, + FOR_ITER_LIST= 172, + FOR_ITER_RANGE= 173, + FOR_ITER_TUPLE= 174, + JUMP_BACKWARD_JIT= 175, + JUMP_BACKWARD_NO_JIT= 176, + LOAD_ATTR_CLASS= 177, + LOAD_ATTR_CLASS_WITH_METACLASS_CHECK= 178, + LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN= 179, + LOAD_ATTR_INSTANCE_VALUE= 180, + LOAD_ATTR_METHOD_LAZY_DICT= 181, + LOAD_ATTR_METHOD_NO_DICT= 182, + LOAD_ATTR_METHOD_WITH_VALUES= 183, + LOAD_ATTR_MODULE= 184, + LOAD_ATTR_NONDESCRIPTOR_NO_DICT= 185, + LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES= 186, + LOAD_ATTR_PROPERTY= 187, + LOAD_ATTR_SLOT= 188, + LOAD_ATTR_WITH_HINT= 189, + LOAD_GLOBAL_BUILTIN= 190, + LOAD_GLOBAL_MODULE= 191, + LOAD_SUPER_ATTR_ATTR= 192, + LOAD_SUPER_ATTR_METHOD= 193, + RESUME_CHECK= 194, + SEND_GEN= 195, + STORE_ATTR_INSTANCE_VALUE= 196, + STORE_ATTR_SLOT= 197, + STORE_ATTR_WITH_HINT= 198, + STORE_SUBSCR_DICT= 199, + STORE_SUBSCR_LIST_INT= 200, + TO_BOOL_ALWAYS_TRUE= 201, + TO_BOOL_BOOL= 202, + TO_BOOL_INT= 203, + TO_BOOL_LIST= 204, + TO_BOOL_NONE= 205, + TO_BOOL_STR= 206, + UNPACK_SEQUENCE_LIST= 207, + UNPACK_SEQUENCE_TUPLE= 208, + UNPACK_SEQUENCE_TWO_TUPLE= 209, +) -opmap = { - 'CACHE': 0, - 'RESERVED': 17, - 'RESUME': 128, - 'INSTRUMENTED_LINE': 254, - 'ENTER_EXECUTOR': 255, - 'BINARY_SLICE': 1, - 'BUILD_TEMPLATE': 2, - 'CALL_FUNCTION_EX': 4, - 'CHECK_EG_MATCH': 5, - 'CHECK_EXC_MATCH': 6, - 'CLEANUP_THROW': 7, - 'DELETE_SUBSCR': 8, - 'END_FOR': 9, - 'END_SEND': 10, - 'EXIT_INIT_CHECK': 11, - 'FORMAT_SIMPLE': 12, - 'FORMAT_WITH_SPEC': 13, - 'GET_AITER': 14, - 'GET_ANEXT': 15, - 'GET_ITER': 16, - 'GET_LEN': 18, - 'GET_YIELD_FROM_ITER': 19, - 'INTERPRETER_EXIT': 20, - 'LOAD_BUILD_CLASS': 21, - 'LOAD_LOCALS': 22, - 'MAKE_FUNCTION': 23, - 'MATCH_KEYS': 24, - 'MATCH_MAPPING': 25, - 'MATCH_SEQUENCE': 26, - 'NOP': 27, - 'NOT_TAKEN': 28, - 'POP_EXCEPT': 29, - 'POP_ITER': 30, - 'POP_TOP': 31, - 'PUSH_EXC_INFO': 32, - 'PUSH_NULL': 33, - 'RETURN_GENERATOR': 34, - 'RETURN_VALUE': 35, - 'SETUP_ANNOTATIONS': 36, - 'STORE_SLICE': 37, - 'STORE_SUBSCR': 38, - 'TO_BOOL': 39, - 'UNARY_INVERT': 40, - 'UNARY_NEGATIVE': 41, - 'UNARY_NOT': 42, - 'WITH_EXCEPT_START': 43, - 'BINARY_OP': 44, - 'BUILD_INTERPOLATION': 45, - 'BUILD_LIST': 46, - 'BUILD_MAP': 47, - 'BUILD_SET': 48, - 'BUILD_SLICE': 49, - 'BUILD_STRING': 50, - 'BUILD_TUPLE': 51, - 'CALL': 52, - 'CALL_INTRINSIC_1': 53, - 'CALL_INTRINSIC_2': 54, - 'CALL_KW': 55, - 'COMPARE_OP': 56, - 'CONTAINS_OP': 57, - 'CONVERT_VALUE': 58, - 'COPY': 59, - 'COPY_FREE_VARS': 60, - 'DELETE_ATTR': 61, - 'DELETE_DEREF': 62, - 'DELETE_FAST': 63, - 'DELETE_GLOBAL': 64, - 'DELETE_NAME': 65, - 'DICT_MERGE': 66, - 'DICT_UPDATE': 67, - 'END_ASYNC_FOR': 68, - 'EXTENDED_ARG': 69, - 'FOR_ITER': 70, - 'GET_AWAITABLE': 71, - 'IMPORT_FROM': 72, - 'IMPORT_NAME': 73, - 'IS_OP': 74, - 'JUMP_BACKWARD': 75, - 'JUMP_BACKWARD_NO_INTERRUPT': 76, - 'JUMP_FORWARD': 77, - 'LIST_APPEND': 78, - 'LIST_EXTEND': 79, - 'LOAD_ATTR': 80, - 'LOAD_COMMON_CONSTANT': 81, - 'LOAD_CONST': 82, - 'LOAD_DEREF': 83, - 'LOAD_FAST': 84, - 'LOAD_FAST_AND_CLEAR': 85, - 'LOAD_FAST_BORROW': 86, - 'LOAD_FAST_BORROW_LOAD_FAST_BORROW': 87, - 'LOAD_FAST_CHECK': 88, - 'LOAD_FAST_LOAD_FAST': 89, - 'LOAD_FROM_DICT_OR_DEREF': 90, - 'LOAD_FROM_DICT_OR_GLOBALS': 91, - 'LOAD_GLOBAL': 92, - 'LOAD_NAME': 93, - 'LOAD_SMALL_INT': 94, - 'LOAD_SPECIAL': 95, - 'LOAD_SUPER_ATTR': 96, - 'MAKE_CELL': 97, - 'MAP_ADD': 98, - 'MATCH_CLASS': 99, - 'POP_JUMP_IF_FALSE': 100, - 'POP_JUMP_IF_NONE': 101, - 'POP_JUMP_IF_NOT_NONE': 102, - 'POP_JUMP_IF_TRUE': 103, - 'RAISE_VARARGS': 104, - 'RERAISE': 105, - 'SEND': 106, - 'SET_ADD': 107, - 'SET_FUNCTION_ATTRIBUTE': 108, - 'SET_UPDATE': 109, - 'STORE_ATTR': 110, - 'STORE_DEREF': 111, - 'STORE_FAST': 112, - 'STORE_FAST_LOAD_FAST': 113, - 'STORE_FAST_STORE_FAST': 114, - 'STORE_GLOBAL': 115, - 'STORE_NAME': 116, - 'SWAP': 117, - 'UNPACK_EX': 118, - 'UNPACK_SEQUENCE': 119, - 'YIELD_VALUE': 120, - 'INSTRUMENTED_END_FOR': 234, - 'INSTRUMENTED_POP_ITER': 235, - 'INSTRUMENTED_END_SEND': 236, - 'INSTRUMENTED_FOR_ITER': 237, - 'INSTRUMENTED_INSTRUCTION': 238, - 'INSTRUMENTED_JUMP_FORWARD': 239, - 'INSTRUMENTED_NOT_TAKEN': 240, - 'INSTRUMENTED_POP_JUMP_IF_TRUE': 241, - 'INSTRUMENTED_POP_JUMP_IF_FALSE': 242, - 'INSTRUMENTED_POP_JUMP_IF_NONE': 243, - 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 244, - 'INSTRUMENTED_RESUME': 245, - 'INSTRUMENTED_RETURN_VALUE': 246, - 'INSTRUMENTED_YIELD_VALUE': 247, - 'INSTRUMENTED_END_ASYNC_FOR': 248, - 'INSTRUMENTED_LOAD_SUPER_ATTR': 249, - 'INSTRUMENTED_CALL': 250, - 'INSTRUMENTED_CALL_KW': 251, - 'INSTRUMENTED_CALL_FUNCTION_EX': 252, - 'INSTRUMENTED_JUMP_BACKWARD': 253, - 'ANNOTATIONS_PLACEHOLDER': 256, - 'JUMP': 257, - 'JUMP_IF_FALSE': 258, - 'JUMP_IF_TRUE': 259, - 'JUMP_NO_INTERRUPT': 260, - 'LOAD_CLOSURE': 261, - 'POP_BLOCK': 262, - 'SETUP_CLEANUP': 263, - 'SETUP_FINALLY': 264, - 'SETUP_WITH': 265, - 'STORE_FAST_MAYBE_NULL': 266, -} +opmap = frozendict( + CACHE= 0, + RESERVED= 17, + RESUME= 128, + INSTRUMENTED_LINE= 254, + ENTER_EXECUTOR= 255, + BINARY_SLICE= 1, + BUILD_TEMPLATE= 2, + CALL_FUNCTION_EX= 4, + CHECK_EG_MATCH= 5, + CHECK_EXC_MATCH= 6, + CLEANUP_THROW= 7, + DELETE_SUBSCR= 8, + END_FOR= 9, + END_SEND= 10, + EXIT_INIT_CHECK= 11, + FORMAT_SIMPLE= 12, + FORMAT_WITH_SPEC= 13, + GET_AITER= 14, + GET_ANEXT= 15, + GET_ITER= 16, + GET_LEN= 18, + GET_YIELD_FROM_ITER= 19, + INTERPRETER_EXIT= 20, + LOAD_BUILD_CLASS= 21, + LOAD_LOCALS= 22, + MAKE_FUNCTION= 23, + MATCH_KEYS= 24, + MATCH_MAPPING= 25, + MATCH_SEQUENCE= 26, + NOP= 27, + NOT_TAKEN= 28, + POP_EXCEPT= 29, + POP_ITER= 30, + POP_TOP= 31, + PUSH_EXC_INFO= 32, + PUSH_NULL= 33, + RETURN_GENERATOR= 34, + RETURN_VALUE= 35, + SETUP_ANNOTATIONS= 36, + STORE_SLICE= 37, + STORE_SUBSCR= 38, + TO_BOOL= 39, + UNARY_INVERT= 40, + UNARY_NEGATIVE= 41, + UNARY_NOT= 42, + WITH_EXCEPT_START= 43, + BINARY_OP= 44, + BUILD_INTERPOLATION= 45, + BUILD_LIST= 46, + BUILD_MAP= 47, + BUILD_SET= 48, + BUILD_SLICE= 49, + BUILD_STRING= 50, + BUILD_TUPLE= 51, + CALL= 52, + CALL_INTRINSIC_1= 53, + CALL_INTRINSIC_2= 54, + CALL_KW= 55, + COMPARE_OP= 56, + CONTAINS_OP= 57, + CONVERT_VALUE= 58, + COPY= 59, + COPY_FREE_VARS= 60, + DELETE_ATTR= 61, + DELETE_DEREF= 62, + DELETE_FAST= 63, + DELETE_GLOBAL= 64, + DELETE_NAME= 65, + DICT_MERGE= 66, + DICT_UPDATE= 67, + END_ASYNC_FOR= 68, + EXTENDED_ARG= 69, + FOR_ITER= 70, + GET_AWAITABLE= 71, + IMPORT_FROM= 72, + IMPORT_NAME= 73, + IS_OP= 74, + JUMP_BACKWARD= 75, + JUMP_BACKWARD_NO_INTERRUPT= 76, + JUMP_FORWARD= 77, + LIST_APPEND= 78, + LIST_EXTEND= 79, + LOAD_ATTR= 80, + LOAD_COMMON_CONSTANT= 81, + LOAD_CONST= 82, + LOAD_DEREF= 83, + LOAD_FAST= 84, + LOAD_FAST_AND_CLEAR= 85, + LOAD_FAST_BORROW= 86, + LOAD_FAST_BORROW_LOAD_FAST_BORROW= 87, + LOAD_FAST_CHECK= 88, + LOAD_FAST_LOAD_FAST= 89, + LOAD_FROM_DICT_OR_DEREF= 90, + LOAD_FROM_DICT_OR_GLOBALS= 91, + LOAD_GLOBAL= 92, + LOAD_NAME= 93, + LOAD_SMALL_INT= 94, + LOAD_SPECIAL= 95, + LOAD_SUPER_ATTR= 96, + MAKE_CELL= 97, + MAP_ADD= 98, + MATCH_CLASS= 99, + POP_JUMP_IF_FALSE= 100, + POP_JUMP_IF_NONE= 101, + POP_JUMP_IF_NOT_NONE= 102, + POP_JUMP_IF_TRUE= 103, + RAISE_VARARGS= 104, + RERAISE= 105, + SEND= 106, + SET_ADD= 107, + SET_FUNCTION_ATTRIBUTE= 108, + SET_UPDATE= 109, + STORE_ATTR= 110, + STORE_DEREF= 111, + STORE_FAST= 112, + STORE_FAST_LOAD_FAST= 113, + STORE_FAST_STORE_FAST= 114, + STORE_GLOBAL= 115, + STORE_NAME= 116, + SWAP= 117, + UNPACK_EX= 118, + UNPACK_SEQUENCE= 119, + YIELD_VALUE= 120, + INSTRUMENTED_END_FOR= 234, + INSTRUMENTED_POP_ITER= 235, + INSTRUMENTED_END_SEND= 236, + INSTRUMENTED_FOR_ITER= 237, + INSTRUMENTED_INSTRUCTION= 238, + INSTRUMENTED_JUMP_FORWARD= 239, + INSTRUMENTED_NOT_TAKEN= 240, + INSTRUMENTED_POP_JUMP_IF_TRUE= 241, + INSTRUMENTED_POP_JUMP_IF_FALSE= 242, + INSTRUMENTED_POP_JUMP_IF_NONE= 243, + INSTRUMENTED_POP_JUMP_IF_NOT_NONE= 244, + INSTRUMENTED_RESUME= 245, + INSTRUMENTED_RETURN_VALUE= 246, + INSTRUMENTED_YIELD_VALUE= 247, + INSTRUMENTED_END_ASYNC_FOR= 248, + INSTRUMENTED_LOAD_SUPER_ATTR= 249, + INSTRUMENTED_CALL= 250, + INSTRUMENTED_CALL_KW= 251, + INSTRUMENTED_CALL_FUNCTION_EX= 252, + INSTRUMENTED_JUMP_BACKWARD= 253, + ANNOTATIONS_PLACEHOLDER= 256, + JUMP= 257, + JUMP_IF_FALSE= 258, + JUMP_IF_TRUE= 259, + JUMP_NO_INTERRUPT= 260, + LOAD_CLOSURE= 261, + POP_BLOCK= 262, + SETUP_CLEANUP= 263, + SETUP_FINALLY= 264, + SETUP_WITH= 265, + STORE_FAST_MAYBE_NULL= 266, +) HAVE_ARGUMENT = 43 MIN_INSTRUMENTED_OPCODE = 234 diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index b6d68f2372850a..1a93279286d080 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -164,13 +164,13 @@ def _build_struct_time(y, m, d, hh, mm, ss, dstflag): return _time.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag)) def _format_time(hh, mm, ss, us, timespec='auto'): - specs = { - 'hours': '{:02d}', - 'minutes': '{:02d}:{:02d}', - 'seconds': '{:02d}:{:02d}:{:02d}', - 'milliseconds': '{:02d}:{:02d}:{:02d}.{:03d}', - 'microseconds': '{:02d}:{:02d}:{:02d}.{:06d}' - } + specs = frozendict( + hours= '{:02d}', + minutes= '{:02d}:{:02d}', + seconds= '{:02d}:{:02d}:{:02d}', + milliseconds= '{:02d}:{:02d}:{:02d}.{:03d}', + microseconds= '{:02d}:{:02d}:{:02d}.{:06d}' + ) if timespec == 'auto': # Skip trailing microseconds when us==0. diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index ef889ea0cc834c..06e6d1df567f0c 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -331,10 +331,11 @@ class FloatOperation(DecimalException, TypeError): Underflow, InvalidOperation, Subnormal, FloatOperation] # Map conditions (per the spec) to signals -_condition_map = {ConversionSyntax:InvalidOperation, - DivisionImpossible:InvalidOperation, - DivisionUndefined:InvalidOperation, - InvalidContext:InvalidOperation} +_condition_map = frozendict({ + ConversionSyntax:InvalidOperation, + DivisionImpossible:InvalidOperation, + DivisionUndefined:InvalidOperation, + InvalidContext:InvalidOperation}) # Valid rounding modes _rounding_modes = (ROUND_DOWN, ROUND_HALF_UP, ROUND_HALF_EVEN, ROUND_CEILING, diff --git a/Lib/bdb.py b/Lib/bdb.py index efc3e0a235ac8e..2f406f1fced4b5 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -20,7 +20,7 @@ class BdbQuit(Exception): E = sys.monitoring.events class _MonitoringTracer: - EVENT_CALLBACK_MAP = { + EVENT_CALLBACK_MAP = frozendict({ E.PY_START: 'call', E.PY_RESUME: 'call', E.PY_THROW: 'call', @@ -32,7 +32,7 @@ class _MonitoringTracer: E.RAISE: 'exception', E.STOP_ITERATION: 'exception', E.INSTRUCTION: 'opcode', - } + }) GLOBAL_EVENTS = E.PY_START | E.PY_RESUME | E.PY_THROW | E.PY_UNWIND | E.RAISE LOCAL_EVENTS = E.LINE | E.JUMP | E.PY_RETURN | E.PY_YIELD | E.STOP_ITERATION diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 3ccb72469286eb..d83e19d77ecee6 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -957,7 +957,8 @@ def _hash_exception(cls, fields, func_builder): # | | | | +------- action # | | | | | # v v v v v -_hash_action = {(False, False, False, False): None, +_hash_action = frozendict( + {(False, False, False, False): None, (False, False, False, True ): None, (False, False, True, False): None, (False, False, True, True ): None, @@ -973,7 +974,7 @@ def _hash_exception(cls, fields, func_builder): (True, True, False, True ): _hash_exception, (True, True, True, False): _hash_add, (True, True, True, True ): _hash_exception, - } + }) # See https://bugs.python.org/issue32929#msg312829 for an if-statement # version of this table. diff --git a/Lib/dis.py b/Lib/dis.py index d6d2c1386dd785..4e7d64d9ed164c 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -65,9 +65,9 @@ _all_opname[op] = name _all_opmap[name] = op -deoptmap = { +deoptmap = frozendict({ specialized: base for base, family in _specializations.items() for specialized in family -} +}) def _try_compile(source, name): """Attempts to compile the given source, first as an expression and @@ -152,7 +152,7 @@ def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets # The inspect module interrogates this dictionary to build its # list of CO_* constants. It is also used by pretty_flags to # turn the co_flags field into a human readable list. -COMPILER_FLAG_NAMES = { +COMPILER_FLAG_NAMES = frozendict({ 1: "OPTIMIZED", 2: "NEWLOCALS", 4: "VARARGS", @@ -165,7 +165,7 @@ def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets 512: "ASYNC_GENERATOR", 0x4000000: "HAS_DOCSTRING", 0x8000000: "METHOD", -} +}) def pretty_flags(flags): """Return pretty representation of code flags.""" diff --git a/Lib/email/headerregistry.py b/Lib/email/headerregistry.py index 543141dc427ebe..d15df7cc2fbfb1 100644 --- a/Lib/email/headerregistry.py +++ b/Lib/email/headerregistry.py @@ -3,8 +3,6 @@ This module provides an implementation of the HeaderRegistry API. The implementation is designed to flexibly follow RFC5322 rules. """ -from types import MappingProxyType - from email import utils from email import errors from email import _header_value_parser as parser @@ -462,7 +460,7 @@ def init(self, *args, **kw): @property def params(self): - return MappingProxyType(self._params) + return frozendict(self._params) class ContentTypeHeader(ParameterizedMIMEHeader): diff --git a/Lib/enum.py b/Lib/enum.py index ad782b8c41e160..43b4982691e214 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1,6 +1,6 @@ import sys import builtins as bltns -from types import MappingProxyType, DynamicClassAttribute +from types import DynamicClassAttribute __all__ = [ @@ -817,7 +817,7 @@ def __members__(cls): This mapping lists all enum members, including aliases. Note that this is a read-only view of the internal mapping. """ - return MappingProxyType(cls._member_map_) + return frozendict(cls._member_map_) def __repr__(cls): if Flag is not None and issubclass(cls, Flag): diff --git a/Lib/functools.py b/Lib/functools.py index a92844ba7227b0..2e8565a660b293 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -170,7 +170,7 @@ def _lt_from_ge(self, other): return op_result return not op_result -_convert = { +_convert = frozendict({ '__lt__': [('__gt__', _gt_from_lt), ('__le__', _le_from_lt), ('__ge__', _ge_from_lt)], @@ -183,7 +183,7 @@ def _lt_from_ge(self, other): '__ge__': [('__le__', _le_from_ge), ('__gt__', _gt_from_ge), ('__lt__', _lt_from_ge)] -} +}) def total_ordering(cls): """Class decorator that fills in missing ordering methods""" diff --git a/Lib/gettext.py b/Lib/gettext.py index 6c11ab2b1eb570..2f77f0e849e9ae 100644 --- a/Lib/gettext.py +++ b/Lib/gettext.py @@ -111,8 +111,9 @@ def _error(value): ('+', '-'), ('*', '/', '%'), ) -_binary_ops = {op: i for i, ops in enumerate(_binary_ops, 1) for op in ops} -_c2py_ops = {'||': 'or', '&&': 'and', '/': '//'} +_binary_ops = frozendict({op: i for i, ops in enumerate(_binary_ops, 1) + for op in ops}) +_c2py_ops = frozendict({'||': 'or', '&&': 'and', '/': '//'}) def _parse(tokens, priority=-1): diff --git a/Lib/imaplib.py b/Lib/imaplib.py index c176736548188c..e5984b0446de39 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -54,7 +54,7 @@ # Commands -Commands = { +Commands = frozendict({ # name valid states 'APPEND': ('AUTH', 'SELECTED'), 'AUTHENTICATE': ('NONAUTH',), @@ -99,7 +99,7 @@ 'UID': ('SELECTED',), 'UNSUBSCRIBE': ('AUTH', 'SELECTED'), 'UNSELECT': ('SELECTED',), - } + }) # Patterns to match server responses @@ -1755,7 +1755,7 @@ def decode(self, inp): return binascii.a2b_base64(inp) Months = ' Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split(' ') -Mon2num = {s.encode():n+1 for n, s in enumerate(Months[1:])} +Mon2num = frozendict({s.encode():n+1 for n, s in enumerate(Months[1:])}) def Internaldate2tuple(resp): """Parse an IMAP4 INTERNALDATE string. diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py index 92ad6352557640..d1490491920845 100644 --- a/Lib/json/decoder.py +++ b/Lib/json/decoder.py @@ -43,19 +43,19 @@ def __reduce__(self): return self.__class__, (self.msg, self.doc, self.pos) -_CONSTANTS = { +_CONSTANTS = frozendict({ '-Infinity': NegInf, 'Infinity': PosInf, 'NaN': NaN, -} +}) HEXDIGITS = re.compile(r'[0-9A-Fa-f]{4}', FLAGS) STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) -BACKSLASH = { +BACKSLASH = frozendict({ '"': '"', '\\': '\\', '/': '/', 'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t', -} +}) def _decode_uXXXX(s, pos, _m=HEXDIGITS.match): esc = _m(s, pos + 1) diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 5cf6d64f3eade6..285ad02ed9544f 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -30,6 +30,9 @@ for i in range(0x20): ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) + +# freeze the dict to prevent accidental modifications +ESCAPE_DCT = frozendict(ESCAPE_DCT) del i INFINITY = float('inf') @@ -79,7 +82,7 @@ class JSONEncoder(object): +-------------------+---------------+ | Python | JSON | +===================+===============+ - | dict | object | + | dict, frozendict | object | +-------------------+---------------+ | list, tuple | array | +-------------------+---------------+ @@ -267,6 +270,7 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, ## HACK: hand-optimized bytecode; turn globals into locals ValueError=ValueError, dict=dict, + frozendict=frozendict, float=float, id=id, int=int, @@ -319,7 +323,7 @@ def _iterencode_list(lst, _current_indent_level): yield buf if isinstance(value, (list, tuple)): chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): + elif isinstance(value, (dict, frozendict)): chunks = _iterencode_dict(value, _current_indent_level) else: chunks = _iterencode(value, _current_indent_level) @@ -406,7 +410,7 @@ def _iterencode_dict(dct, _current_indent_level): else: if isinstance(value, (list, tuple)): chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): + elif isinstance(value, (dict, frozendict)): chunks = _iterencode_dict(value, _current_indent_level) else: chunks = _iterencode(value, _current_indent_level) @@ -440,7 +444,7 @@ def _iterencode(o, _current_indent_level): yield _floatstr(o) elif isinstance(o, (list, tuple)): yield from _iterencode_list(o, _current_indent_level) - elif isinstance(o, dict): + elif isinstance(o, (dict, frozendict)): yield from _iterencode_dict(o, _current_indent_level) else: if markers is not None: diff --git a/Lib/json/tool.py b/Lib/json/tool.py index 050c2fe2161e3e..14a5c9ecb74d13 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -22,13 +22,7 @@ (?Pnull) ''', re.VERBOSE) -_group_to_theme_color = { - "key": "definition", - "string": "string", - "number": "number", - "boolean": "keyword", - "null": "keyword", -} +_group_to_theme_color = frozendict(key="definition",string="string",number="number",boolean="keyword",null="keyword") def _colorize_json(json_str, theme): diff --git a/Lib/locale.py b/Lib/locale.py index 37cafb4a601b3c..a3aef3e421da87 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -757,6 +757,7 @@ def getpreferredencoding(do_setlocale=True): for k, v in sorted(locale_encoding_alias.items()): k = k.replace('_', '') locale_encoding_alias.setdefault(k, v) +locale_encoding_alias = frozendict(locale_encoding_alias) del k, v # @@ -909,7 +910,7 @@ def getpreferredencoding(do_setlocale=True): # removed 'el_gr@euro' # removed 'uz_uz@cyrillic' -locale_alias = { +locale_alias = frozendict({ 'a3': 'az_AZ.KOI8-C', 'a3_az': 'az_AZ.KOI8-C', 'a3_az.koic': 'az_AZ.KOI8-C', @@ -1509,7 +1510,7 @@ def getpreferredencoding(do_setlocale=True): 'zh_tw.euctw': 'zh_TW.eucTW', 'zu': 'zu_ZA.ISO8859-1', 'zu_za': 'zu_ZA.ISO8859-1', -} +}) # # This maps Windows language identifiers to locale strings. @@ -1525,7 +1526,7 @@ def getpreferredencoding(do_setlocale=True): # locale code. # -windows_locale = { +windows_locale = frozendict({ 0x0436: "af_ZA", # Afrikaans 0x041c: "sq_AL", # Albanian 0x0484: "gsw_FR",# Alsatian - France @@ -1736,7 +1737,7 @@ def getpreferredencoding(do_setlocale=True): 0x0478: "ii_CN", # Yi - PRC 0x046a: "yo_NG", # Yoruba - Nigeria 0x0435: "zu_ZA", # Zulu -} +}) def _print_locale(): diff --git a/Lib/opcode.py b/Lib/opcode.py index 0e9520b6832499..ef183475972053 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -45,78 +45,78 @@ hascompare = [opmap["COMPARE_OP"]] -_cache_format = { - "LOAD_GLOBAL": { +_cache_format = frozendict( + LOAD_GLOBAL= { "counter": 1, "index": 1, "module_keys_version": 1, "builtin_keys_version": 1, }, - "BINARY_OP": { + BINARY_OP= { "counter": 1, "descr": 4, }, - "UNPACK_SEQUENCE": { + UNPACK_SEQUENCE= { "counter": 1, }, - "COMPARE_OP": { + COMPARE_OP= { "counter": 1, }, - "CONTAINS_OP": { + CONTAINS_OP= { "counter": 1, }, - "FOR_ITER": { + FOR_ITER= { "counter": 1, }, - "LOAD_SUPER_ATTR": { + LOAD_SUPER_ATTR= { "counter": 1, }, - "LOAD_ATTR": { + LOAD_ATTR= { "counter": 1, "version": 2, "keys_version": 2, "descr": 4, }, - "STORE_ATTR": { + STORE_ATTR= { "counter": 1, "version": 2, "index": 1, }, - "CALL": { + CALL= { "counter": 1, "func_version": 2, }, - "CALL_KW": { + CALL_KW= { "counter": 1, "func_version": 2, }, - "STORE_SUBSCR": { + STORE_SUBSCR= { "counter": 1, }, - "SEND": { + SEND= { "counter": 1, }, - "JUMP_BACKWARD": { + JUMP_BACKWARD= { "counter": 1, }, - "TO_BOOL": { + TO_BOOL= { "counter": 1, "version": 2, }, - "POP_JUMP_IF_TRUE": { + POP_JUMP_IF_TRUE= { "counter": 1, }, - "POP_JUMP_IF_FALSE": { + POP_JUMP_IF_FALSE= { "counter": 1, }, - "POP_JUMP_IF_NONE": { + POP_JUMP_IF_NONE= { "counter": 1, }, - "POP_JUMP_IF_NOT_NONE": { + POP_JUMP_IF_NOT_NONE= { "counter": 1, }, -} +) -_inline_cache_entries = { +_inline_cache_entries = frozendict({ name : sum(value.values()) for (name, value) in _cache_format.items() -} +}) diff --git a/Lib/optparse.py b/Lib/optparse.py index 02ff7140882ed6..2f885876b0e691 100644 --- a/Lib/optparse.py +++ b/Lib/optparse.py @@ -407,10 +407,11 @@ def _parse_num(val, type): def _parse_int(val): return _parse_num(val, int) -_builtin_cvt = { "int" : (_parse_int, _("integer")), - "long" : (_parse_int, _("integer")), - "float" : (float, _("floating-point")), - "complex" : (complex, _("complex")) } +_builtin_cvt = frozendict( + { "int" : (_parse_int, _("integer")), + "long" : (_parse_int, _("integer")), + "float" : (float, _("floating-point")), + "complex" : (complex, _("complex")) }) def check_builtin(option, opt, value): (cvt, what) = _builtin_cvt[option.type] diff --git a/Lib/pickle.py b/Lib/pickle.py index 729c215514ad24..be346aacdf086a 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -185,6 +185,7 @@ def __init__(self, value): BYTEARRAY8 = b'\x96' # push bytearray NEXT_BUFFER = b'\x97' # push next out-of-band buffer READONLY_BUFFER = b'\x98' # make top of stack readonly +FROZENDICT = b'\x99' __all__.extend(x for x in dir() if x.isupper() and not x.startswith('_')) @@ -1064,6 +1065,31 @@ def save_dict(self, obj): dispatch[dict] = save_dict + def save_frozendict(self, obj): + save = self.save + write = self.write + + if self.proto < 5: + self.save_reduce(frozendict, (dict(obj),), obj=obj) + return + + write(MARK) + for k, v in obj.items(): + save(k) + save(v) + + if id(obj) in self.memo: + # If the object is already in the memo, this means it is + # recursive. In this case, throw away everything we put on the + # stack, and fetch the object back from the memo. + write(POP_MARK + self.get(self.memo[id(obj)][0])) + return + + write(FROZENDICT) + self.memoize(obj) + + dispatch[frozendict] = save_frozendict + def _batch_setitems(self, items, obj): # Helper to batch up SETITEMS sequences; proto >= 1 only save = self.save @@ -1589,6 +1615,13 @@ def load_dict(self): self.append(d) dispatch[DICT[0]] = load_dict + def load_frozendict(self): + items = self.pop_mark() + d = frozendict({items[i]: items[i+1] for i in range(0, len(items), 2)}) + self.append(d) + + dispatch[FROZENDICT[0]] = load_frozendict + # INST and OBJ differ only in how they get a class object. It's not # only sensible to do the rest in a common routine, the two routines # previously diverged and grew different bugs. diff --git a/Lib/pickletools.py b/Lib/pickletools.py index 254b6c7fcc9dd2..1dff5add96ebad 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -1035,6 +1035,11 @@ def __repr__(self): obtype=dict, doc="A Python dict object.") +pyfrozendict = StackObject( + name="frozendict", + obtype=frozendict, + doc="A Python frozendict object.") + pyset = StackObject( name="set", obtype=set, @@ -1384,6 +1389,23 @@ def __init__(self, name, code, arg, proto=5, doc="Make an out-of-band buffer object read-only."), + I(name='FROZENDICT', + code='\x99', + arg=None, + stack_before=[markobject, stackslice], + stack_after=[pyfrozendict], + proto=5, + doc="""Build a frozendict out of the topmost stack slice, after markobject. + + All the stack entries following the topmost markobject are placed into + a single Python dict, which single dict object replaces all of the + stack from the topmost markobject onward. The stack slice alternates + key, value, key, value, .... For example, + + Stack before: ... markobject 1 2 3 'abc' + Stack after: ... {1: 2, 3: 'abc'} + """), + # Ways to spell None. I(name='NONE', diff --git a/Lib/platform.py b/Lib/platform.py index 4db93bea2a39e1..d3db1933a672a5 100644 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -127,7 +127,7 @@ # Based on the description of the PHP's version_compare(): # http://php.net/manual/en/function.version-compare.php -_ver_stages = { +_ver_stages = frozendict({ # any string not found in this dict, will get 0 assigned 'dev': 10, 'alpha': 20, 'a': 20, @@ -136,7 +136,7 @@ 'RC': 50, 'rc': 50, # number, will get 100 assigned 'pl': 200, 'p': 200, -} +}) def _comparable_version(version): @@ -706,11 +706,11 @@ def _syscmd_file(target, default=''): # Default values for architecture; non-empty strings override the # defaults given as parameters -_default_architecture = { +_default_architecture = frozendict({ 'win32': ('', 'WindowsPE'), 'win16': ('', 'Windows'), 'dos': ('', 'MSDOS'), -} +}) def architecture(executable=sys.executable, bits='', linkage=''): diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 67e832db217319..952b0670cf77b1 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -457,7 +457,7 @@ class InvalidFileException (ValueError): def __init__(self, message="Invalid file"): ValueError.__init__(self, message) -_BINARY_FORMAT = {1: 'B', 2: 'H', 4: 'L', 8: 'Q'} +_BINARY_FORMAT = frozendict({1: 'B', 2: 'H', 4: 'L', 8: 'Q'}) _undefined = object() diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 293c3189589e36..ef6cc1001542ac 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -3679,7 +3679,7 @@ def f() -> annotation: ... * a class that inherits from any of the above - The standard library classes "dict" and "types.MappingProxyType" + The standard library classes "dict" and "frozendict" are mappings. [4] A string literal appearing as the first statement in the function @@ -13174,8 +13174,7 @@ class dict(iterable, /, **kwargs) See also: - "types.MappingProxyType" can be used to create a read-only view of a - "dict". + "frozendict" can be used to create a read-only view of a "dict". Dictionary view objects diff --git a/Lib/ssl.py b/Lib/ssl.py index 7ad7969a8217f8..65cd19e47288ae 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -150,7 +150,8 @@ source=_ssl) PROTOCOL_SSLv23 = _SSLMethod.PROTOCOL_SSLv23 = _SSLMethod.PROTOCOL_TLS -_PROTOCOL_NAMES = {value: name for name, value in _SSLMethod.__members__.items()} +_PROTOCOL_NAMES = frozendict({ + value: name for name, value in _SSLMethod.__members__.items()}) _SSLv2_IF_EXISTS = getattr(_SSLMethod, 'PROTOCOL_SSLv2', None) diff --git a/Lib/stringprep.py b/Lib/stringprep.py index 44ecdb266ce8b9..cb8f56efbc57cb 100644 --- a/Lib/stringprep.py +++ b/Lib/stringprep.py @@ -21,7 +21,7 @@ def in_table_b1(code): return ord(code) in b1_set -b3_exceptions = { +b3_exceptions = frozendict({ 0xb5:'\u03bc', 0xdf:'ss', 0x130:'i\u0307', 0x149:'\u02bcn', 0x17f:'s', 0x1f0:'j\u030c', 0x345:'\u03b9', 0x37a:' \u03b9', 0x390:'\u03b9\u0308\u0301', 0x3b0:'\u03c5\u0308\u0301', 0x3c2:'\u03c3', 0x3d0:'\u03b2', @@ -184,7 +184,7 @@ def in_table_b1(code): 0x1d79c:'\u03bd', 0x1d79d:'\u03be', 0x1d79e:'\u03bf', 0x1d79f:'\u03c0', 0x1d7a0:'\u03c1', 0x1d7a1:'\u03b8', 0x1d7a2:'\u03c3', 0x1d7a3:'\u03c4', 0x1d7a4:'\u03c5', 0x1d7a5:'\u03c6', 0x1d7a6:'\u03c7', 0x1d7a7:'\u03c8', -0x1d7a8:'\u03c9', 0x1d7bb:'\u03c3', } +0x1d7a8:'\u03c9', 0x1d7bb:'\u03c3', }) def map_table_b3(code): r = b3_exceptions.get(ord(code)) diff --git a/Lib/symtable.py b/Lib/symtable.py index 4c832e68f94cbd..de2cc4a9cf0cc0 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -401,7 +401,7 @@ def get_namespace(self): _flags = [('USE', USE)] _flags.extend(kv for kv in globals().items() if kv[0].startswith('DEF_')) _scopes_names = ('FREE', 'LOCAL', 'GLOBAL_IMPLICIT', 'GLOBAL_EXPLICIT', 'CELL') -_scopes_value_to_name = {globals()[n]: n for n in _scopes_names} +_scopes_value_to_name = frozendict({globals()[n]: n for n in _scopes_names}) def main(args): diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 7db3a40c9b33cf..54790cb6d96a97 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -133,14 +133,14 @@ # Fields in a pax header that are numbers, all other fields # are treated as strings. -PAX_NUMBER_FIELDS = { +PAX_NUMBER_FIELDS = frozendict({ "atime": float, "ctime": float, "mtime": float, "uid": int, "gid": int, "size": int -} +}) #--------------------------------------------------------- # initialization @@ -860,11 +860,11 @@ def data_filter(member, dest_path): return member.replace(**new_attrs, deep=False) return member -_NAMED_FILTERS = { +_NAMED_FILTERS = frozendict({ "fully_trusted": fully_trusted_filter, "tar": tar_filter, "data": data_filter, -} +}) #------------------ # Exported Classes diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index 20306e1526d7b8..ccc1268bdff1df 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -4,7 +4,7 @@ from test import support -class BasicTestMappingProtocol(unittest.TestCase): +class BasicTestImmutableMappingProtocol(unittest.TestCase): # This base class can be used to check that an object conforms to the # mapping protocol @@ -22,10 +22,7 @@ def _empty_mapping(self): def _full_mapping(self, data): """Return a mapping object with the value contained in data dictionary""" - x = self._empty_mapping() - for key, value in data.items(): - x[key] = value - return x + return self.type2test(data) def __init__(self, *args, **kw): unittest.TestCase.__init__(self, *args, **kw) @@ -88,6 +85,72 @@ def check_iterandlist(iter, lst, ref): self.assertEqual(d.get(knownkey, knownvalue), knownvalue) self.assertNotIn(knownkey, d) + def test_constructor(self): + self.assertEqual(self._empty_mapping(), self._empty_mapping()) + + def test_bool(self): + self.assertTrue(not self._empty_mapping()) + self.assertTrue(self.reference) + self.assertTrue(bool(self._empty_mapping()) is False) + self.assertTrue(bool(self.reference) is True) + + def test_keys(self): + d = self._empty_mapping() + self.assertEqual(list(d.keys()), []) + d = self.reference + self.assertIn(list(self.inmapping.keys())[0], d.keys()) + self.assertNotIn(list(self.other.keys())[0], d.keys()) + self.assertRaises(TypeError, d.keys, None) + + def test_values(self): + d = self._empty_mapping() + self.assertEqual(list(d.values()), []) + + self.assertRaises(TypeError, d.values, None) + + def test_items(self): + d = self._empty_mapping() + self.assertEqual(list(d.items()), []) + + self.assertRaises(TypeError, d.items, None) + + def test_len(self): + d = self._empty_mapping() + self.assertEqual(len(d), 0) + + def test_getitem(self): + d = self.reference + self.assertEqual(d[list(self.inmapping.keys())[0]], + list(self.inmapping.values())[0]) + + self.assertRaises(TypeError, d.__getitem__) + + # no test_fromkeys or test_copy as both os.environ and selves don't support it + + def test_get(self): + d = self._empty_mapping() + self.assertTrue(d.get(list(self.other.keys())[0]) is None) + self.assertEqual(d.get(list(self.other.keys())[0], 3), 3) + d = self.reference + self.assertTrue(d.get(list(self.other.keys())[0]) is None) + self.assertEqual(d.get(list(self.other.keys())[0], 3), 3) + self.assertEqual(d.get(list(self.inmapping.keys())[0]), + list(self.inmapping.values())[0]) + self.assertEqual(d.get(list(self.inmapping.keys())[0], 3), + list(self.inmapping.values())[0]) + self.assertRaises(TypeError, d.get) + self.assertRaises(TypeError, d.get, None, None, None) + + +class BasicTestMappingProtocol(BasicTestImmutableMappingProtocol): + def _full_mapping(self, data): + """Return a mapping object with the value contained in data + dictionary""" + x = self._empty_mapping() + for key, value in data.items(): + x[key] = value + return x + def test_write(self): # Test for write operations on mapping p = self._empty_mapping() @@ -130,46 +193,6 @@ def test_write(self): p=self._empty_mapping() self.assertRaises(KeyError, p.popitem) - def test_constructor(self): - self.assertEqual(self._empty_mapping(), self._empty_mapping()) - - def test_bool(self): - self.assertTrue(not self._empty_mapping()) - self.assertTrue(self.reference) - self.assertTrue(bool(self._empty_mapping()) is False) - self.assertTrue(bool(self.reference) is True) - - def test_keys(self): - d = self._empty_mapping() - self.assertEqual(list(d.keys()), []) - d = self.reference - self.assertIn(list(self.inmapping.keys())[0], d.keys()) - self.assertNotIn(list(self.other.keys())[0], d.keys()) - self.assertRaises(TypeError, d.keys, None) - - def test_values(self): - d = self._empty_mapping() - self.assertEqual(list(d.values()), []) - - self.assertRaises(TypeError, d.values, None) - - def test_items(self): - d = self._empty_mapping() - self.assertEqual(list(d.items()), []) - - self.assertRaises(TypeError, d.items, None) - - def test_len(self): - d = self._empty_mapping() - self.assertEqual(len(d), 0) - - def test_getitem(self): - d = self.reference - self.assertEqual(d[list(self.inmapping.keys())[0]], - list(self.inmapping.values())[0]) - - self.assertRaises(TypeError, d.__getitem__) - def test_update(self): # mapping argument d = self._empty_mapping() @@ -265,22 +288,6 @@ def __next__(self): self.assertRaises(ValueError, d.update, [(1, 2, 3)]) - # no test_fromkeys or test_copy as both os.environ and selves don't support it - - def test_get(self): - d = self._empty_mapping() - self.assertTrue(d.get(list(self.other.keys())[0]) is None) - self.assertEqual(d.get(list(self.other.keys())[0], 3), 3) - d = self.reference - self.assertTrue(d.get(list(self.other.keys())[0]) is None) - self.assertEqual(d.get(list(self.other.keys())[0], 3), 3) - self.assertEqual(d.get(list(self.inmapping.keys())[0]), - list(self.inmapping.values())[0]) - self.assertEqual(d.get(list(self.inmapping.keys())[0], 3), - list(self.inmapping.values())[0]) - self.assertRaises(TypeError, d.get) - self.assertRaises(TypeError, d.get, None, None, None) - def test_setdefault(self): d = self._empty_mapping() self.assertRaises(TypeError, d.setdefault) diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 2e6c2bbdf19409..af3a390150a549 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1664,6 +1664,9 @@ class Dict(dict): class SubclassMappingTests(mapping_tests.BasicTestMappingProtocol): type2test = Dict +class FrozenDictMappingTests(mapping_tests.BasicTestImmutableMappingProtocol): + type2test = frozendict + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index 0fa74407e3c436..0429b1fd3c071a 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -742,7 +742,7 @@ def non_Python_modules(): r""" >>> import builtins >>> tests = doctest.DocTestFinder().find(builtins) - >>> 750 < len(tests) < 800 # approximate number of objects with docstrings + >>> 750 < len(tests) < 850 # approximate number of objects with docstrings True >>> real_tests = [t for t in tests if len(t.examples) > 0] >>> len(real_tests) # objects that actually have doctests diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index dd3b7d9c5b4b5b..04e38ab323aeca 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -6103,7 +6103,8 @@ def _test_builtin_methods_have_signatures(self, cls, no_signature, unsupported_s self.assertRaises(ValueError, inspect.signature, getattr(cls, name)) def test_builtins_have_signatures(self): - no_signature = {'type', 'super', 'bytearray', 'bytes', 'dict', 'int', 'str'} + no_signature = {'type', 'super', 'bytearray', 'bytes', + 'dict', 'frozendict', 'int', 'str'} # These need PEP 457 groups needs_groups = {"range", "slice", "dir", "getattr", "next", "iter", "vars"} diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index cf990874621eae..f4a6e2e81b4c70 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -506,7 +506,7 @@ def test__all__(self): 'StackObject', 'pyint', 'pylong', 'pyinteger_or_bool', 'pybool', 'pyfloat', 'pybytes_or_str', 'pystring', 'pybytes', 'pybytearray', - 'pyunicode', 'pynone', 'pytuple', 'pylist', 'pydict', + 'pyunicode', 'pynone', 'pytuple', 'pylist', 'pydict', 'pyfrozendict', 'pyset', 'pyfrozenset', 'pybuffer', 'anyobject', 'markobject', 'stackslice', 'OpcodeInfo', 'opcodes', 'code2op', diff --git a/Lib/token.py b/Lib/token.py index f61723cc09da02..f76de6d24e84d8 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -78,12 +78,13 @@ # Special definitions for cooperation with parser NT_OFFSET = 256 -tok_name = {value: name - for name, value in globals().items() - if isinstance(value, int) and not name.startswith('_')} +tok_name = frozendict({ + value: name + for name, value in globals().items() + if isinstance(value, int) and not name.startswith('_')}) __all__.extend(tok_name.values()) -EXACT_TOKEN_TYPES = { +EXACT_TOKEN_TYPES = frozendict({ '!': EXCLAMATION, '!=': NOTEQUAL, '%': PERCENT, @@ -132,7 +133,7 @@ '|=': VBAREQUAL, '}': RBRACE, '~': TILDE, -} +}) def ISTERMINAL(x: int) -> bool: return x < NT_OFFSET diff --git a/Lib/tomllib/_parser.py b/Lib/tomllib/_parser.py index 3ee47aa9e0afba..35ce41be745dd2 100644 --- a/Lib/tomllib/_parser.py +++ b/Lib/tomllib/_parser.py @@ -4,8 +4,6 @@ from __future__ import annotations -from types import MappingProxyType - from ._re import ( RE_DATETIME, RE_LOCALTIME, @@ -42,7 +40,7 @@ KEY_INITIAL_CHARS = BARE_KEY_CHARS | frozenset("\"'") HEXDIGIT_CHARS = frozenset("abcdef" "ABCDEF" "0123456789") -BASIC_STR_ESCAPE_REPLACEMENTS = MappingProxyType( +BASIC_STR_ESCAPE_REPLACEMENTS = frozendict( { "\\b": "\u0008", # backspace "\\t": "\u0009", # tab diff --git a/Lib/typing.py b/Lib/typing.py index eb0519986a8952..189d320e382e83 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1915,7 +1915,7 @@ def _allow_reckless_class_checks(depth=2): return _caller(depth) in {'abc', '_py_abc', 'functools', None} -_PROTO_ALLOWLIST = { +_PROTO_ALLOWLIST = frozendict({ 'collections.abc': [ 'Callable', 'Awaitable', 'Iterable', 'Iterator', 'AsyncIterable', 'AsyncIterator', 'Hashable', 'Sized', 'Container', 'Collection', @@ -1924,7 +1924,7 @@ def _allow_reckless_class_checks(depth=2): 'contextlib': ['AbstractContextManager', 'AbstractAsyncContextManager'], 'io': ['Reader', 'Writer'], 'os': ['PathLike'], -} +}) @functools.cache diff --git a/Modules/_json.c b/Modules/_json.c index 14714d4b346546..34626dc82e7236 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1597,7 +1597,7 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, _Py_LeaveRecursiveCall(); return rv; } - else if (PyDict_Check(obj)) { + else if (PyDict_Check(obj) || PyFrozenDict_Check(obj)) { if (_Py_EnterRecursiveCall(" while encoding a JSON object")) return -1; rv = encoder_listencode_dict(s, writer, obj, indent_level, indent_cache); diff --git a/Modules/_pickle.c b/Modules/_pickle.c index bfb2830f3893d6..13cace30863713 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -136,7 +136,8 @@ enum opcode { /* Protocol 5 */ BYTEARRAY8 = '\x96', NEXT_BUFFER = '\x97', - READONLY_BUFFER = '\x98' + READONLY_BUFFER = '\x98', + FROZENDICT = '\x99', }; enum { @@ -324,7 +325,7 @@ _Pickle_InitState(PickleState *st) PyObject_GetAttrString(compat_pickle, "NAME_MAPPING"); if (!st->name_mapping_2to3) goto error; - if (!PyDict_CheckExact(st->name_mapping_2to3)) { + if (!_PyAnyDict_CheckExact(st->name_mapping_2to3)) { PyErr_Format(PyExc_RuntimeError, "_compat_pickle.NAME_MAPPING should be a dict, not %.200s", Py_TYPE(st->name_mapping_2to3)->tp_name); @@ -334,7 +335,7 @@ _Pickle_InitState(PickleState *st) PyObject_GetAttrString(compat_pickle, "IMPORT_MAPPING"); if (!st->import_mapping_2to3) goto error; - if (!PyDict_CheckExact(st->import_mapping_2to3)) { + if (!_PyAnyDict_CheckExact(st->import_mapping_2to3)) { PyErr_Format(PyExc_RuntimeError, "_compat_pickle.IMPORT_MAPPING should be a dict, " "not %.200s", Py_TYPE(st->import_mapping_2to3)->tp_name); @@ -592,6 +593,36 @@ Pdata_poplist(Pdata *self, Py_ssize_t start) return list; } +static PyObject * +Pdata_poplist2(PickleState *state, Pdata *self, Py_ssize_t start) +{ + if (start < self->fence) { + Pdata_stack_underflow(state, self); + return NULL; + } + + Py_ssize_t len = (Py_SIZE(self) - start) >> 1; + + PyObject *list = PyList_New(len); + if (list == NULL) { + return NULL; + } + + for (Py_ssize_t i = start, j = 0; j < len; i+=2, j++) { + PyObject *subtuple = PyTuple_New(2); + if (subtuple == NULL) { + return NULL; + } + + PyTuple_SET_ITEM(subtuple, 0, self->data[i]); + PyTuple_SET_ITEM(subtuple, 1, self->data[i+1]); + PyList_SET_ITEM(list, j, subtuple); + } + + Py_SET_SIZE(self, start); + return list; +} + typedef struct { PyObject *me_key; Py_ssize_t me_value; @@ -3445,6 +3476,64 @@ save_dict(PickleState *state, PicklerObject *self, PyObject *obj) return status; } +static int +save_frozendict(PickleState *state, PicklerObject *self, PyObject *obj) +{ + const char mark_op = MARK; + const char frozendict_op = FROZENDICT; + + if (self->fast && !fast_save_enter(self, obj)) { + return -1; + } + + if (self->proto < 4) { + PyObject *items = PyDict_Items(obj); + if (items == NULL) { + return -1; + } + + PyObject *reduce_value; + reduce_value = Py_BuildValue("(O(O))", (PyObject*)&PyFrozenDict_Type, + items); + Py_DECREF(items); + if (reduce_value == NULL) { + return -1; + } + + /* save_reduce() will memoize the object automatically. */ + int status = save_reduce(state, self, reduce_value, obj); + Py_DECREF(reduce_value); + return status; + } + + if (_Pickler_Write(self, &mark_op, 1) < 0) { + return -1; + } + + PyObject *key = NULL, *value = NULL; + Py_ssize_t pos = 0; + while (PyDict_Next(obj, &pos, &key, &value)) { + int res = save(state, self, key, 0); + if (res < 0) { + return -1; + } + + res = save(state, self, value, 0); + if (res < 0) { + return -1; + } + } + + if (_Pickler_Write(self, &frozendict_op, 1) < 0) { + return -1; + } + + if (memo_put(state, self, obj) < 0) { + return -1; + } + return 0; +} + static int save_set(PickleState *state, PicklerObject *self, PyObject *obj) { @@ -4411,6 +4500,10 @@ save(PickleState *st, PicklerObject *self, PyObject *obj, int pers_save) status = save_dict(st, self, obj); goto done; } + else if (type == &PyFrozenDict_Type) { + status = save_frozendict(st, self, obj); + goto done; + } else if (type == &PySet_Type) { status = save_set(st, self, obj); goto done; @@ -5831,6 +5924,30 @@ load_dict(PickleState *st, UnpicklerObject *self) return 0; } + +static int +load_frozendict(PickleState *st, UnpicklerObject *self) +{ + Py_ssize_t i = marker(st, self); + if (i < 0) { + return -1; + } + + PyObject *items = Pdata_poplist2(st, self->stack, i); + if (items == NULL) { + return -1; + } + + PyObject *frozendict = PyFrozenDict_New(items); + Py_DECREF(items); + if (frozendict == NULL) { + return -1; + } + + PDATA_PUSH(self->stack, frozendict, -1); + return 0; +} + static int load_frozenset(PickleState *state, UnpicklerObject *self) { @@ -6946,6 +7063,7 @@ load(PickleState *st, UnpicklerObject *self) OP(LIST, load_list) OP(EMPTY_DICT, load_empty_dict) OP(DICT, load_dict) + OP(FROZENDICT, load_frozendict) OP(EMPTY_SET, load_empty_set) OP(ADDITEMS, load_additems) OP(FROZENSET, load_frozenset) diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index 9557d68e759497..509cb498d026b0 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -1,11 +1,5 @@ /* Errno module */ -// Need limited C API version 3.13 for Py_mod_gil -#include "pyconfig.h" // Py_GIL_DISABLED -#ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030d0000 -#endif - #include "Python.h" #include // EPIPE @@ -96,10 +90,6 @@ errno_exec(PyObject *module) if (error_dict == NULL) { return -1; } - if (PyDict_SetItemString(module_dict, "errorcode", error_dict) < 0) { - Py_DECREF(error_dict); - return -1; - } /* Macro so I don't have to edit each and every line below... */ #define add_errcode(name, code, comment) \ @@ -947,6 +937,18 @@ errno_exec(PyObject *module) add_errcode("ENOTCAPABLE", ENOTCAPABLE, "Capabilities insufficient"); #endif + PyObject *frozendict = PyFrozenDict_New(error_dict); + if (frozendict == NULL) { + Py_DECREF(error_dict); + return -1; + } + if (PyDict_SetItemString(module_dict, "errorcode", frozendict) < 0) { + Py_DECREF(error_dict); + Py_DECREF(frozendict); + return -1; + } + Py_DECREF(frozendict); + Py_DECREF(error_dict); return 0; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 65eed151c2829d..f91f7b3263efb9 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -135,6 +135,10 @@ As a consequence of this, split keys have a maximum size of 16. #include "stringlib/eq.h" // unicode_eq() #include +// Forward declarations +static PyObject* frozendict_new(PyTypeObject *type, PyObject *args, + PyObject *kwds); + /*[clinic input] class dict "PyDictObject *" "&PyDict_Type" @@ -655,7 +659,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) do { if (!(expr)) { _PyObject_ASSERT_FAILED_MSG(op, Py_STRINGIFY(expr)); } } while (0) assert(op != NULL); - CHECK(PyDict_Check(op)); + CHECK(_PyAnyDict_Check(op)); PyDictObject *mp = (PyDictObject *)op; PyDictKeysObject *keys = mp->ma_keys; @@ -2277,7 +2281,7 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, static PyObject * dict_getitem(PyObject *op, PyObject *key, const char *warnmsg) { - if (!PyDict_Check(op)) { + if (!_PyAnyDict_Check(op)) { return NULL; } PyDictObject *mp = (PyDictObject *)op; @@ -2441,7 +2445,7 @@ _PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, Py int PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) { - if (!PyDict_Check(op)) { + if (!_PyAnyDict_Check(op)) { PyErr_BadInternalCall(); *result = NULL; return -1; @@ -2497,7 +2501,7 @@ PyDict_GetItemWithError(PyObject *op, PyObject *key) PyDictObject*mp = (PyDictObject *)op; PyObject *value; - if (!PyDict_Check(op)) { + if (!_PyAnyDict_Check(op)) { PyErr_BadInternalCall(); return NULL; } @@ -2658,7 +2662,7 @@ setitem_take2_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) assert(key); assert(value); - assert(PyDict_Check(mp)); + assert(_PyAnyDict_Check(mp)); Py_hash_t hash = _PyObject_HashFast(key); if (hash == -1) { dict_unhashable_type(key); @@ -2705,6 +2709,16 @@ PyDict_SetItem(PyObject *op, PyObject *key, PyObject *value) Py_NewRef(key), Py_NewRef(value)); } +static int +_PyAnyDict_SetItem(PyObject *op, PyObject *key, PyObject *value) +{ + assert(_PyAnyDict_Check(op)); + assert(key); + assert(value); + return _PyDict_SetItem_Take2((PyDictObject *)op, + Py_NewRef(key), Py_NewRef(value)); +} + static int setitem_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) { @@ -2992,7 +3006,7 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, PyObject *key, *value; Py_hash_t hash; - if (!PyDict_Check(op)) + if (!_PyAnyDict_Check(op)) return 0; mp = (PyDictObject *)op; @@ -3291,7 +3305,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) return NULL; } - if (PyDict_CheckExact(d)) { + if (_PyAnyDict_CheckExact(d)) { Py_BEGIN_CRITICAL_SECTION(d); while ((key = PyIter_Next(it)) != NULL) { status = setitem_lock_held((PyDictObject *)d, key, value); @@ -3764,7 +3778,7 @@ merge_from_seq2_lock_held(PyObject *d, PyObject *seq2, int override) PyObject *fast; /* item as a 2-tuple or 2-list */ assert(d != NULL); - assert(PyDict_Check(d)); + assert(_PyAnyDict_Check(d)); assert(seq2 != NULL); it = PyObject_GetIter(seq2); @@ -3962,7 +3976,7 @@ dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) * things quite efficiently. For the latter, we only require that * PyMapping_Keys() and PyObject_GetItem() be supported. */ - if (a == NULL || !PyDict_Check(a) || b == NULL) { + if (a == NULL || !_PyAnyDict_Check(a) || b == NULL) { PyErr_BadInternalCall(); return -1; } @@ -4112,13 +4126,19 @@ copy_lock_held(PyObject *o) PyObject *copy; PyDictObject *mp; PyInterpreterState *interp = _PyInterpreterState_GET(); + int frozendict = PyFrozenDict_Check(o); ASSERT_DICT_LOCKED(o); mp = (PyDictObject *)o; if (mp->ma_used == 0) { /* The dict is empty; just return a new dict. */ - return PyDict_New(); + if (frozendict) { + return PyFrozenDict_New(NULL); + } + else { + return PyDict_New(); + } } if (_PyDict_HasSplitTable(mp)) { @@ -4127,7 +4147,13 @@ copy_lock_held(PyObject *o) if (newvalues == NULL) { return PyErr_NoMemory(); } - split_copy = PyObject_GC_New(PyDictObject, &PyDict_Type); + if (frozendict) { + split_copy = (PyDictObject *)PyObject_GC_New(PyFrozenDictObject, + &PyFrozenDict_Type); + } + else { + split_copy = PyObject_GC_New(PyDictObject, &PyDict_Type); + } if (split_copy == NULL) { free_values(newvalues, false); return NULL; @@ -4140,13 +4166,18 @@ copy_lock_held(PyObject *o) split_copy->ma_used = mp->ma_used; split_copy->_ma_watcher_tag = 0; dictkeys_incref(mp->ma_keys); + if (frozendict) { + PyFrozenDictObject *frozen = (PyFrozenDictObject *)split_copy; + frozen->ma_hash = -1; + } _PyObject_GC_TRACK(split_copy); return (PyObject *)split_copy; } if (Py_TYPE(mp)->tp_iter == dict_iter && mp->ma_values == NULL && - (mp->ma_used >= (mp->ma_keys->dk_nentries * 2) / 3)) + (mp->ma_used >= (mp->ma_keys->dk_nentries * 2) / 3) && + !frozendict) { /* Use fast-copy if: @@ -4178,7 +4209,12 @@ copy_lock_held(PyObject *o) return (PyObject *)new; } - copy = PyDict_New(); + if (frozendict) { + copy = PyFrozenDict_New(NULL); + } + else { + copy = PyDict_New(); + } if (copy == NULL) return NULL; if (dict_merge(interp, copy, o, 1) == 0) @@ -4190,7 +4226,7 @@ copy_lock_held(PyObject *o) PyObject * PyDict_Copy(PyObject *o) { - if (o == NULL || !PyDict_Check(o)) { + if (o == NULL || !_PyAnyDict_Check(o)) { PyErr_BadInternalCall(); return NULL; } @@ -4297,7 +4333,7 @@ dict_richcompare(PyObject *v, PyObject *w, int op) int cmp; PyObject *res; - if (!PyDict_Check(v) || !PyDict_Check(w)) { + if (!_PyAnyDict_Check(v) || !_PyAnyDict_Check(w)) { res = Py_NotImplemented; } else if (op == Py_EQ || op == Py_NE) { @@ -4741,7 +4777,7 @@ dict___sizeof___impl(PyDictObject *self) static PyObject * dict_or(PyObject *self, PyObject *other) { - if (!PyDict_Check(self) || !PyDict_Check(other)) { + if (!_PyAnyDict_Check(self) || !_PyAnyDict_Check(other)) { Py_RETURN_NOTIMPLEMENTED; } PyObject *new = PyDict_Copy(self); @@ -4917,7 +4953,15 @@ dict_vectorcall(PyObject *type, PyObject * const*args, return NULL; } - PyObject *self = dict_new(_PyType_CAST(type), NULL, NULL); + PyObject *self; + if (Py_Is((PyTypeObject*)type, &PyFrozenDict_Type) + || PyType_IsSubtype((PyTypeObject*)type, &PyFrozenDict_Type)) + { + self = frozendict_new(_PyType_CAST(type), NULL, NULL); + } + else { + self = dict_new(_PyType_CAST(type), NULL, NULL); + } if (self == NULL) { return NULL; } @@ -4930,7 +4974,8 @@ dict_vectorcall(PyObject *type, PyObject * const*args, } if (kwnames != NULL) { for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(kwnames); i++) { - if (PyDict_SetItem(self, PyTuple_GET_ITEM(kwnames, i), args[i]) < 0) { + PyObject *key = PyTuple_GET_ITEM(kwnames, i); // borrowed + if (_PyAnyDict_SetItem(self, key, args[i]) < 0) { Py_DECREF(self); return NULL; } @@ -5003,6 +5048,7 @@ PyTypeObject PyDict_Type = { .tp_version_tag = _Py_TYPE_VERSION_DICT, }; + /* For backward compatibility with old dictionary interface */ PyObject * @@ -5197,7 +5243,7 @@ dictiter_iternextkey_lock_held(PyDictObject *d, PyObject *self) Py_ssize_t i; PyDictKeysObject *k; - assert (PyDict_Check(d)); + assert (_PyAnyDict_Check(d)); ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { @@ -5321,7 +5367,7 @@ dictiter_iternextvalue_lock_held(PyDictObject *d, PyObject *self) PyObject *value; Py_ssize_t i; - assert (PyDict_Check(d)); + assert (_PyAnyDict_Check(d)); ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { @@ -5443,7 +5489,7 @@ dictiter_iternextitem_lock_held(PyDictObject *d, PyObject *self, PyObject *key, *value; Py_ssize_t i; - assert (PyDict_Check(d)); + assert (_PyAnyDict_Check(d)); ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { @@ -5958,7 +6004,7 @@ _PyDictView_New(PyObject *dict, PyTypeObject *type) PyErr_BadInternalCall(); return NULL; } - if (!PyDict_Check(dict)) { + if (!_PyAnyDict_Check(dict)) { /* XXX Get rid of this restriction later */ PyErr_Format(PyExc_TypeError, "%s() requires a dict argument, not '%s'", @@ -6878,6 +6924,11 @@ _PyObject_MaterializeManagedDict(PyObject *obj) int _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value) { + if (!PyDict_Check(dict)) { + PyErr_BadInternalCall(); + return -1; + } + if (value == NULL) { Py_hash_t hash = _PyObject_HashFast(name); if (hash == -1) { @@ -7775,3 +7826,166 @@ _PyObject_InlineValuesConsistencyCheck(PyObject *obj) return 0; } #endif + +// --- frozendict implementation --------------------------------------------- + +static PyNumberMethods frozendict_as_number = { + .nb_or = dict_or, +}; + +static PyMappingMethods frozendict_as_mapping = { + dict_length, /*mp_length*/ + dict_subscript, /*mp_subscript*/ +}; + +static PyMethodDef frozendict_methods[] = { + DICT___CONTAINS___METHODDEF + {"__getitem__", dict_subscript, METH_O | METH_COEXIST, + getitem__doc__}, + DICT___SIZEOF___METHODDEF + DICT_GET_METHODDEF + DICT_KEYS_METHODDEF + DICT_ITEMS_METHODDEF + DICT_VALUES_METHODDEF + DICT_FROMKEYS_METHODDEF + DICT_COPY_METHODDEF + DICT___REVERSED___METHODDEF + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {NULL, NULL} /* sentinel */ +}; + + +static PyObject * +frozendict_repr(PyObject *self) +{ + PyObject *repr = dict_repr(self); + if (repr == NULL) { + return NULL; + } + assert(PyUnicode_Check(repr)); + + PyObject *res = PyUnicode_FromFormat("%s(%U)", + Py_TYPE(self)->tp_name, + repr); + Py_DECREF(repr); + return res; +} + +static Py_hash_t +frozendict_hash(PyObject *op) +{ + PyFrozenDictObject *self = _PyFrozenDictObject_CAST(op); + Py_hash_t hash = self->ma_hash; + if (hash != -1) { + return hash; + } + + PyObject *items = _PyDictView_New(op, &PyDictItems_Type); + if (items == NULL) { + return -1; + } + PyObject *frozenset = PyFrozenSet_New(items); + Py_DECREF(items); + if (frozenset == NULL) { + return -1; + } + + hash = PyObject_Hash(frozenset); + Py_DECREF(frozenset); + if (hash == -1) { + return -1; + } + + self->ma_hash = hash; + return hash; +} + + +static PyObject * +frozendict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *d = dict_new(type, args, kwds); + if (d == NULL) { + return NULL; + } + PyFrozenDictObject *self = _PyFrozenDictObject_CAST(d); + self->ma_hash = -1; + + if (args != NULL) { + if (dict_update_common(d, args, kwds, "frozendict") < 0) { + Py_DECREF(d); + return NULL; + } + } + else { + assert(kwds == NULL); + } + + return d; +} + + +PyObject* +PyFrozenDict_New(PyObject *iterable) +{ + if (iterable != NULL) { + PyObject *args = PyTuple_Pack(1, iterable); + if (args == NULL) { + return NULL; + } + PyObject *frozendict = frozendict_new(&PyFrozenDict_Type, args, NULL); + Py_DECREF(args); + return frozendict; + } + else { + PyObject *args = Py_GetConstantBorrowed(Py_CONSTANT_EMPTY_TUPLE); + return frozendict_new(&PyFrozenDict_Type, args, NULL); + } +} + + +PyTypeObject PyFrozenDict_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "frozendict", + sizeof(PyFrozenDictObject), + 0, + dict_dealloc, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_as_async */ + frozendict_repr, /* tp_repr */ + &frozendict_as_number, /* tp_as_number */ + &dict_as_sequence, /* tp_as_sequence */ + &frozendict_as_mapping, /* tp_as_mapping */ + frozendict_hash, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_BASETYPE | + _Py_TPFLAGS_MATCH_SELF | Py_TPFLAGS_MAPPING, /* tp_flags */ + dictionary_doc, /* tp_doc */ + dict_traverse, /* tp_traverse */ + dict_tp_clear, /* tp_clear */ + dict_richcompare, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + dict_iter, /* tp_iter */ + 0, /* tp_iternext */ + frozendict_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + dict_init, /* tp_init */ + _PyType_AllocNoTrack, /* tp_alloc */ + frozendict_new, /* tp_new */ + PyObject_GC_Del, /* tp_free */ + .tp_vectorcall = dict_vectorcall, + .tp_version_tag = _Py_TYPE_VERSION_DICT, +}; diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 9dee03bdb5ee55..1346e7598247cb 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1473,6 +1473,7 @@ module_dir(PyObject *self, PyObject *args) return result; } + static PyMethodDef module_methods[] = { {"__dir__", module_dir, METH_NOARGS, PyDoc_STR("__dir__() -> list\nspecialized dir() implementation")}, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 58228d6248522e..c47dcf54ec3eeb 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3998,7 +3998,7 @@ subtype_dict(PyObject *obj, void *context) int _PyObject_SetDict(PyObject *obj, PyObject *value) { - if (value != NULL && !PyDict_Check(value)) { + if (value != NULL && !_PyAnyDict_Check(value)) { PyErr_Format(PyExc_TypeError, "__dict__ must be set to a dictionary, " "not a '%.200s'", Py_TYPE(value)->tp_name); @@ -5999,7 +5999,7 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) for (Py_ssize_t i = 0; i < n; i++) { PyObject *base = PyTuple_GET_ITEM(mro, i); PyObject *dict = lookup_tp_dict(_PyType_CAST(base)); - assert(dict && PyDict_Check(dict)); + assert(dict && _PyAnyDict_Check(dict)); if (_PyDict_GetItemRef_KnownHash((PyDictObject *)dict, name, hash, &res) < 0) { *error = -1; goto done; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index c2d780ac9b9270..c45e61846794dd 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -3479,6 +3479,7 @@ _PyBuiltin_Init(PyInterpreterState *interp) SETBUILTIN("enumerate", &PyEnum_Type); SETBUILTIN("filter", &PyFilter_Type); SETBUILTIN("float", &PyFloat_Type); + SETBUILTIN("frozendict", &PyFrozenDict_Type); SETBUILTIN("frozenset", &PyFrozenSet_Type); SETBUILTIN("property", &PyProperty_Type); SETBUILTIN("int", &PyLong_Type); diff --git a/Python/marshal.c b/Python/marshal.c index 8b56de6575559c..e3fef318fa142a 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -67,6 +67,7 @@ module marshal #define TYPE_TUPLE '(' // See also TYPE_SMALL_TUPLE. #define TYPE_LIST '[' #define TYPE_DICT '{' +#define TYPE_FROZENDICT '}' #define TYPE_CODE 'c' #define TYPE_UNICODE 'u' #define TYPE_UNKNOWN '?' @@ -571,10 +572,15 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(PyList_GET_ITEM(v, i), p); } } - else if (PyDict_CheckExact(v)) { + else if (_PyAnyDict_CheckExact(v)) { Py_ssize_t pos; PyObject *key, *value; - W_TYPE(TYPE_DICT, p); + if (PyFrozenDict_CheckExact(v)) { + W_TYPE(TYPE_FROZENDICT, p); + } + else { + W_TYPE(TYPE_DICT, p); + } /* This one is NULL object terminated! */ pos = 0; while (PyDict_Next(v, &pos, &key, &value)) { @@ -1416,6 +1422,7 @@ r_object(RFILE *p) break; case TYPE_DICT: + case TYPE_FROZENDICT: v = PyDict_New(); R_REF(v); if (v == NULL) @@ -1439,7 +1446,16 @@ r_object(RFILE *p) Py_DECREF(val); } if (PyErr_Occurred()) { - Py_SETREF(v, NULL); + Py_CLEAR(v); + } + if (type == TYPE_FROZENDICT && v != NULL) { + PyObject *frozendict = PyFrozenDict_New(v); + if (frozendict != NULL) { + Py_SETREF(v, frozendict); + } + else { + Py_CLEAR(v); + } } retval = v; break; diff --git a/Tools/cases_generator/py_metadata_generator.py b/Tools/cases_generator/py_metadata_generator.py index 3ec06faf338488..73cdc9f3b31350 100644 --- a/Tools/cases_generator/py_metadata_generator.py +++ b/Tools/cases_generator/py_metadata_generator.py @@ -30,17 +30,17 @@ def get_specialized(analysis: Analysis) -> set[str]: def generate_specializations(analysis: Analysis, out: CWriter) -> None: - out.emit("_specializations = {\n") + out.emit("_specializations = frozendict(\n") for family in analysis.families.values(): - out.emit(f'"{family.name}": [\n') + out.emit(f'{family.name}= [\n') for member in family.members: out.emit(f' "{member.name}",\n') out.emit("],\n") - out.emit("}\n\n") + out.emit(")\n\n") def generate_specialized_opmap(analysis: Analysis, out: CWriter) -> None: - out.emit("_specialized_opmap = {\n") + out.emit("_specialized_opmap = frozendict(\n") names = [] for family in analysis.families.values(): for member in family.members: @@ -48,17 +48,17 @@ def generate_specialized_opmap(analysis: Analysis, out: CWriter) -> None: continue names.append(member.name) for name in sorted(names): - out.emit(f"'{name}': {analysis.opmap[name]},\n") - out.emit("}\n\n") + out.emit(f"{name}= {analysis.opmap[name]},\n") + out.emit(")\n\n") def generate_opmap(analysis: Analysis, out: CWriter) -> None: specialized = get_specialized(analysis) - out.emit("opmap = {\n") + out.emit("opmap = frozendict(\n") for inst, op in analysis.opmap.items(): if inst not in specialized: - out.emit(f"'{inst}': {analysis.opmap[inst]},\n") - out.emit("}\n\n") + out.emit(f"{inst}= {analysis.opmap[inst]},\n") + out.emit(")\n\n") def generate_py_metadata( From 49aca48431fb881811a9d2220f616cf249d30e59 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 13 Nov 2025 14:21:30 +0100 Subject: [PATCH 54/58] Update Tools/build/generate_token.py for token.py --- Tools/build/generate_token.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Tools/build/generate_token.py b/Tools/build/generate_token.py index 9ee5ec86e75d47..1ff7a7c95f6af4 100755 --- a/Tools/build/generate_token.py +++ b/Tools/build/generate_token.py @@ -269,14 +269,15 @@ def make_rst(infile, outfile='Doc/library/token-list.inc', # Special definitions for cooperation with parser NT_OFFSET = %d -tok_name = {value: name - for name, value in globals().items() - if isinstance(value, int) and not name.startswith('_')} +tok_name = frozendict({ + value: name + for name, value in globals().items() + if isinstance(value, int) and not name.startswith('_')}) __all__.extend(tok_name.values()) -EXACT_TOKEN_TYPES = { +EXACT_TOKEN_TYPES = frozendict({ %s -} +}) def ISTERMINAL(x: int) -> bool: return x < NT_OFFSET From 3299299ffaa0c8fbe6c2a5298af59524f20a5900 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 13 Nov 2025 14:22:50 +0100 Subject: [PATCH 55/58] Try to fix build on Ubuntu and macOS --- Objects/dictobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index f91f7b3263efb9..292d8e2f6e0824 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5595,7 +5595,7 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, Py_ssize_t i; PyDictKeysObject *k; - assert (PyDict_Check(d)); + assert (_PyAnyDict_Check(d)); if (di->di_used != _Py_atomic_load_ssize_relaxed(&d->ma_used)) { PyErr_SetString(PyExc_RuntimeError, From 18ce520e8bbbdc05249adc8c063c62f246b83067 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 13 Nov 2025 14:39:22 +0100 Subject: [PATCH 56/58] Fix make check-c-globals --- Tools/c-analyzer/cpython/ignored.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 4621ad250f4633..8811e0e458186f 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -760,6 +760,7 @@ Modules/clinic/md5module.c.h _md5_md5 _keywords - Modules/clinic/grpmodule.c.h grp_getgrgid _keywords - Modules/clinic/grpmodule.c.h grp_getgrnam _keywords - Objects/object.c - constants static PyObject*[] +Objects/dictobject.c - PyFrozenDict_Type - ## False positives From 1a72eec987345c43e59a31930cf3f8626e520d6c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 13 Nov 2025 15:37:59 +0100 Subject: [PATCH 57/58] Add PyFrozenDict_Type to static_types --- Include/internal/pycore_typeobject.h | 1 + Objects/dictobject.c | 2 +- Objects/object.c | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 3661f171e2b013..a9039cfcd45473 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -25,6 +25,7 @@ extern "C" { #define _Py_TYPE_VERSION_BYTEARRAY 9 #define _Py_TYPE_VERSION_BYTES 10 #define _Py_TYPE_VERSION_COMPLEX 11 +#define _Py_TYPE_VERSION_FROZENDICT 12 #define _Py_TYPE_VERSION_NEXT 16 diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 292d8e2f6e0824..13f810d1af12c6 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -7987,5 +7987,5 @@ PyTypeObject PyFrozenDict_Type = { frozendict_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ .tp_vectorcall = dict_vectorcall, - .tp_version_tag = _Py_TYPE_VERSION_DICT, + .tp_version_tag = _Py_TYPE_VERSION_FROZENDICT, }; diff --git a/Objects/object.c b/Objects/object.c index 0540112d7d2acf..98eb0ab5a20f7a 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2476,8 +2476,9 @@ static PyTypeObject* static_types[] = { &PyEnum_Type, &PyFilter_Type, &PyFloat_Type, - &PyFrame_Type, &PyFrameLocalsProxy_Type, + &PyFrame_Type, + &PyFrozenDict_Type, &PyFrozenSet_Type, &PyFunction_Type, &PyGen_Type, From 9d6ca58b0de297a9aaa3fdef849717936a201311 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 13 Nov 2025 16:28:07 +0100 Subject: [PATCH 58/58] Update Doc/library/stdtypes.rst Co-authored-by: Nice Zombies --- Doc/library/stdtypes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index d74345416f3644..c6a9893f5b57a8 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4926,8 +4926,8 @@ Mapping Types --- :class:`dict`, :class:`frozendict` pair: built-in function; len A :term:`mapping` object maps :term:`hashable` values to arbitrary objects. -Mappings are mutable objects. There is currently two standard mapping -types, the :dfn:`dictionary` and :class:`frozendict`. +There are currently two standard mapping types, the :dfn:`dictionary` and +:class:`frozendict`. (For other containers see the built-in :class:`list`, :class:`set`, and :class:`tuple` classes, and the :mod:`collections` module.)