Merge branch 'main' into tstrings

Author: lysnikolaou

.gitignore

@@ -143,6 +143,7 @@ Tools/unicode/data/
 /profile-clean-stamp
 /profile-run-stamp
 /profile-bolt-stamp
+/profile-gen-stamp
 /pybuilddir.txt
 /pyconfig.h
 /python-config

Doc/c-api/unicode.rst

@@ -33,8 +33,14 @@ Python:
 
 .. c:var:: PyTypeObject PyUnicode_Type
 
-   This instance of :c:type:`PyTypeObject` represents the Python Unicode type.  It
-   is exposed to Python code as :py:class:`str`.
+   This instance of :c:type:`PyTypeObject` represents the Python Unicode type.
+   It is exposed to Python code as :py:class:`str`.
+
+
+.. c:var:: PyTypeObject PyUnicodeIter_Type
+
+   This instance of :c:type:`PyTypeObject` represents the Python Unicode
+   iterator type. It is used to iterate over Unicode string objects.
 
 
 .. c:type:: Py_UCS4

Doc/deprecations/pending-removal-in-future.rst

@@ -13,8 +13,6 @@ although there is currently no date scheduled for their removal.
     deprecated.
   * The :class:`argparse.FileType` type converter is deprecated.
 
-* :mod:`array`'s ``'u'`` format code (:gh:`57281`)
-
 * :mod:`builtins`:
 
   * ``bool(NotImplemented)``.
@@ -49,6 +47,8 @@ although there is currently no date scheduled for their removal.
   :data:`calendar.FEBRUARY`.
   (Contributed by Prince Roshan in :gh:`103636`.)
 
+* :mod:`codecs`: use :func:`open` instead of :func:`codecs.open`. (:gh:`133038`)
+
 * :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method
   instead.
 

Doc/library/codecs.rst

@@ -208,6 +208,10 @@ wider range of codecs when working with binary files:
    .. versionchanged:: 3.11
       The ``'U'`` mode has been removed.
 
+   .. deprecated:: next
+
+      :func:`codecs.open` has been superseded by :func:`open`.
+
 
 .. function:: EncodedFile(file, data_encoding, file_encoding=None, errors='strict')
 

Doc/library/pdb.rst

@@ -188,6 +188,21 @@ slightly different way:
    .. versionadded:: 3.14
       The *commands* argument.
 
+
+.. awaitablefunction:: set_trace_async(*, header=None, commands=None)
+
+   async version of :func:`set_trace`. This function should be used inside an
+   async function with :keyword:`await`.
+
+   .. code-block:: python
+
+      async def f():
+          await pdb.set_trace_async()
+
+   :keyword:`await` statements are supported if the debugger is invoked by this function.
+
+   .. versionadded:: 3.14
+
 .. function:: post_mortem(t=None)
 
    Enter post-mortem debugging of the given exception or

Doc/whatsnew/3.14.rst

@@ -1239,6 +1239,11 @@ pdb
   backend by default, which is configurable.
   (Contributed by Tian Gao in :gh:`124533`.)
 
+* :func:`pdb.set_trace_async` is added to support debugging asyncio
+  coroutines. :keyword:`await` statements are supported with this
+  function.
+  (Contributed by Tian Gao in :gh:`132576`.)
+
 
 pickle
 ------
@@ -1663,6 +1668,10 @@ Deprecated
   as a single positional argument.
   (Contributed by Serhiy Storchaka in :gh:`109218`.)
 
+* :mod:`codecs`:
+  :func:`codecs.open` is now deprecated. Use :func:`open` instead.
+  (Contributed by Inada Naoki in :gh:`133036`.)
+
 * :mod:`functools`:
   Calling the Python implementation of :func:`functools.reduce` with *function*
   or *sequence* as keyword arguments is now deprecated.
@@ -1708,6 +1717,13 @@ Deprecated
   Deprecate :meth:`symtable.Class.get_methods` due to the lack of interest.
   (Contributed by Bénédikt Tran in :gh:`119698`.)
 
+* :mod:`tkinter`:
+  The :class:`!tkinter.Variable` methods :meth:`!trace_variable`,
+  :meth:`!trace_vdelete` and :meth:`!trace_vinfo` are now deprecated.
+  Use :meth:`!trace_add`, :meth:`!trace_remove` and :meth:`!trace_info`
+  instead.
+  (Contributed by Serhiy Storchaka in :gh:`120220`.)
+
 * :mod:`urllib.parse`:
   Accepting objects with false values (like ``0`` and ``[]``) except empty
   strings, byte-like objects and ``None`` in :mod:`urllib.parse` functions
@@ -2351,3 +2367,10 @@ Removed
 * Remove the private ``_Py_InitializeMain()`` function. It was a
   :term:`provisional API` added to Python 3.8 by :pep:`587`.
   (Contributed by Victor Stinner in :gh:`129033`.)
+
+* The undocumented APIs :c:macro:`!Py_C_RECURSION_LIMIT` and
+  :c:member:`!PyThreadState.c_recursion_remaining`, added in 3.13, are removed
+  without a deprecation period.
+  Please use :c:func:`Py_EnterRecursiveCall` to guard against runaway recursion
+  in C code.
+  (Removed in :gh:`133079`, see also :gh:`130396`.)

Include/cpython/pystate.h

@@ -118,8 +118,6 @@ struct _ts {
 
     int py_recursion_remaining;
     int py_recursion_limit;
-
-    int c_recursion_remaining; /* Retained for backwards compatibility. Do not use */
     int recursion_headroom; /* Allow 50 more calls to handle any errors. */
 
     /* 'tracing' keeps track of the execution depth when tracing/profiling.
@@ -210,8 +208,6 @@ struct _ts {
     _PyRemoteDebuggerSupport remote_debugger_support;
 };
 
-# define Py_C_RECURSION_LIMIT 5000
-
 /* other API */
 
 /* Similar to PyThreadState_Get(), but don't issue a fatal error

Include/internal/pycore_interpframe.h

@@ -18,6 +18,7 @@ extern "C" {
     ((int)((IF)->instr_ptr - _PyFrame_GetBytecode((IF))))
 
 static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) {
+    assert(!PyStackRef_IsNull(f->f_executable));
     PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable);
     assert(PyCode_Check(executable));
     return (PyCodeObject *)executable;

Include/internal/pycore_stackref.h

@@ -63,11 +63,13 @@ extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _P
 
 static const _PyStackRef PyStackRef_NULL = { .index = 0 };
 
-#define PyStackRef_None ((_PyStackRef){ .index = 1 } )
-#define PyStackRef_False ((_PyStackRef){ .index = 2 })
-#define PyStackRef_True ((_PyStackRef){ .index = 3 })
+// Use the first 3 even numbers for None, True and False.
+// Odd numbers are reserved for (tagged) integers
+#define PyStackRef_None ((_PyStackRef){ .index = 2 } )
+#define PyStackRef_False ((_PyStackRef){ .index = 4 })
+#define PyStackRef_True ((_PyStackRef){ .index = 6 })
 
-#define LAST_PREDEFINED_STACKREF_INDEX 3
+#define INITIAL_STACKREF_INDEX 8
 
 static inline int
 PyStackRef_IsNull(_PyStackRef ref)
@@ -96,6 +98,7 @@ PyStackRef_IsNone(_PyStackRef ref)
 static inline PyObject *
 _PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumber)
 {
+    assert((ref.index & 1) == 0);
     _Py_stackref_record_borrow(ref, filename, linenumber);
     return _Py_stackref_get_object(ref);
 }
@@ -132,31 +135,45 @@ _PyStackRef_FromPyObjectImmortal(PyObject *obj, const char *filename, int linenu
 }
 #define PyStackRef_FromPyObjectImmortal(obj) _PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj), __FILE__, __LINE__)
 
+static inline bool
+PyStackRef_IsTaggedInt(_PyStackRef ref)
+{
+    return (ref.index & 1) == 1;
+}
+
 static inline void
 _PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber)
 {
+    if (PyStackRef_IsTaggedInt(ref)) {
+        return;
+    }
     PyObject *obj = _Py_stackref_close(ref, filename, linenumber);
     Py_DECREF(obj);
 }
 #define PyStackRef_CLOSE(REF) _PyStackRef_CLOSE((REF), __FILE__, __LINE__)
 
+
 static inline void
 _PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber)
 {
     if (PyStackRef_IsNull(ref)) {
         return;
     }
-    PyObject *obj = _Py_stackref_close(ref, filename, linenumber);
-    Py_DECREF(obj);
+    _PyStackRef_CLOSE(ref, filename, linenumber);
 }
 #define PyStackRef_XCLOSE(REF) _PyStackRef_XCLOSE((REF), __FILE__, __LINE__)
 
 static inline _PyStackRef
 _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber)
 {
-    PyObject *obj = _Py_stackref_get_object(ref);
-    Py_INCREF(obj);
-    return _Py_stackref_create(obj, filename, linenumber);
+    if (PyStackRef_IsTaggedInt(ref)) {
+        return ref;
+    }
+    else {
+        PyObject *obj = _Py_stackref_get_object(ref);
+        Py_INCREF(obj);
+        return _Py_stackref_create(obj, filename, linenumber);
+    }
 }
 #define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__)
 
@@ -210,8 +227,40 @@ _PyStackRef_FromPyObjectNewMortal(PyObject *obj, const char *filename, int linen
 
 extern int PyStackRef_Is(_PyStackRef a, _PyStackRef b);
 
+extern bool PyStackRef_IsTaggedInt(_PyStackRef ref);
+
+extern intptr_t PyStackRef_UntagInt(_PyStackRef ref);
+
+extern _PyStackRef PyStackRef_TagInt(intptr_t i);
+
+extern bool
+PyStackRef_IsNullOrInt(_PyStackRef ref);
+
 #else
 
+#define Py_INT_TAG 3
+
+static inline bool
+PyStackRef_IsTaggedInt(_PyStackRef i)
+{
+    return (i.bits & Py_INT_TAG) == Py_INT_TAG;
+}
+
+static inline _PyStackRef
+PyStackRef_TagInt(intptr_t i)
+{
+    assert(Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (i << 2), 2) == i);
+    return (_PyStackRef){ .bits = ((((uintptr_t)i) << 2) | Py_INT_TAG) };
+}
+
+static inline intptr_t
+PyStackRef_UntagInt(_PyStackRef i)
+{
+    assert((i.bits & Py_INT_TAG) == Py_INT_TAG);
+    intptr_t val = (intptr_t)i.bits;
+    return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, val, 2);
+}
+
 
 #ifdef Py_GIL_DISABLED
 
@@ -232,6 +281,8 @@ static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED};
 #define PyStackRef_IsTrue(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_True)
 #define PyStackRef_IsFalse(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_False)
 
+#define PyStackRef_IsNullOrInt(stackref) (PyStackRef_IsNull(stackref) || PyStackRef_IsTaggedInt(stackref))
+
 static inline PyObject *
 PyStackRef_AsPyObjectBorrow(_PyStackRef stackref)
 {
@@ -451,6 +502,7 @@ PyStackRef_RefcountOnObject(_PyStackRef ref)
 static inline PyObject *
 PyStackRef_AsPyObjectBorrow(_PyStackRef ref)
 {
+    assert(!PyStackRef_IsTaggedInt(ref));
     return BITS_TO_PTR_MASKED(ref);
 }
 
@@ -587,6 +639,12 @@ PyStackRef_CLOSE(_PyStackRef ref)
 }
 #endif
 
+static inline bool
+PyStackRef_IsNullOrInt(_PyStackRef ref)
+{
+    return PyStackRef_IsNull(ref) || PyStackRef_IsTaggedInt(ref);
+}
+
 static inline void
 PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct)
 {
@@ -726,7 +784,7 @@ _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out)
 // Like Py_VISIT but for _PyStackRef fields
 #define _Py_VISIT_STACKREF(ref)                                         \
     do {                                                                \
-        if (!PyStackRef_IsNull(ref)) {                                  \
+        if (!PyStackRef_IsNullOrInt(ref)) {                             \
             int vret = _PyGC_VisitStackRef(&(ref), visit, arg);         \
             if (vret)                                                   \
                 return vret;                                            \

InternalDocs/garbage_collector.md

@@ -622,7 +622,7 @@ used as follows.  If the buffer has reached the maximum size, new object
 pointers found while following references are pushed to the stack, rather than
 put in the buffer.  When dequeuing objects from the buffer, we will "prime" the
 buffer if the current length drops below the low threshold.  Priming means
-popping objects from the stack and enqueing them into the buffer.  While
+popping objects from the stack and enqueuing them into the buffer.  While
 priming, we will fill it only until the high threshold is reached.
 
 To measure the effectiveness of the buffer, some benchmark programs were run