gh-92810: Reduce memory usage by ABCMeta.__subclasscheck__

Doc/whatsnew/3.15.rst

@@ -413,6 +413,33 @@ arguments (:pep:`791`).
 Improved modules
 ================
 
+abc
+---
+
+* Reduce memory usage of :func:`issubclass` checks for :class:`abc.ABCMeta` subclasses.
+
+  :class:`abc.ABCMeta` subclasses can trigger downstream checks:
+
+  ``issubclass(Some, Class)`` -> ``issubclass(Some, Parent)`` -> ``issubclass(Some, Top)``
+  (nothing found) -> ``issubclass(Some, Class.__subclasses__)`` -> ``issubclass(Some, SubClass)``
+  -> ``issubclass(Some, SubClass.__subclasses__)`` -> ...
+
+  Due to caching of ``issubclass`` result within each ABC class,
+  this could lead to memory bloat in large class trees, e.g. thousands of subclasses.
+
+  Now :meth:`!abc.ABCMeta.register` recursively calls ``Parent.register(subclass)``,
+  ``Top.register(subclass)`` and so on for all base classes, so downstream checks are not needed.
+  Also :meth:`!abc.ABCMeta.__new__` checks if ``__subclasses__`` method is present within class,
+  and if not, disables recursive downstream checks in :func:`issubclass`
+  for specific abstract class and all its children.
+
+  This reduces both the number of checks, and the RAM usage by internal caches:
+
+  ``issubclass(Some, Class)`` -> ``issubclass(Some, Parent)`` -> ``issubclass(Some, Top)``
+  (nothing found, stops here)
+
+  (Contributed by Maxim Martynov in :gh:`92810`.)
+
 argparse
 --------
 

Include/internal/pycore_pyatomic_ft_wrappers.h

@@ -45,6 +45,8 @@ extern "C" {
     _Py_atomic_load_uint16_relaxed(&value)
 #define FT_ATOMIC_LOAD_UINT32_RELAXED(value) \
     _Py_atomic_load_uint32_relaxed(&value)
+#define FT_ATOMIC_LOAD_UINT64_RELAXED(value) \
+    _Py_atomic_load_uint64_relaxed(&value)
 #define FT_ATOMIC_LOAD_ULONG_RELAXED(value) \
     _Py_atomic_load_ulong_relaxed(&value)
 #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \
@@ -61,6 +63,8 @@ extern "C" {
     _Py_atomic_store_uint16_relaxed(&value, new_value)
 #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \
     _Py_atomic_store_uint32_relaxed(&value, new_value)
+#define FT_ATOMIC_STORE_UINT64_RELAXED(value, new_value) \
+    _Py_atomic_store_uint64_relaxed(&value, new_value)
 #define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) \
     _Py_atomic_store_char_relaxed(&value, new_value)
 #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) \
@@ -115,6 +119,8 @@ extern "C" {
     _Py_atomic_load_ullong_relaxed(&value)
 #define FT_ATOMIC_ADD_SSIZE(value, new_value) \
     (void)_Py_atomic_add_ssize(&value, new_value)
+#define FT_ATOMIC_ADD_UINT64(value, new_value) \
+    (void)_Py_atomic_add_uint64(&value, new_value)
 #define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock)
 #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock)
 
@@ -132,6 +138,7 @@ extern "C" {
 #define FT_ATOMIC_LOAD_UINT8_RELAXED(value) value
 #define FT_ATOMIC_LOAD_UINT16_RELAXED(value) value
 #define FT_ATOMIC_LOAD_UINT32_RELAXED(value) value
+#define FT_ATOMIC_LOAD_UINT64_RELAXED(value) value
 #define FT_ATOMIC_LOAD_ULONG_RELAXED(value) value
 #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value
 #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value
@@ -140,6 +147,7 @@ extern "C" {
 #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value
 #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value
 #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value
+#define FT_ATOMIC_STORE_UINT64_RELAXED(value, new_value) value = new_value
 #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) value
 #define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) value = new_value
 #define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) value
@@ -167,6 +175,7 @@ extern "C" {
 #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value
 #define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value
 #define FT_ATOMIC_ADD_SSIZE(value, new_value) (void)(value += new_value)
+#define FT_ATOMIC_ADD_UINT64(value, new_value) (void)(value += new_value)
 #define FT_MUTEX_LOCK(lock) do {} while (0)
 #define FT_MUTEX_UNLOCK(lock) do {} while (0)
 

Lib/_py_abc.py

@@ -49,6 +49,14 @@ def __new__(mcls, name, bases, namespace, /, **kwargs):
         cls._abc_cache = WeakSet()
         cls._abc_negative_cache = WeakSet()
         cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter
+
+        # Performance optimization for common case
+        cls._abc_should_check_subclasses = False
+        if "__subclasses__" in namespace:
+            cls._abc_should_check_subclasses = True
+            for base in bases:
+                if hasattr(base, "_abc_should_check_subclasses"):
+                    base._abc_should_check_subclasses = True
         return cls
 
     def register(cls, subclass):
@@ -65,8 +73,20 @@ def register(cls, subclass):
         if issubclass(cls, subclass):
             # This would create a cycle, which is bad for the algorithm below
             raise RuntimeError("Refusing to create an inheritance cycle")
+        # Add registry entry
         cls._abc_registry.add(subclass)
         ABCMeta._abc_invalidation_counter += 1  # Invalidate negative cache
+        # Recursively register the subclass in all ABC bases,
+        # to avoid recursive lookups down the class tree.
+        # >>> class Ancestor1(ABC): pass
+        # >>> class Ancestor2(Ancestor1): pass
+        # >>> class Other: pass
+        # >>> Ancestor2.register(Other)  # calls Ancestor1.register(Other)
+        # >>> issubclass(Other, Ancestor2) is True
+        # >>> issubclass(Other, Ancestor1) is True  # already in registry
+        for base in cls.__bases__:
+            if hasattr(base, "_abc_registry"):
+                base.register(subclass)
         return subclass
 
     def _dump_registry(cls, file=None):
@@ -137,11 +157,16 @@ def __subclasscheck__(cls, subclass):
             if issubclass(subclass, rcls):
                 cls._abc_cache.add(subclass)
                 return True
-        # Check if it's a subclass of a subclass (recursive)
-        for scls in cls.__subclasses__():
-            if issubclass(subclass, scls):
-                cls._abc_cache.add(subclass)
-                return True
+        # Check if it's a subclass of a subclass (recursive).
+        # If __subclasses__ contain only ABCs,
+        # calling issubclass(...) will trigger the same __subclasscheck__
+        # on *every* element of class inheritance tree.
+        # Performing that only in resence of `def __subclasses__()` classmethod
+        if cls._abc_should_check_subclasses:
+            for scls in cls.__subclasses__():
+                if issubclass(subclass, scls):
+                    cls._abc_cache.add(subclass)
+                    return True
         # No dice; update negative cache
         cls._abc_negative_cache.add(subclass)
         return False

Lib/abc.py

@@ -104,7 +104,7 @@ class ABCMeta(type):
         """
         def __new__(mcls, name, bases, namespace, /, **kwargs):
             cls = super().__new__(mcls, name, bases, namespace, **kwargs)
-            _abc_init(cls)
+            _abc_init(cls, bases, namespace)
             return cls
 
         def register(cls, subclass):