GC executors

Author: Fidget-Spinner

Include/internal/pycore_optimizer.h

@@ -82,11 +82,12 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateDependencyLockHeld(PyInterpreterState *
 PyAPI_FUNC(void) _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation);
 PyAPI_FUNC(void) _Py_Executors_InvalidateAllLockHeld(PyInterpreterState *interp, int is_invalidation);
 PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyThreadState *tstate);
+PyAPI_FUNC(void) _Py_Executors_InvalidateColdGC(PyInterpreterState *interp);
 #else
 #  define _Py_Executors_InvalidateDependency(A, B, C) ((void)0)
 #  define _Py_Executors_InvalidateAll(A, B) ((void)0)
 #  define _Py_Executors_InvalidateCold(A) ((void)0)
-
+#  define _Py_Executors_InvalidateColdGC(A) ((void)0)
 #endif
 
 // Used as the threshold to trigger executor invalidation when

Python/gc.c

@@ -12,6 +12,7 @@
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "pycore_tuple.h"         // _PyTuple_MaybeUntrack()
 #include "pycore_weakref.h"       // _PyWeakref_ClearRef()
+#include "pycore_optimizer.h"     // _Py_Executors_InvalidateColdGC
 
 #include "pydtrace.h"
 
@@ -1735,6 +1736,7 @@ gc_collect_full(PyThreadState *tstate,
     gcstate->old[1].count = 0;
     completed_scavenge(gcstate);
     _PyGC_ClearAllFreeLists(tstate->interp);
+    _Py_Executors_InvalidateColdGC(tstate->interp);
     validate_spaces(gcstate);
     add_stats(gcstate, 2, stats);
 }

Python/gc_free_threading.c

@@ -10,6 +10,7 @@
 #include "pycore_interp.h"        // PyInterpreterState.gc
 #include "pycore_interpframe.h"   // _PyFrame_GetLocalsArray()
 #include "pycore_object_alloc.h"  // _PyObject_MallocWithType()
+#include "pycore_optimizer.h"
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "pycore_tstate.h"        // _PyThreadStateImpl
 #include "pycore_tuple.h"         // _PyTuple_MaybeUntrack()
@@ -2310,6 +2311,8 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state,
     }
     _PyEval_StartTheWorld(interp);
 
+    _Py_Executors_InvalidateColdGC(interp);
+
     if (err < 0) {
         cleanup_worklist(&state->unreachable);
         cleanup_worklist(&state->legacy_finalizers);

Python/optimizer.c

@@ -1797,8 +1797,8 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation)
 // Unlike _PyExecutor_InvalidateDependency, this is not for correctness but memory savings.
 // Thus there is no need to lock the runtime or traverse everything. We simply make a
 // best-effort attempt to clean things up.
-void
-_Py_Executors_InvalidateCold(PyThreadState *tstate)
+PyObject *
+_Py_Executors_CollectCold(PyThreadState *tstate)
 {
     /* Walk the list of executors */
     /* TO DO -- Use a tree to avoid traversing as many objects */
@@ -1813,8 +1813,8 @@ _Py_Executors_InvalidateCold(PyThreadState *tstate)
         _PyExecutorObject *next = exec->vm_data.links.next;
 
         if (!exec->vm_data.warm || !exec->vm_data.valid) {
+            FT_ATOMIC_STORE_UINT8(exec->vm_data.valid, 0);
             if (PyList_Append(invalidate, (PyObject *)exec) < 0) {
-                FT_ATOMIC_STORE_UINT8(exec->vm_data.valid, 0);
                 goto error;
             }
         }
@@ -1824,19 +1824,70 @@ _Py_Executors_InvalidateCold(PyThreadState *tstate)
 
         exec = next;
     }
+    return invalidate;
+error:
+    PyErr_Clear();
+    Py_XDECREF(invalidate);
+    return NULL;
+}
+
+void
+_Py_Executors_ClearColdList(PyObject *invalidate)
+{
+    if (invalidate == NULL) {
+        return;
+    }
     for (Py_ssize_t i = 0; i < PyList_GET_SIZE(invalidate); i++) {
         PyObject *exec = PyList_GET_ITEM(invalidate, i);
         executor_clear(exec);
     }
     Py_DECREF(invalidate);
+}
+
+void
+_Py_Executors_InvalidateCold(PyThreadState *tstate)
+{
+    _Py_Executors_ClearColdList(_Py_Executors_CollectCold(tstate));
+}
+
+void
+_Py_Executors_InvalidateColdGC(PyInterpreterState *interp)
+{
+    PyObject *invalidate = PyList_New(0);
+    if (invalidate == NULL) {
+        PyErr_Clear();
+        return;
+    }
+    _PyEval_StopTheWorld(interp);
+    HEAD_LOCK(interp->runtime);
+    _Py_FOR_EACH_TSTATE_UNLOCKED(interp, p)
+    {
+        PyObject *more_invalidate = _Py_Executors_CollectCold(p);
+        if (more_invalidate == NULL) {
+            goto error;
+        }
+        int err = PyList_Extend(invalidate, more_invalidate);
+        Py_DECREF(more_invalidate);
+        if (err) {
+            goto error;
+        }
+    }
+    HEAD_UNLOCK(interp->runtime);
+    _PyEval_StartTheWorld(interp);
+    // We can only clear the list after unlocking the runtime.
+    // Otherwise, it may deadlock.
+    _Py_Executors_ClearColdList(invalidate);
     return;
 error:
     PyErr_Clear();
-    Py_XDECREF(invalidate);
-    // If we're truly out of memory, wiping out everything is a fine fallback
-    _Py_Executors_InvalidateAll(_PyInterpreterState_GET(), 0);
+    Py_DECREF(invalidate);
+    // Invalidate all the executors if we run out of memory.
+    // This means the next run will remove all executors.
+    _Py_Executors_InvalidateAllLockHeld(interp, 0);
+    HEAD_UNLOCK(interp->runtime);
 }
 
+
 static void
 write_str(PyObject *str, FILE *out)
 {