Re-enable the type/function reverse cache

Author: Fidget-Spinner

Lib/test/test_capi/test_opt.py

@@ -64,7 +64,6 @@ def iter_opnames(ex):
 def get_opnames(ex):
     return list(iter_opnames(ex))
 
-FT_TEST_DISABLED_REVERSE_TYPE_CACHE = "FT build reverse type cache is disabled for now"
 
 @requires_jit_enabled
 class TestExecutorInvalidation(unittest.TestCase):
@@ -1186,7 +1185,6 @@ def test_modified_local_is_seen_by_optimized_code(self):
         self.assertIs(type(s), float)
         self.assertEqual(s, 1024.0)
 
-    @unittest.skipIf(Py_GIL_DISABLED, FT_TEST_DISABLED_REVERSE_TYPE_CACHE)
     def test_guard_type_version_removed(self):
         def thing(a):
             x = 0
@@ -1205,7 +1203,6 @@ class Foo:
         guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION")
         self.assertEqual(guard_type_version_count, 1)
 
-    @unittest.skipIf(Py_GIL_DISABLED, FT_TEST_DISABLED_REVERSE_TYPE_CACHE)
     def test_guard_type_version_removed_inlined(self):
         """
         Verify that the guard type version if we have an inlined function
@@ -1232,7 +1229,6 @@ class Foo:
         guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION")
         self.assertEqual(guard_type_version_count, 1)
 
-    @unittest.skipIf(Py_GIL_DISABLED, FT_TEST_DISABLED_REVERSE_TYPE_CACHE)
     def test_guard_type_version_removed_invalidation(self):
 
         def thing(a):
@@ -1263,7 +1259,6 @@ class Bar:
         self.assertEqual(opnames[:load_attr_top].count("_GUARD_TYPE_VERSION"), 1)
         self.assertEqual(opnames[call:load_attr_bottom].count("_CHECK_VALIDITY"), 2)
 
-    @unittest.skipIf(Py_GIL_DISABLED, FT_TEST_DISABLED_REVERSE_TYPE_CACHE)
     def test_guard_type_version_removed_escaping(self):
 
         def thing(a):
@@ -1287,7 +1282,6 @@ class Foo:
         self.assertEqual(opnames[:load_attr_top].count("_GUARD_TYPE_VERSION"), 1)
         self.assertEqual(opnames[call:load_attr_bottom].count("_CHECK_VALIDITY"), 2)
 
-    @unittest.skipIf(Py_GIL_DISABLED, "FT build only specializes on deferred methods/functions/classes")
     def test_guard_type_version_executor_invalidated(self):
         """
         Verify that the executor is invalided on a type change.
@@ -2334,7 +2328,6 @@ def testfunc(n):
         self.assertNotIn("_TO_BOOL_BOOL", uops)
         self.assertIn("_GUARD_IS_TRUE_POP", uops)
 
-    @unittest.skipIf(Py_GIL_DISABLED, "FT build only specializes on deferred methods/functions/classes")
     def test_set_type_version_sets_type(self):
         class C:
             A = 1
@@ -2367,7 +2360,6 @@ def testfunc(n):
         self.assertNotIn("_LOAD_SMALL_INT", uops)
         self.assertIn("_LOAD_CONST_INLINE_BORROW", uops)
 
-    @unittest.skipIf(Py_GIL_DISABLED, FT_TEST_DISABLED_REVERSE_TYPE_CACHE)
     def test_cached_attributes(self):
         class C:
             A = 1
@@ -2506,7 +2498,6 @@ def testfunc(n):
 
         self.assertIn("_POP_TOP_NOP", uops)
 
-    @unittest.skipIf(Py_GIL_DISABLED, "FT build immortalizes constants")
     def test_pop_top_specialize_int(self):
         def testfunc(n):
             for _ in range(n):
@@ -2520,7 +2511,6 @@ def testfunc(n):
 
         self.assertIn("_POP_TOP_INT", uops)
 
-    @unittest.skipIf(Py_GIL_DISABLED, "FT build immortalizes constants")
     def test_pop_top_specialize_float(self):
         def testfunc(n):
             for _ in range(n):

Objects/funcobject.c

@@ -298,7 +298,7 @@ functions is running.
 
 */
 
-#ifndef Py_GIL_DISABLED
+#if _Py_TIER2
 static inline struct _func_version_cache_item *
 get_cache_item(PyInterpreterState *interp, uint32_t version)
 {
@@ -315,11 +315,13 @@ _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version)
     // This should only be called from MAKE_FUNCTION. No code is specialized
     // based on the version, so we do not need to stop the world to set it.
     func->func_version = version;
-#ifndef Py_GIL_DISABLED
+#if _Py_TIER2
     PyInterpreterState *interp = _PyInterpreterState_GET();
+    FT_MUTEX_LOCK(&interp->func_state.mutex);
     struct _func_version_cache_item *slot = get_cache_item(interp, version);
     slot->func = func;
     slot->code = func->func_code;
+    FT_MUTEX_UNLOCK(&interp->func_state.mutex);
 #endif
 }
 
@@ -330,13 +332,15 @@ func_clear_version(PyInterpreterState *interp, PyFunctionObject *func)
         // Version was never set or has already been cleared.
         return;
     }
-#ifndef Py_GIL_DISABLED
+#if _Py_TIER2
+    FT_MUTEX_LOCK(&interp->func_state.mutex);
     struct _func_version_cache_item *slot =
         get_cache_item(interp, func->func_version);
     if (slot->func == func) {
         slot->func = NULL;
         // Leave slot->code alone, there may be use for it.
     }
+    FT_MUTEX_UNLOCK(&interp->func_state.mutex);
 #endif
     func->func_version = FUNC_VERSION_CLEARED;
 }
@@ -358,8 +362,9 @@ _PyFunction_ClearVersion(PyFunctionObject *func)
 void
 _PyFunction_ClearCodeByVersion(uint32_t version)
 {
-#ifndef Py_GIL_DISABLED
+#if _Py_TIER2
     PyInterpreterState *interp = _PyInterpreterState_GET();
+    FT_MUTEX_LOCK(&interp->func_state.mutex);
     struct _func_version_cache_item *slot = get_cache_item(interp, version);
     if (slot->code) {
         assert(PyCode_Check(slot->code));
@@ -369,15 +374,17 @@ _PyFunction_ClearCodeByVersion(uint32_t version)
             slot->func = NULL;
         }
     }
+    FT_MUTEX_UNLOCK(&interp->func_state.mutex);
 #endif
 }
 
 PyFunctionObject *
 _PyFunction_LookupByVersion(uint32_t version, PyObject **p_code)
 {
-#ifdef Py_GIL_DISABLED
-    return NULL;
-#else
+#if _Py_TIER2
+    // This function does not need locking/atomics as it can only be
+    // called from the optimizer, which is currently disabled
+    // when there are multiple threads.
     PyInterpreterState *interp = _PyInterpreterState_GET();
     struct _func_version_cache_item *slot = get_cache_item(interp, version);
     if (slot->code) {
@@ -401,6 +408,9 @@ _PyFunction_LookupByVersion(uint32_t version, PyObject **p_code)
 uint32_t
 _PyFunction_GetVersionForCurrentState(PyFunctionObject *func)
 {
+    // This function does not need locking/atomics as it can only be
+    // called from the optimizer, which is currently disabled
+    // when there are multiple threads.
     return func->func_version;
 }
 

Objects/typeobject.c

@@ -1148,15 +1148,18 @@ static void
 set_version_unlocked(PyTypeObject *tp, unsigned int version)
 {
     assert(version == 0 || (tp->tp_versions_used != _Py_ATTR_CACHE_UNUSED));
-#ifndef Py_GIL_DISABLED
+#if _Py_TIER2
     PyInterpreterState *interp = _PyInterpreterState_GET();
+    BEGIN_TYPE_LOCK();
     // lookup the old version and set to null
     if (tp->tp_version_tag != 0) {
         PyTypeObject **slot =
             interp->types.type_version_cache
             + (tp->tp_version_tag % TYPE_VERSION_CACHE_SIZE);
         *slot = NULL;
     }
+#endif
+#ifndef Py_GIL_DISABLED
     if (version) {
         tp->tp_versions_used++;
     }
@@ -1166,13 +1169,14 @@ set_version_unlocked(PyTypeObject *tp, unsigned int version)
     }
 #endif
     FT_ATOMIC_STORE_UINT_RELAXED(tp->tp_version_tag, version);
-#ifndef Py_GIL_DISABLED
+#if _Py_TIER2
     if (version != 0) {
         PyTypeObject **slot =
             interp->types.type_version_cache
             + (version % TYPE_VERSION_CACHE_SIZE);
         *slot = tp;
     }
+    END_TYPE_LOCK();
 #endif
 }
 
@@ -1357,9 +1361,12 @@ _PyType_SetVersion(PyTypeObject *tp, unsigned int version)
 PyTypeObject *
 _PyType_LookupByVersion(unsigned int version)
 {
-#ifdef Py_GIL_DISABLED
+#ifndef _Py_TIER2
     return NULL;
 #else
+    // This function does not need locking/atomics as it can only be
+    // called from the optimizer, which is currently disabled
+    // when there are multiple threads.
     PyInterpreterState *interp = _PyInterpreterState_GET();
     PyTypeObject **slot =
         interp->types.type_version_cache

Python/optimizer.c

@@ -1655,12 +1655,12 @@ jit_tracer_invalidate_dependency(PyThreadState *tstate, void *obj)
     _Py_BloomFilter_Init(&obj_filter);
     _Py_BloomFilter_Add(&obj_filter, obj);
     _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
-    PyMutex_Lock(&_tstate->jit_tracer_state.lock);
+    FT_MUTEX_LOCK(&_tstate->jit_tracer_state.lock);
     if (bloom_filter_may_contain(&_tstate->jit_tracer_state.prev_state.dependencies, &obj_filter))
     {
         FT_ATOMIC_STORE_UINT8(_tstate->jit_tracer_state.prev_state.dependencies_still_valid, 0);
     }
-    PyMutex_Unlock(&_tstate->jit_tracer_state.lock);
+    FT_MUTEX_UNLOCK(&_tstate->jit_tracer_state.lock);
 }
 
 void