revert commits

Author: xuantengh

Include/internal/pycore_optimizer.h

@@ -116,7 +116,8 @@ PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj);
 // Export for '_testinternalcapi' shared extension.
 PyAPI_FUNC(_PyOptimizerObject *) _Py_GetOptimizer(void);
 PyAPI_FUNC(int) _Py_SetTier2Optimizer(_PyOptimizerObject* optimizer);
-PyAPI_FUNC(void) _Py_SetUOpOptimize(_PyOptimizerObject* opt);
+PyAPI_FUNC(PyObject *) _PyOptimizer_NewCounter(void);
+PyAPI_FUNC(PyObject *) _PyOptimizer_NewUOpOptimizer(void);
 
 #define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3
 #define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6

Lib/test/test_capi/test_opt.py

@@ -140,8 +140,89 @@ def get_opnames(ex):
 
 @requires_specialization
 @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds")
-@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer") and
-                     hasattr(_testinternalcapi, "new_uop_optimizer"),
+@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"),
+                     "Requires optimizer infrastructure")
+class TestExecutorInvalidation(unittest.TestCase):
+
+    def setUp(self):
+        self.old = _testinternalcapi.get_optimizer()
+        self.opt = _testinternalcapi.new_counter_optimizer()
+        _testinternalcapi.set_optimizer(self.opt)
+
+    def tearDown(self):
+        _testinternalcapi.set_optimizer(self.old)
+
+    def test_invalidate_object(self):
+        # Generate a new set of functions at each call
+        ns = {}
+        func_src = "\n".join(
+            f"""
+            def f{n}():
+                for _ in range(1000):
+                    pass
+            """ for n in range(5)
+        )
+        exec(textwrap.dedent(func_src), ns, ns)
+        funcs = [ ns[f'f{n}'] for n in range(5)]
+        objects = [object() for _ in range(5)]
+
+        for f in funcs:
+            f()
+        executors = [get_first_executor(f) for f in funcs]
+        # Set things up so each executor depends on the objects
+        # with an equal or lower index.
+        for i, exe in enumerate(executors):
+            self.assertTrue(exe.is_valid())
+            for obj in objects[:i+1]:
+                _testinternalcapi.add_executor_dependency(exe, obj)
+            self.assertTrue(exe.is_valid())
+        # Assert that the correct executors are invalidated
+        # and check that nothing crashes when we invalidate
+        # an executor multiple times.
+        for i in (4,3,2,1,0):
+            _testinternalcapi.invalidate_executors(objects[i])
+            for exe in executors[i:]:
+                self.assertFalse(exe.is_valid())
+            for exe in executors[:i]:
+                self.assertTrue(exe.is_valid())
+
+    def test_uop_optimizer_invalidation(self):
+        # Generate a new function at each call
+        ns = {}
+        exec(textwrap.dedent("""
+            def f():
+                for i in range(1000):
+                    pass
+        """), ns, ns)
+        f = ns['f']
+        opt = _testinternalcapi.new_uop_optimizer()
+        with temporary_optimizer(opt):
+            f()
+        exe = get_first_executor(f)
+        self.assertIsNotNone(exe)
+        self.assertTrue(exe.is_valid())
+        _testinternalcapi.invalidate_executors(f.__code__)
+        self.assertFalse(exe.is_valid())
+
+    def test_sys__clear_internal_caches(self):
+        def f():
+            for _ in range(1000):
+                pass
+        opt = _testinternalcapi.new_uop_optimizer()
+        with temporary_optimizer(opt):
+            f()
+        exe = get_first_executor(f)
+        self.assertIsNotNone(exe)
+        self.assertTrue(exe.is_valid())
+        sys._clear_internal_caches()
+        self.assertFalse(exe.is_valid())
+        exe = get_first_executor(f)
+        self.assertIsNone(exe)
+
+
+@requires_specialization
+@unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds")
+@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"),
                      "Requires optimizer infrastructure")
 class TestExecutorInvalidation(unittest.TestCase):
 
@@ -589,8 +670,7 @@ def testfunc(n):
 
 @requires_specialization
 @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds")
-@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer") and
-                     hasattr(_testinternalcapi, "new_uop_optimizer"),
+@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"),
                      "Requires optimizer infrastructure")
 @unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.")
 class TestUopsOptimization(unittest.TestCase):

Lib/test/test_monitoring.py

@@ -1959,6 +1959,35 @@ def callback(code, instruction_offset):
         sys.monitoring.set_events(0, 0)
 
 
+class TestOptimizer(MonitoringTestBase, unittest.TestCase):
+
+    def setUp(self):
+        _testinternalcapi = import_module("_testinternalcapi")
+        if hasattr(_testinternalcapi, "get_optimizer"):
+            self.old_opt = _testinternalcapi.get_optimizer()
+            opt = _testinternalcapi.new_counter_optimizer()
+            _testinternalcapi.set_optimizer(opt)
+        super(TestOptimizer, self).setUp()
+
+    def tearDown(self):
+        super(TestOptimizer, self).tearDown()
+        import _testinternalcapi
+        if hasattr(_testinternalcapi, "get_optimizer"):
+            _testinternalcapi.set_optimizer(self.old_opt)
+
+    def test_for_loop(self):
+        def test_func(x):
+            i = 0
+            while i < x:
+                i += 1
+
+        code = test_func.__code__
+        sys.monitoring.set_local_events(TEST_TOOL, code, E.PY_START)
+        self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.PY_START)
+        test_func(1000)
+        sys.monitoring.set_local_events(TEST_TOOL, code, 0)
+        self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), 0)
+
 class TestTier2Optimizer(CheckEvents):
 
     def test_monitoring_already_opimized_loop(self):

Modules/_testinternalcapi.c

@@ -989,6 +989,18 @@ get_co_framesize(PyObject *self, PyObject *arg)
 
 #ifdef _Py_TIER2
 
+static PyObject *
+new_counter_optimizer(PyObject *self, PyObject *arg)
+{
+    return _PyOptimizer_NewCounter();
+}
+
+static PyObject *
+new_uop_optimizer(PyObject *self, PyObject *arg)
+{
+    return _PyOptimizer_NewUOpOptimizer();
+}
+
 static PyObject *
 set_optimizer(PyObject *self, PyObject *opt)
 {
@@ -2100,6 +2112,8 @@ static PyMethodDef module_functions[] = {
 #ifdef _Py_TIER2
     {"get_optimizer", get_optimizer,  METH_NOARGS, NULL},
     {"set_optimizer", set_optimizer,  METH_O, NULL},
+    {"new_counter_optimizer", new_counter_optimizer, METH_NOARGS, NULL},
+    {"new_uop_optimizer", new_uop_optimizer, METH_NOARGS, NULL},
     {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL},
     {"invalidate_executors", invalidate_executors, METH_O, NULL},
 #endif

Python/optimizer.c

@@ -1308,10 +1308,15 @@ PyTypeObject _PyUOpOptimizer_Type = {
     .tp_dealloc = uop_opt_dealloc,
 };
 
-void
-_Py_SetUOpOptimize(_PyOptimizerObject *opt)
+PyObject *
+_PyOptimizer_NewUOpOptimizer(void)
 {
+    _PyOptimizerObject *opt = PyObject_New(_PyOptimizerObject, &_PyUOpOptimizer_Type);
+    if (opt == NULL) {
+        return NULL;
+    }
     opt->optimize = uop_optimize;
+    return (PyObject *)opt;
 }
 
 static void
@@ -1392,6 +1397,18 @@ PyTypeObject _PyCounterOptimizer_Type = {
     .tp_dealloc = (destructor)PyObject_Free,
 };
 
+PyObject *
+_PyOptimizer_NewCounter(void)
+{
+    _PyCounterOptimizerObject *opt = (_PyCounterOptimizerObject *)_PyObject_New(&_PyCounterOptimizer_Type);
+    if (opt == NULL) {
+        return NULL;
+    }
+    opt->base.optimize = counter_optimize;
+    opt->count = 0;
+    return (PyObject *)opt;
+}
+
 
 /*****************************************
  *        Executor management

Python/pylifecycle.c

@@ -1321,11 +1321,10 @@ init_interp_main(PyThreadState *tstate)
             } else
 #endif
             {
-                PyObject *opt = PyObject_New(_PyOptimizerObject, &_PyUOpOptimizer_Type);
+                PyObject *opt = _PyOptimizer_NewUOpOptimizer();
                 if (opt == NULL) {
                     return _PyStatus_ERR("can't initialize optimizer");
                 }
-                _Py_SetUOpOptimize((_PyOptimizerObject *)opt);
                 if (_Py_SetTier2Optimizer((_PyOptimizerObject *)opt)) {
                     return _PyStatus_ERR("can't install optimizer");
                 }