Cache TOS across side exits: Work in progress

Author: markshannon

Include/internal/pycore_interp_structs.h

@@ -15,7 +15,6 @@ extern "C" {
 #include "pycore_tstate.h"        // _PyThreadStateImpl
 #include "pycore_typedefs.h"      // _PyRuntimeState
 
-
 #define CODE_MAX_WATCHERS 8
 #define CONTEXT_MAX_WATCHERS 8
 #define FUNC_MAX_WATCHERS 8
@@ -951,7 +950,7 @@ struct _is {
     bool jit;
     struct _PyExecutorObject *executor_list_head;
     struct _PyExecutorObject *executor_deletion_list_head;
-    struct _PyExecutorObject *cold_executor;
+    struct _PyExecutorObject **cold_executors;
     int executor_deletion_list_remaining_capacity;
     size_t trace_run_counter;
     _rare_events rare_events;

Include/internal/pycore_optimizer.h

@@ -69,7 +69,8 @@ typedef struct {
 
 typedef struct _PyExitData {
     uint32_t target;
-    uint16_t index;
+    uint16_t index:12;
+    uint16_t tos_cache:4;
     _Py_BackoffCounter temperature;
     struct _PyExecutorObject *executor;
 } _PyExitData;
@@ -352,15 +353,15 @@ extern int _Py_uop_frame_pop(JitOptContext *ctx);
 
 PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored);
 
-PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyExecutorObject **exec_ptr, int chain_depth);
+PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyExecutorObject **exec_ptr, int chain_depth, int tos_cache);
 
 static inline _PyExecutorObject *_PyExecutor_FromExit(_PyExitData *exit)
 {
     _PyExitData *exit0 = exit - exit->index;
     return (_PyExecutorObject *)(((char *)exit0) - offsetof(_PyExecutorObject, exits));
 }
 
-extern _PyExecutorObject *_PyExecutor_GetColdExecutor(void);
+extern _PyExecutorObject **_PyExecutor_GetColdExecutors(void);
 
 PyAPI_FUNC(void) _PyExecutor_ClearExit(_PyExitData *exit);
 
@@ -373,7 +374,7 @@ static inline int is_terminator(const _PyUOpInstruction *uop)
     );
 }
 
-extern void _PyExecutor_Free(_PyExecutorObject *self);
+extern void _PyExecutor_FreeColdExecutors(_PyExecutorObject **cold);
 
 PyAPI_FUNC(int) _PyDumpExecutors(FILE *out);
 #ifdef _Py_TIER2