Skip to content

Commit ccf1b0b

Browse files
markshannonmarkshannon
authored
GH-132508: Use tagged integers on the evaluation stack for the last instruction offset (GH-132545)
1 parent caee16f commit ccf1b0b

File tree

14 files changed

+129
-60
lines changed

14 files changed

+129
-60
lines changed

‎Include/internal/pycore_interpframe.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ extern "C" {
1818
((int)((IF)->instr_ptr - _PyFrame_GetBytecode((IF))))
1919

2020
static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) {
21+
assert(!PyStackRef_IsNull(f->f_executable));
2122
PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable);
2223
assert(PyCode_Check(executable));
2324
return (PyCodeObject *)executable;

‎Include/internal/pycore_stackref.h‎

Lines changed: 68 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -63,11 +63,13 @@ extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _P
6363

6464
static const _PyStackRef PyStackRef_NULL = { .index = 0 };
6565

66-
#define PyStackRef_None ((_PyStackRef){ .index = 1 } )
67-
#define PyStackRef_False ((_PyStackRef){ .index = 2 })
68-
#define PyStackRef_True ((_PyStackRef){ .index = 3 })
66+
// Use the first 3 even numbers for None, True and False.
67+
// Odd numbers are reserved for (tagged) integers
68+
#define PyStackRef_None ((_PyStackRef){ .index = 2 } )
69+
#define PyStackRef_False ((_PyStackRef){ .index = 4 })
70+
#define PyStackRef_True ((_PyStackRef){ .index = 6 })
6971

70-
#define LAST_PREDEFINED_STACKREF_INDEX 3
72+
#define INITIAL_STACKREF_INDEX 8
7173

7274
static inline int
7375
PyStackRef_IsNull(_PyStackRef ref)
@@ -96,6 +98,7 @@ PyStackRef_IsNone(_PyStackRef ref)
9698
static inline PyObject *
9799
_PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumber)
98100
{
101+
assert((ref.index & 1) == 0);
99102
_Py_stackref_record_borrow(ref, filename, linenumber);
100103
return _Py_stackref_get_object(ref);
101104
}
@@ -132,31 +135,45 @@ _PyStackRef_FromPyObjectImmortal(PyObject *obj, const char *filename, int linenu
132135
}
133136
#define PyStackRef_FromPyObjectImmortal(obj) _PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj), __FILE__, __LINE__)
134137

138+
static inline bool
139+
PyStackRef_IsTaggedInt(_PyStackRef ref)
140+
{
141+
return (ref.index & 1) == 1;
142+
}
143+
135144
static inline void
136145
_PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber)
137146
{
147+
if (PyStackRef_IsTaggedInt(ref)) {
148+
return;
149+
}
138150
PyObject *obj = _Py_stackref_close(ref, filename, linenumber);
139151
Py_DECREF(obj);
140152
}
141153
#define PyStackRef_CLOSE(REF) _PyStackRef_CLOSE((REF), __FILE__, __LINE__)
142154

155+
143156
static inline void
144157
_PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber)
145158
{
146159
if (PyStackRef_IsNull(ref)) {
147160
return;
148161
}
149-
PyObject *obj = _Py_stackref_close(ref, filename, linenumber);
150-
Py_DECREF(obj);
162+
_PyStackRef_CLOSE(ref, filename, linenumber);
151163
}
152164
#define PyStackRef_XCLOSE(REF) _PyStackRef_XCLOSE((REF), __FILE__, __LINE__)
153165

154166
static inline _PyStackRef
155167
_PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber)
156168
{
157-
PyObject *obj = _Py_stackref_get_object(ref);
158-
Py_INCREF(obj);
159-
return _Py_stackref_create(obj, filename, linenumber);
169+
if (PyStackRef_IsTaggedInt(ref)) {
170+
return ref;
171+
}
172+
else {
173+
PyObject *obj = _Py_stackref_get_object(ref);
174+
Py_INCREF(obj);
175+
return _Py_stackref_create(obj, filename, linenumber);
176+
}
160177
}
161178
#define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__)
162179

@@ -210,8 +227,40 @@ _PyStackRef_FromPyObjectNewMortal(PyObject *obj, const char *filename, int linen
210227

211228
extern int PyStackRef_Is(_PyStackRef a, _PyStackRef b);
212229

230+
extern bool PyStackRef_IsTaggedInt(_PyStackRef ref);
231+
232+
extern intptr_t PyStackRef_UntagInt(_PyStackRef ref);
233+
234+
extern _PyStackRef PyStackRef_TagInt(intptr_t i);
235+
236+
extern bool
237+
PyStackRef_IsNullOrInt(_PyStackRef ref);
238+
213239
#else
214240

241+
#define Py_INT_TAG 3
242+
243+
static inline bool
244+
PyStackRef_IsTaggedInt(_PyStackRef i)
245+
{
246+
return (i.bits & Py_INT_TAG) == Py_INT_TAG;
247+
}
248+
249+
static inline _PyStackRef
250+
PyStackRef_TagInt(intptr_t i)
251+
{
252+
assert(Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (i << 2), 2) == i);
253+
return (_PyStackRef){ .bits = ((((uintptr_t)i) << 2) | Py_INT_TAG) };
254+
}
255+
256+
static inline intptr_t
257+
PyStackRef_UntagInt(_PyStackRef i)
258+
{
259+
assert((i.bits & Py_INT_TAG) == Py_INT_TAG);
260+
intptr_t val = (intptr_t)i.bits;
261+
return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, val, 2);
262+
}
263+
215264

216265
#ifdef Py_GIL_DISABLED
217266

@@ -232,6 +281,8 @@ static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED};
232281
#define PyStackRef_IsTrue(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_True)
233282
#define PyStackRef_IsFalse(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_False)
234283

284+
#define PyStackRef_IsNullOrInt(stackref) (PyStackRef_IsNull(stackref) || PyStackRef_IsTaggedInt(stackref))
285+
235286
static inline PyObject *
236287
PyStackRef_AsPyObjectBorrow(_PyStackRef stackref)
237288
{
@@ -451,6 +502,7 @@ PyStackRef_RefcountOnObject(_PyStackRef ref)
451502
static inline PyObject *
452503
PyStackRef_AsPyObjectBorrow(_PyStackRef ref)
453504
{
505+
assert(!PyStackRef_IsTaggedInt(ref));
454506
return BITS_TO_PTR_MASKED(ref);
455507
}
456508

@@ -587,6 +639,12 @@ PyStackRef_CLOSE(_PyStackRef ref)
587639
}
588640
#endif
589641

642+
static inline bool
643+
PyStackRef_IsNullOrInt(_PyStackRef ref)
644+
{
645+
return PyStackRef_IsNull(ref) || PyStackRef_IsTaggedInt(ref);
646+
}
647+
590648
static inline void
591649
PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct)
592650
{
@@ -726,7 +784,7 @@ _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out)
726784
// Like Py_VISIT but for _PyStackRef fields
727785
#define _Py_VISIT_STACKREF(ref) \
728786
do { \
729-
if (!PyStackRef_IsNull(ref)) { \
787+
if (!PyStackRef_IsNullOrInt(ref)) { \
730788
int vret = _PyGC_VisitStackRef(&(ref), visit, arg); \
731789
if (vret) \
732790
return vret; \

‎Misc/NEWS.d/next/Core_and_Builtins/2025-04-15-10-09-49.gh-issue-132508.zVe3iI.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Uses tagged integers on the evaluation stack to represent the instruction
2+
offsets when reraising an exception. This avoids the need to box the integer
3+
which could fail in low memory conditions.

‎Objects/frameobject.c‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1813,15 +1813,16 @@ frame_lineno_set_impl(PyFrameObject *self, PyObject *value)
18131813
start_stack = pop_value(start_stack);
18141814
}
18151815
while (start_stack > best_stack) {
1816+
_PyStackRef popped = _PyFrame_StackPop(self->f_frame);
18161817
if (top_of_stack(start_stack) == Except) {
18171818
/* Pop exception stack as well as the evaluation stack */
1818-
PyObject *exc = PyStackRef_AsPyObjectBorrow(_PyFrame_StackPop(self->f_frame));
1819+
PyObject *exc = PyStackRef_AsPyObjectBorrow(popped);
18191820
assert(PyExceptionInstance_Check(exc) || exc == Py_None);
18201821
PyThreadState *tstate = _PyThreadState_GET();
18211822
Py_XSETREF(tstate->exc_info->exc_value, exc == Py_None ? NULL : exc);
18221823
}
18231824
else {
1824-
PyStackRef_XCLOSE(_PyFrame_StackPop(self->f_frame));
1825+
PyStackRef_XCLOSE(popped);
18251826
}
18261827
start_stack = pop_value(start_stack);
18271828
}

‎Python/bytecodes.c‎

Lines changed: 4 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1384,16 +1384,7 @@ dummy_func(
13841384

13851385
assert(oparg >= 0 && oparg <= 2);
13861386
if (oparg) {
1387-
PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]);
1388-
if (PyLong_Check(lasti)) {
1389-
frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti);
1390-
assert(!_PyErr_Occurred(tstate));
1391-
}
1392-
else {
1393-
_PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int");
1394-
Py_DECREF(exc);
1395-
ERROR_NO_POP();
1396-
}
1387+
frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]);
13971388
}
13981389
assert(exc && PyExceptionInstance_Check(exc));
13991390
_PyErr_SetRaisedException(tstate, exc);
@@ -3472,7 +3463,7 @@ dummy_func(
34723463
if (tb == NULL) {
34733464
tb = Py_None;
34743465
}
3475-
assert(PyStackRef_LongCheck(lasti));
3466+
assert(PyStackRef_IsTaggedInt(lasti));
34763467
(void)lasti; // Shut up compiler warning if asserts are off
34773468
PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb};
34783469
int has_self = !PyStackRef_IsNull(exit_self);
@@ -5378,11 +5369,8 @@ dummy_func(
53785369
}
53795370
if (lasti) {
53805371
int frame_lasti = _PyInterpreterFrame_LASTI(frame);
5381-
PyObject *lasti = PyLong_FromLong(frame_lasti);
5382-
if (lasti == NULL) {
5383-
goto exception_unwind;
5384-
}
5385-
_PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(lasti));
5372+
_PyStackRef lasti = PyStackRef_TagInt(frame_lasti);
5373+
_PyFrame_StackPush(frame, lasti);
53865374
}
53875375

53885376
/* Make the raw exception data

‎Python/ceval.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,10 @@ dump_item(_PyStackRef item)
146146
printf("<NULL>");
147147
return;
148148
}
149+
if (PyStackRef_IsTaggedInt(item)) {
150+
printf("%" PRId64, (int64_t)PyStackRef_UntagInt(item));
151+
return;
152+
}
149153
PyObject *obj = PyStackRef_AsPyObjectBorrow(item);
150154
if (obj == NULL) {
151155
printf("<nil>");

‎Python/executor_cases.c.h‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Python/gc.c‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -547,6 +547,7 @@ _PyGC_VisitStackRef(_PyStackRef *ref, visitproc visit, void *arg)
547547
// This is a bit tricky! We want to ignore stackrefs with embedded
548548
// refcounts when computing the incoming references, but otherwise treat
549549
// them like normal.
550+
assert(!PyStackRef_IsTaggedInt(*ref));
550551
if (!PyStackRef_RefcountOnObject(*ref) && (visit == visit_decref)) {
551552
return 0;
552553
}
@@ -560,7 +561,9 @@ _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg)
560561
_PyStackRef *ref = _PyFrame_GetLocalsArray(frame);
561562
/* locals and stack */
562563
for (; ref < frame->stackpointer; ref++) {
563-
_Py_VISIT_STACKREF(*ref);
564+
if (!PyStackRef_IsTaggedInt(*ref)) {
565+
_Py_VISIT_STACKREF(*ref);
566+
}
564567
}
565568
return 0;
566569
}
@@ -1495,8 +1498,11 @@ mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, b
14951498
objects_marked += move_to_reachable(func, &reachable, visited_space);
14961499
while (sp > locals) {
14971500
sp--;
1501+
if (PyStackRef_IsNullOrInt(*sp)) {
1502+
continue;
1503+
}
14981504
PyObject *op = PyStackRef_AsPyObjectBorrow(*sp);
1499-
if (op == NULL || _Py_IsImmortal(op)) {
1505+
if (_Py_IsImmortal(op)) {
15001506
continue;
15011507
}
15021508
if (_PyObject_IS_GC(op)) {

‎Python/gc_free_threading.c‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,7 @@ frame_disable_deferred_refcounting(_PyInterpreterFrame *frame)
265265

266266
frame->f_funcobj = PyStackRef_AsStrongReference(frame->f_funcobj);
267267
for (_PyStackRef *ref = frame->localsplus; ref < frame->stackpointer; ref++) {
268-
if (!PyStackRef_IsNull(*ref) && PyStackRef_IsDeferred(*ref)) {
268+
if (!PyStackRef_IsNullOrInt(*ref) && PyStackRef_IsDeferred(*ref)) {
269269
*ref = PyStackRef_AsStrongReference(*ref);
270270
}
271271
}
@@ -420,7 +420,7 @@ gc_visit_heaps(PyInterpreterState *interp, mi_block_visit_fun *visitor,
420420
static inline void
421421
gc_visit_stackref(_PyStackRef stackref)
422422
{
423-
if (PyStackRef_IsDeferred(stackref) && !PyStackRef_IsNull(stackref)) {
423+
if (PyStackRef_IsDeferred(stackref) && !PyStackRef_IsNullOrInt(stackref)) {
424424
PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref);
425425
if (_PyObject_GC_IS_TRACKED(obj) && !gc_is_frozen(obj)) {
426426
gc_add_refs(obj, 1);
@@ -817,7 +817,7 @@ gc_abort_mark_alive(PyInterpreterState *interp,
817817
static int
818818
gc_visit_stackref_mark_alive(gc_mark_args_t *args, _PyStackRef stackref)
819819
{
820-
if (!PyStackRef_IsNull(stackref)) {
820+
if (!PyStackRef_IsNullOrInt(stackref)) {
821821
PyObject *op = PyStackRef_AsPyObjectBorrow(stackref);
822822
if (gc_mark_enqueue(op, args) < 0) {
823823
return -1;
@@ -1706,6 +1706,7 @@ _PyGC_VisitStackRef(_PyStackRef *ref, visitproc visit, void *arg)
17061706
// This is a bit tricky! We want to ignore deferred references when
17071707
// computing the incoming references, but otherwise treat them like
17081708
// regular references.
1709+
assert(!PyStackRef_IsTaggedInt(*ref));
17091710
if (!PyStackRef_IsDeferred(*ref) ||
17101711
(visit != visit_decref && visit != visit_decref_unreachable))
17111712
{

‎Python/generated_cases.c.h‎

Lines changed: 4 additions & 20 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)