From ace6bc5052244cd46177f39625b6f1b455952c32 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 21 Mar 2025 08:58:19 -0700 Subject: [PATCH 01/10] gh-130907: Error when accessing __annotations__ on a partially defined module --- Lib/test/test_type_annotations.py | 16 +++++++++++++- ...-03-21-08-47-36.gh-issue-130907.rGg-ge.rst | 2 ++ Objects/moduleobject.c | 22 +++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-03-21-08-47-36.gh-issue-130907.rGg-ge.rst diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 0afcd76af153e7..059678b16e4091 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -1,9 +1,11 @@ import annotationlib import inspect +import tempfile import textwrap import types import unittest -from test.support import run_code, check_syntax_error +from pathlib import Path +from test.support import run_code, check_syntax_error, import_helper class TypeAnnotationTests(unittest.TestCase): @@ -109,6 +111,18 @@ class D(metaclass=C): del D.__annotations__ self.assertEqual(D.__annotations__, {}) + def test_partially_executed_module(self): + partialexe = import_helper.import_module("test.typinganndata.partialexecution") + self.assertEqual( + partialexe.a.__annotations__, + {"v1": int, "v2": int}, + ) + self.assertIsInstance(partialexe.b.exc, RuntimeError) + self.assertEqual( + str(partialexe.b.exc), + "cannot access __annotations__ while module is initializing", + ) + def build_module(code: str, name: str = "top") -> types.ModuleType: ns = run_code(code) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-03-21-08-47-36.gh-issue-130907.rGg-ge.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-21-08-47-36.gh-issue-130907.rGg-ge.rst new file mode 100644 index 00000000000000..e9f71f3f06ae1c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-21-08-47-36.gh-issue-130907.rGg-ge.rst @@ -0,0 +1,2 @@ +Raise an error if the ``__annotations__`` attribute is accessed on a module +that has not finished executing. diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 2412f994d4bf51..bd8924e42f427d 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1247,6 +1247,28 @@ module_get_annotations(PyObject *self, void *Py_UNUSED(ignored)) PyObject *annotations; if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) == 0) { + PyObject *spec; + if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__spec__), &spec) < 0) { + Py_DECREF(dict); + return NULL; + } + if (spec != NULL) { + int rc = _PyModuleSpec_IsInitializing(spec); + if (rc < 0) { + Py_DECREF(spec); + Py_DECREF(dict); + return NULL; + } + Py_DECREF(spec); + if (rc) { + PyErr_SetString(PyExc_RuntimeError, + "cannot access __annotations__ " + "while module is initializing"); + Py_DECREF(dict); + return NULL; + } + } + PyObject *annotate; int annotate_result = PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate); if (annotate_result < 0) { From 60c633a6759caf759e39c81eb8603bbcd0e27644 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 21 Mar 2025 12:24:42 -0700 Subject: [PATCH 02/10] Remove unnecessary imports --- Lib/test/test_type_annotations.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 059678b16e4091..2aa7b7fe643280 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -1,10 +1,8 @@ import annotationlib import inspect -import tempfile import textwrap import types import unittest -from pathlib import Path from test.support import run_code, check_syntax_error, import_helper From 27004f8153c7f0b0d7ed0a2bf14826943b170951 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 21 Mar 2025 12:27:31 -0700 Subject: [PATCH 03/10] add missing --- Lib/test/test_type_annotations.py | 2 +- Lib/test/typinganndata/partialexecution/__init__.py | 1 + Lib/test/typinganndata/partialexecution/a.py | 5 +++++ Lib/test/typinganndata/partialexecution/b.py | 8 ++++++++ 4 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 Lib/test/typinganndata/partialexecution/__init__.py create mode 100644 Lib/test/typinganndata/partialexecution/a.py create mode 100644 Lib/test/typinganndata/partialexecution/b.py diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 2aa7b7fe643280..bb268585ea8821 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -110,7 +110,7 @@ class D(metaclass=C): self.assertEqual(D.__annotations__, {}) def test_partially_executed_module(self): - partialexe = import_helper.import_module("test.typinganndata.partialexecution") + partialexe = import_helper.import_fresh_module("test.typinganndata.partialexecution") self.assertEqual( partialexe.a.__annotations__, {"v1": int, "v2": int}, diff --git a/Lib/test/typinganndata/partialexecution/__init__.py b/Lib/test/typinganndata/partialexecution/__init__.py new file mode 100644 index 00000000000000..c39074ea84b7e6 --- /dev/null +++ b/Lib/test/typinganndata/partialexecution/__init__.py @@ -0,0 +1 @@ +from . import a diff --git a/Lib/test/typinganndata/partialexecution/a.py b/Lib/test/typinganndata/partialexecution/a.py new file mode 100644 index 00000000000000..ed0b8dcbd55530 --- /dev/null +++ b/Lib/test/typinganndata/partialexecution/a.py @@ -0,0 +1,5 @@ +v1: int + +from . import b + +v2: int diff --git a/Lib/test/typinganndata/partialexecution/b.py b/Lib/test/typinganndata/partialexecution/b.py new file mode 100644 index 00000000000000..b9c1e2c627c06a --- /dev/null +++ b/Lib/test/typinganndata/partialexecution/b.py @@ -0,0 +1,8 @@ +from . import a + +try: + a.__annotations__ +except Exception as e: + exc = e +else: + exc = None From d1e17613178a14842564cf0c8a13a32a0344fe56 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 8 Apr 2025 21:37:50 -0700 Subject: [PATCH 04/10] Treat all module-level annotations as conditional --- Include/internal/pycore_compile.h | 2 + .../internal/pycore_instruction_sequence.h | 1 + Lib/test/test_type_annotations.py | 6 +-- Lib/test/typinganndata/partialexecution/b.py | 7 +--- Objects/moduleobject.c | 10 ++--- Python/codegen.c | 23 ++++++++++- Python/compile.c | 8 +++- Python/instruction_sequence.c | 38 +++++++++++++++++-- Python/symtable.c | 6 ++- 9 files changed, 76 insertions(+), 25 deletions(-) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index b98dfb0cebbd3f..a8f8ec2abcc1d8 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -133,6 +133,8 @@ int _PyCompile_EnterScope(struct _PyCompiler *c, identifier name, int scope_type void _PyCompile_ExitScope(struct _PyCompiler *c); Py_ssize_t _PyCompile_AddConst(struct _PyCompiler *c, PyObject *o); _PyInstructionSequence *_PyCompile_InstrSequence(struct _PyCompiler *c); +void _PyCompile_SetInstrSequence(struct _PyCompiler *c, + _PyInstructionSequence *instr_sequence); int _PyCompile_FutureFeatures(struct _PyCompiler *c); void _PyCompile_DeferredAnnotations( struct _PyCompiler *c, PyObject **deferred_annotations, diff --git a/Include/internal/pycore_instruction_sequence.h b/Include/internal/pycore_instruction_sequence.h index 099f2fd124007a..2422bbee2ed49c 100644 --- a/Include/internal/pycore_instruction_sequence.h +++ b/Include/internal/pycore_instruction_sequence.h @@ -66,6 +66,7 @@ _PyJumpTargetLabel _PyInstructionSequence_NewLabel(_PyInstructionSequence *seq); int _PyInstructionSequence_ApplyLabelMap(_PyInstructionSequence *seq); int _PyInstructionSequence_InsertInstruction(_PyInstructionSequence *seq, int pos, int opcode, int oparg, _Py_SourceLocation loc); +int _PyInstructionSequence_PrependSequence(_PyInstructionSequence *seq, _PyInstructionSequence *nested); int _PyInstructionSequence_AddNested(_PyInstructionSequence *seq, _PyInstructionSequence *nested); void PyInstructionSequence_Fini(_PyInstructionSequence *seq); diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 0ff7ce77bf22d2..d16c02f463dbcd 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -115,11 +115,7 @@ def test_partially_executed_module(self): partialexe.a.__annotations__, {"v1": int, "v2": int}, ) - self.assertIsInstance(partialexe.b.exc, RuntimeError) - self.assertEqual( - str(partialexe.b.exc), - "cannot access __annotations__ while module is initializing", - ) + self.assertEqual(partialexe.b.annos, {"v1": int}) @cpython_only def test_no_cell(self): diff --git a/Lib/test/typinganndata/partialexecution/b.py b/Lib/test/typinganndata/partialexecution/b.py index b9c1e2c627c06a..36b8d2e52a3c63 100644 --- a/Lib/test/typinganndata/partialexecution/b.py +++ b/Lib/test/typinganndata/partialexecution/b.py @@ -1,8 +1,3 @@ from . import a -try: - a.__annotations__ -except Exception as e: - exc = e -else: - exc = None +annos = a.__annotations__ diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 37438a540d4df4..16f1b2cacd4f83 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1251,6 +1251,7 @@ module_get_annotations(PyObject *self, void *Py_UNUSED(ignored)) Py_DECREF(dict); return NULL; } + bool is_initializing = false; if (spec != NULL) { int rc = _PyModuleSpec_IsInitializing(spec); if (rc < 0) { @@ -1260,11 +1261,7 @@ module_get_annotations(PyObject *self, void *Py_UNUSED(ignored)) } Py_DECREF(spec); if (rc) { - PyErr_SetString(PyExc_RuntimeError, - "cannot access __annotations__ " - "while module is initializing"); - Py_DECREF(dict); - return NULL; + is_initializing = true; } } @@ -1295,7 +1292,8 @@ module_get_annotations(PyObject *self, void *Py_UNUSED(ignored)) annotations = PyDict_New(); } Py_XDECREF(annotate); - if (annotations) { + // Do not cache annotations if the module is still initializing + if (annotations && !is_initializing) { int result = PyDict_SetItem( dict, &_Py_ID(__annotations__), annotations); if (result) { diff --git a/Python/codegen.c b/Python/codegen.c index dc50737840f002..18c925d8721aea 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -792,6 +792,17 @@ codegen_process_deferred_annotations(compiler *c, location loc) return SUCCESS; } + instr_sequence *old_instr_seq = INSTR_SEQUENCE(c); + instr_sequence *nested_instr_seq = NULL; + int scope_type = SCOPE_TYPE(c); + if (scope_type == COMPILE_SCOPE_MODULE) { + nested_instr_seq = (instr_sequence *)_PyInstructionSequence_New(); + if (nested_instr_seq == NULL) { + goto error; + } + _PyCompile_SetInstrSequence(c, nested_instr_seq); + } + // It's possible that ste_annotations_block is set but // u_deferred_annotations is not, because the former is still // set if there are only non-simple annotations (i.e., annotations @@ -800,7 +811,6 @@ codegen_process_deferred_annotations(compiler *c, location loc) PySTEntryObject *ste = SYMTABLE_ENTRY(c); assert(ste->ste_annotation_block != NULL); void *key = (void *)((uintptr_t)ste->ste_id + 1); - int scope_type = SCOPE_TYPE(c); if (codegen_setup_annotations_scope(c, loc, key, ste->ste_annotation_block->ste_name) < 0) { goto error; @@ -817,8 +827,19 @@ codegen_process_deferred_annotations(compiler *c, location loc) RETURN_IF_ERROR(codegen_leave_annotations_scope(c, loc)); RETURN_IF_ERROR(codegen_nameop(c, loc, &_Py_ID(__annotate__), Store)); + if (nested_instr_seq != NULL) { + RETURN_IF_ERROR( + _PyInstructionSequence_PrependSequence(old_instr_seq, nested_instr_seq)); + _PyCompile_SetInstrSequence(c, old_instr_seq); + PyInstructionSequence_Fini(nested_instr_seq); + } + return SUCCESS; error: + if (nested_instr_seq != NULL) { + PyInstructionSequence_Fini(nested_instr_seq); + _PyCompile_SetInstrSequence(c, old_instr_seq); + } Py_XDECREF(deferred_anno); Py_XDECREF(conditional_annotation_indices); return ERROR; diff --git a/Python/compile.c b/Python/compile.c index 430898b5966842..f3c3e6cc92a05f 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1154,7 +1154,7 @@ _PyCompile_AddDeferredAnnotation(compiler *c, stmt_ty s, } Py_DECREF(ptr); PyObject *index; - if (c->u->u_in_conditional_block) { + if (c->u->u_scope_type == COMPILE_SCOPE_MODULE || c->u->u_in_conditional_block) { index = PyLong_FromLong(c->u->u_next_conditional_annotation_index); if (index == NULL) { return ERROR; @@ -1231,6 +1231,12 @@ _PyCompile_InstrSequence(compiler *c) return c->u->u_instr_sequence; } +void +_PyCompile_SetInstrSequence(compiler *c, instr_sequence *seq) +{ + c->u->u_instr_sequence = seq; +} + int _PyCompile_FutureFeatures(compiler *c) { diff --git a/Python/instruction_sequence.c b/Python/instruction_sequence.c index 4ca85eec345d38..b15d68b1ef7eec 100644 --- a/Python/instruction_sequence.c +++ b/Python/instruction_sequence.c @@ -34,10 +34,9 @@ typedef _Py_SourceLocation location; } static int -instr_sequence_next_inst(instr_sequence *seq) { +instr_sequence_grow(instr_sequence *seq, int size) { assert(seq->s_instrs != NULL || seq->s_used == 0); - _Py_c_array_t array = { .array = (void*)seq->s_instrs, .allocated_entries = seq->s_allocated, @@ -45,13 +44,19 @@ instr_sequence_next_inst(instr_sequence *seq) { .initial_num_entries = INITIAL_INSTR_SEQUENCE_SIZE, }; - RETURN_IF_ERROR(_Py_CArray_EnsureCapacity(&array, seq->s_used + 1)); + RETURN_IF_ERROR(_Py_CArray_EnsureCapacity(&array, seq->s_used + size)); seq->s_instrs = array.array; seq->s_allocated = array.allocated_entries; assert(seq->s_allocated >= 0); assert(seq->s_used < seq->s_allocated); - return seq->s_used++; + seq->s_used += size; + return seq->s_used - 1; +} + +static int +instr_sequence_next_inst(instr_sequence *seq) { + return instr_sequence_grow(seq, 1); } _PyJumpTargetLabel @@ -154,6 +159,31 @@ _PyInstructionSequence_InsertInstruction(instr_sequence *seq, int pos, return SUCCESS; } +int +_PyInstructionSequence_PrependSequence(instr_sequence *seq, + instr_sequence *nested) +{ + if (nested->s_used == 0) { + return SUCCESS; + } + // Merging labelmaps is not supported + assert(nested->s_labelmap_size == 0 && nested->s_nested == NULL); + + int last_idx = instr_sequence_grow(seq, nested->s_used); + + RETURN_IF_ERROR(last_idx); + for (int i = last_idx - nested->s_used; i >= 0; i--) { + seq->s_instrs[i + nested->s_used] = seq->s_instrs[i]; + } + for (int i=0; i < nested->s_used; i++) { + seq->s_instrs[i] = nested->s_instrs[i]; + } + for(int lbl=0; lbl < seq->s_labelmap_size; lbl++) { + seq->s_labelmap[lbl] += nested->s_used; + } + return SUCCESS; +} + int _PyInstructionSequence_AddNested(instr_sequence *seq, instr_sequence *nested) { diff --git a/Python/symtable.c b/Python/symtable.c index 35c9a0e295c60c..8a7bec284ce9f0 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -2749,8 +2749,10 @@ symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key) // Annotations in local scopes are not executed and should not affect the symtable bool is_unevaluated = st->st_cur->ste_type == FunctionBlock; - if ((st->st_cur->ste_type == ClassBlock || st->st_cur->ste_type == ModuleBlock) - && st->st_cur->ste_in_conditional_block + // Module-level annotations are always considered conditional because the module + // may be partially executed. + if ((((st->st_cur->ste_type == ClassBlock && st->st_cur->ste_in_conditional_block) + || st->st_cur->ste_type == ModuleBlock)) && !st->st_cur->ste_has_conditional_annotations) { st->st_cur->ste_has_conditional_annotations = 1; From f550d65d7e8b7bcba1a1f62d551595f0162f0b78 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 8 Apr 2025 22:01:43 -0700 Subject: [PATCH 05/10] fixes --- .../internal/pycore_instruction_sequence.h | 3 +- Lib/test/test_dis.py | 42 ++++++++++++------- Makefile.pre.in | 1 + Python/codegen.c | 2 +- Python/instruction_sequence.c | 11 ++--- 5 files changed, 37 insertions(+), 22 deletions(-) diff --git a/Include/internal/pycore_instruction_sequence.h b/Include/internal/pycore_instruction_sequence.h index 2422bbee2ed49c..907d5759206d5e 100644 --- a/Include/internal/pycore_instruction_sequence.h +++ b/Include/internal/pycore_instruction_sequence.h @@ -66,7 +66,8 @@ _PyJumpTargetLabel _PyInstructionSequence_NewLabel(_PyInstructionSequence *seq); int _PyInstructionSequence_ApplyLabelMap(_PyInstructionSequence *seq); int _PyInstructionSequence_InsertInstruction(_PyInstructionSequence *seq, int pos, int opcode, int oparg, _Py_SourceLocation loc); -int _PyInstructionSequence_PrependSequence(_PyInstructionSequence *seq, _PyInstructionSequence *nested); +int _PyInstructionSequence_PrependSequence(_PyInstructionSequence *seq, int pos, + _PyInstructionSequence *nested); int _PyInstructionSequence_AddNested(_PyInstructionSequence *seq, _PyInstructionSequence *nested); void PyInstructionSequence_Fini(_PyInstructionSequence *seq); diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 58ba86fb43092a..9cbfaedf497ead 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -382,24 +382,36 @@ def wrap_func_w_kwargs(): # leading newline is for a reason (tests lineno) dis_annot_stmt_str = """\ - 0 RESUME 0 + -- MAKE_CELL 0 (__conditional_annotations__) - 2 LOAD_SMALL_INT 1 - STORE_NAME 0 (x) + 0 RESUME 0 - 4 LOAD_SMALL_INT 1 - LOAD_NAME 1 (lst) - LOAD_NAME 2 (fun) - PUSH_NULL - LOAD_SMALL_INT 0 - CALL 1 - STORE_SUBSCR + 2 LOAD_CONST 1 (", line 2>) + MAKE_FUNCTION + STORE_NAME 4 (__annotate__) + BUILD_SET 0 + STORE_NAME 0 (__conditional_annotations__) + LOAD_SMALL_INT 1 + STORE_NAME 1 (x) + LOAD_NAME 0 (__conditional_annotations__) + LOAD_SMALL_INT 0 + SET_ADD 1 + POP_TOP - 2 LOAD_CONST 1 (", line 2>) - MAKE_FUNCTION - STORE_NAME 3 (__annotate__) - LOAD_CONST 2 (None) - RETURN_VALUE + 3 LOAD_NAME 0 (__conditional_annotations__) + LOAD_SMALL_INT 1 + SET_ADD 1 + POP_TOP + + 4 LOAD_SMALL_INT 1 + LOAD_NAME 2 (lst) + LOAD_NAME 3 (fun) + PUSH_NULL + LOAD_SMALL_INT 0 + CALL 1 + STORE_SUBSCR + LOAD_CONST 2 (None) + RETURN_VALUE """ fn_with_annotate_str = """ diff --git a/Makefile.pre.in b/Makefile.pre.in index e10c78d6403472..194f3f540945df 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2666,6 +2666,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/translationdata/getopt \ test/translationdata/optparse \ test/typinganndata \ + test/typinganndata/partialexecution \ test/wheeldata \ test/xmltestdata \ test/xmltestdata/c14n-20 \ diff --git a/Python/codegen.c b/Python/codegen.c index 18c925d8721aea..df81f8abf76ee1 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -829,7 +829,7 @@ codegen_process_deferred_annotations(compiler *c, location loc) if (nested_instr_seq != NULL) { RETURN_IF_ERROR( - _PyInstructionSequence_PrependSequence(old_instr_seq, nested_instr_seq)); + _PyInstructionSequence_PrependSequence(old_instr_seq, 1, nested_instr_seq)); _PyCompile_SetInstrSequence(c, old_instr_seq); PyInstructionSequence_Fini(nested_instr_seq); } diff --git a/Python/instruction_sequence.c b/Python/instruction_sequence.c index b15d68b1ef7eec..c076b46dde4c60 100644 --- a/Python/instruction_sequence.c +++ b/Python/instruction_sequence.c @@ -160,23 +160,24 @@ _PyInstructionSequence_InsertInstruction(instr_sequence *seq, int pos, } int -_PyInstructionSequence_PrependSequence(instr_sequence *seq, +_PyInstructionSequence_PrependSequence(instr_sequence *seq, int pos, instr_sequence *nested) { + assert(pos >= 0 && pos <= seq->s_used); + // Merging labelmaps is not supported + assert(nested->s_labelmap_size == 0 && nested->s_nested == NULL); if (nested->s_used == 0) { return SUCCESS; } - // Merging labelmaps is not supported - assert(nested->s_labelmap_size == 0 && nested->s_nested == NULL); int last_idx = instr_sequence_grow(seq, nested->s_used); RETURN_IF_ERROR(last_idx); - for (int i = last_idx - nested->s_used; i >= 0; i--) { + for (int i = last_idx - nested->s_used; i >= pos; i--) { seq->s_instrs[i + nested->s_used] = seq->s_instrs[i]; } for (int i=0; i < nested->s_used; i++) { - seq->s_instrs[i] = nested->s_instrs[i]; + seq->s_instrs[i + pos] = nested->s_instrs[i]; } for(int lbl=0; lbl < seq->s_labelmap_size; lbl++) { seq->s_labelmap[lbl] += nested->s_used; From c2ecce50108d2151ccf08b69c50220571ad5424a Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 8 Apr 2025 22:39:43 -0700 Subject: [PATCH 06/10] fix test --- Lib/test/test_grammar.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 3ea4e47ca50a16..35cd6984267b3b 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -383,9 +383,10 @@ def test_var_annot_simple_exec(self): gns = {}; lns = {} exec("'docstring'\n" "x: int = 5\n", gns, lns) + self.assertNotIn('__annotate__', gns) + + gns.update(lns) # __annotate__ looks at globals self.assertEqual(lns["__annotate__"](annotationlib.Format.VALUE), {'x': int}) - with self.assertRaises(KeyError): - gns['__annotate__'] def test_var_annot_rhs(self): ns = {} From 137cea703c815c9576c9a9c2bd0c8ca6924507e5 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 9 Apr 2025 06:39:40 -0700 Subject: [PATCH 07/10] rename --- .../internal/pycore_instruction_sequence.h | 4 ++-- Python/codegen.c | 2 +- Python/instruction_sequence.c | 20 +++++++++---------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Include/internal/pycore_instruction_sequence.h b/Include/internal/pycore_instruction_sequence.h index 907d5759206d5e..f68d6db771fe67 100644 --- a/Include/internal/pycore_instruction_sequence.h +++ b/Include/internal/pycore_instruction_sequence.h @@ -66,8 +66,8 @@ _PyJumpTargetLabel _PyInstructionSequence_NewLabel(_PyInstructionSequence *seq); int _PyInstructionSequence_ApplyLabelMap(_PyInstructionSequence *seq); int _PyInstructionSequence_InsertInstruction(_PyInstructionSequence *seq, int pos, int opcode, int oparg, _Py_SourceLocation loc); -int _PyInstructionSequence_PrependSequence(_PyInstructionSequence *seq, int pos, - _PyInstructionSequence *nested); +int _PyInstructionSequence_InjectSequence(_PyInstructionSequence *seq, int pos, + _PyInstructionSequence *injected); int _PyInstructionSequence_AddNested(_PyInstructionSequence *seq, _PyInstructionSequence *nested); void PyInstructionSequence_Fini(_PyInstructionSequence *seq); diff --git a/Python/codegen.c b/Python/codegen.c index df81f8abf76ee1..a844a238ff3d31 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -829,7 +829,7 @@ codegen_process_deferred_annotations(compiler *c, location loc) if (nested_instr_seq != NULL) { RETURN_IF_ERROR( - _PyInstructionSequence_PrependSequence(old_instr_seq, 1, nested_instr_seq)); + _PyInstructionSequence_InjectSequence(old_instr_seq, 1, nested_instr_seq)); _PyCompile_SetInstrSequence(c, old_instr_seq); PyInstructionSequence_Fini(nested_instr_seq); } diff --git a/Python/instruction_sequence.c b/Python/instruction_sequence.c index c076b46dde4c60..87948a675b0bad 100644 --- a/Python/instruction_sequence.c +++ b/Python/instruction_sequence.c @@ -160,27 +160,27 @@ _PyInstructionSequence_InsertInstruction(instr_sequence *seq, int pos, } int -_PyInstructionSequence_PrependSequence(instr_sequence *seq, int pos, - instr_sequence *nested) +_PyInstructionSequence_InjectSequence(instr_sequence *seq, int pos, + instr_sequence *injected) { assert(pos >= 0 && pos <= seq->s_used); // Merging labelmaps is not supported - assert(nested->s_labelmap_size == 0 && nested->s_nested == NULL); - if (nested->s_used == 0) { + assert(injected->s_labelmap_size == 0 && injected->s_nested == NULL); + if (injected->s_used == 0) { return SUCCESS; } - int last_idx = instr_sequence_grow(seq, nested->s_used); + int last_idx = instr_sequence_grow(seq, injected->s_used); RETURN_IF_ERROR(last_idx); - for (int i = last_idx - nested->s_used; i >= pos; i--) { - seq->s_instrs[i + nested->s_used] = seq->s_instrs[i]; + for (int i = last_idx - injected->s_used; i >= pos; i--) { + seq->s_instrs[i + injected->s_used] = seq->s_instrs[i]; } - for (int i=0; i < nested->s_used; i++) { - seq->s_instrs[i + pos] = nested->s_instrs[i]; + for (int i=0; i < injected->s_used; i++) { + seq->s_instrs[i + pos] = injected->s_instrs[i]; } for(int lbl=0; lbl < seq->s_labelmap_size; lbl++) { - seq->s_labelmap[lbl] += nested->s_used; + seq->s_labelmap[lbl] += injected->s_used; } return SUCCESS; } From ee8e8570e756d719ed7fa3ec577fb2919eccd7bc Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 21 Apr 2025 07:51:17 -0700 Subject: [PATCH 08/10] Approach suggested by Irit --- .../internal/pycore_instruction_sequence.h | 7 ++- Include/internal/pycore_opcode_metadata.h | 24 ++++++---- Include/opcode_ids.h | 21 +++++---- Lib/_opcode_metadata.py | 21 +++++---- Lib/test/test_dis.py | 3 +- ...-03-21-08-47-36.gh-issue-130907.rGg-ge.rst | 5 +- Python/bytecodes.c | 4 ++ Python/codegen.c | 6 ++- Python/flowgraph.c | 24 +++++++++- Python/instruction_sequence.c | 47 +++++++------------ 10 files changed, 96 insertions(+), 66 deletions(-) diff --git a/Include/internal/pycore_instruction_sequence.h b/Include/internal/pycore_instruction_sequence.h index f68d6db771fe67..b5c927735374be 100644 --- a/Include/internal/pycore_instruction_sequence.h +++ b/Include/internal/pycore_instruction_sequence.h @@ -45,6 +45,9 @@ typedef struct instruction_sequence { /* PyList of instruction sequences of nested functions */ PyObject *s_nested; + + /* Code for creating annotations, spliced into the main sequence later */ + struct instruction_sequence *s_annotations_code; } _PyInstructionSequence; typedef struct { @@ -66,8 +69,8 @@ _PyJumpTargetLabel _PyInstructionSequence_NewLabel(_PyInstructionSequence *seq); int _PyInstructionSequence_ApplyLabelMap(_PyInstructionSequence *seq); int _PyInstructionSequence_InsertInstruction(_PyInstructionSequence *seq, int pos, int opcode, int oparg, _Py_SourceLocation loc); -int _PyInstructionSequence_InjectSequence(_PyInstructionSequence *seq, int pos, - _PyInstructionSequence *injected); +int _PyInstructionSequence_SetAnnotationsCode(_PyInstructionSequence *seq, + _PyInstructionSequence *annotations); int _PyInstructionSequence_AddNested(_PyInstructionSequence *seq, _PyInstructionSequence *nested); void PyInstructionSequence_Fini(_PyInstructionSequence *seq); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 521f7a92cf08c4..52de734bcd7616 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -20,6 +20,7 @@ extern "C" { #define IS_PSEUDO_INSTR(OP) ( \ ((OP) == LOAD_CLOSURE) || \ ((OP) == STORE_FAST_MAYBE_NULL) || \ + ((OP) == ANNOTATIONS_PLACEHOLDER) || \ ((OP) == JUMP) || \ ((OP) == JUMP_NO_INTERRUPT) || \ ((OP) == JUMP_IF_FALSE) || \ @@ -35,6 +36,8 @@ extern int _PyOpcode_num_popped(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA int _PyOpcode_num_popped(int opcode, int oparg) { switch(opcode) { + case ANNOTATIONS_PLACEHOLDER: + return 0; case BINARY_OP: return 2; case BINARY_OP_ADD_FLOAT: @@ -514,6 +517,8 @@ extern int _PyOpcode_num_pushed(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA int _PyOpcode_num_pushed(int opcode, int oparg) { switch(opcode) { + case ANNOTATIONS_PLACEHOLDER: + return 0; case BINARY_OP: return 1; case BINARY_OP_ADD_FLOAT: @@ -1004,7 +1009,7 @@ enum InstructionFormat { }; #define IS_VALID_OPCODE(OP) \ - (((OP) >= 0) && ((OP) < 266) && \ + (((OP) >= 0) && ((OP) < 267) && \ (_PyOpcode_opcode_metadata[(OP)].valid_entry)) #define HAS_ARG_FLAG (1) @@ -1058,9 +1063,9 @@ struct opcode_metadata { uint16_t flags; }; -extern const struct opcode_metadata _PyOpcode_opcode_metadata[266]; +extern const struct opcode_metadata _PyOpcode_opcode_metadata[267]; #ifdef NEED_OPCODE_METADATA -const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { +const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1285,6 +1290,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [ANNOTATIONS_PLACEHOLDER] = { true, -1, HAS_PURE_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_TRUE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1491,9 +1497,10 @@ _PyOpcode_macro_expansion[256] = { }; #endif // NEED_OPCODE_METADATA -extern const char *_PyOpcode_OpName[266]; +extern const char *_PyOpcode_OpName[267]; #ifdef NEED_OPCODE_METADATA -const char *_PyOpcode_OpName[266] = { +const char *_PyOpcode_OpName[267] = { + [ANNOTATIONS_PLACEHOLDER] = "ANNOTATIONS_PLACEHOLDER", [BINARY_OP] = "BINARY_OP", [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", @@ -2025,11 +2032,12 @@ struct pseudo_targets { uint8_t as_sequence; uint8_t targets[4]; }; -extern const struct pseudo_targets _PyOpcode_PseudoTargets[10]; +extern const struct pseudo_targets _PyOpcode_PseudoTargets[11]; #ifdef NEED_OPCODE_METADATA -const struct pseudo_targets _PyOpcode_PseudoTargets[10] = { +const struct pseudo_targets _PyOpcode_PseudoTargets[11] = { [LOAD_CLOSURE-256] = { 0, { LOAD_FAST, 0, 0, 0 } }, [STORE_FAST_MAYBE_NULL-256] = { 0, { STORE_FAST, 0, 0, 0 } }, + [ANNOTATIONS_PLACEHOLDER-256] = { 0, { NOP, 0, 0, 0 } }, [JUMP-256] = { 0, { JUMP_FORWARD, JUMP_BACKWARD, 0, 0 } }, [JUMP_NO_INTERRUPT-256] = { 0, { JUMP_FORWARD, JUMP_BACKWARD_NO_INTERRUPT, 0, 0 } }, [JUMP_IF_FALSE-256] = { 1, { COPY, TO_BOOL, POP_JUMP_IF_FALSE, 0 } }, @@ -2043,7 +2051,7 @@ const struct pseudo_targets _PyOpcode_PseudoTargets[10] = { #endif // NEED_OPCODE_METADATA static inline bool is_pseudo_target(int pseudo, int target) { - if (pseudo < 256 || pseudo >= 266) { + if (pseudo < 256 || pseudo >= 267) { return false; } for (int i = 0; _PyOpcode_PseudoTargets[pseudo-256].targets[i]; i++) { diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 898dc580f4148e..d93f028b732406 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -234,16 +234,17 @@ extern "C" { #define INSTRUMENTED_JUMP_BACKWARD 253 #define INSTRUMENTED_LINE 254 #define ENTER_EXECUTOR 255 -#define JUMP 256 -#define JUMP_IF_FALSE 257 -#define JUMP_IF_TRUE 258 -#define JUMP_NO_INTERRUPT 259 -#define LOAD_CLOSURE 260 -#define POP_BLOCK 261 -#define SETUP_CLEANUP 262 -#define SETUP_FINALLY 263 -#define SETUP_WITH 264 -#define STORE_FAST_MAYBE_NULL 265 +#define ANNOTATIONS_PLACEHOLDER 256 +#define JUMP 257 +#define JUMP_IF_FALSE 258 +#define JUMP_IF_TRUE 259 +#define JUMP_NO_INTERRUPT 260 +#define LOAD_CLOSURE 261 +#define POP_BLOCK 262 +#define SETUP_CLEANUP 263 +#define SETUP_FINALLY 264 +#define SETUP_WITH 265 +#define STORE_FAST_MAYBE_NULL 266 #define HAVE_ARGUMENT 42 #define MIN_SPECIALIZED_OPCODE 129 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 15900265a01270..4d30b6503fd1e9 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -350,16 +350,17 @@ 'INSTRUMENTED_CALL_KW': 251, 'INSTRUMENTED_CALL_FUNCTION_EX': 252, 'INSTRUMENTED_JUMP_BACKWARD': 253, - 'JUMP': 256, - 'JUMP_IF_FALSE': 257, - 'JUMP_IF_TRUE': 258, - 'JUMP_NO_INTERRUPT': 259, - 'LOAD_CLOSURE': 260, - 'POP_BLOCK': 261, - 'SETUP_CLEANUP': 262, - 'SETUP_FINALLY': 263, - 'SETUP_WITH': 264, - 'STORE_FAST_MAYBE_NULL': 265, + 'ANNOTATIONS_PLACEHOLDER': 256, + 'JUMP': 257, + 'JUMP_IF_FALSE': 258, + 'JUMP_IF_TRUE': 259, + 'JUMP_NO_INTERRUPT': 260, + 'LOAD_CLOSURE': 261, + 'POP_BLOCK': 262, + 'SETUP_CLEANUP': 263, + 'SETUP_FINALLY': 264, + 'SETUP_WITH': 265, + 'STORE_FAST_MAYBE_NULL': 266, } HAVE_ARGUMENT = 42 diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index f2ce9deaf32284..f2586fcee57d87 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1007,7 +1007,8 @@ def test_boundaries(self): def test_widths(self): long_opcodes = set(['JUMP_BACKWARD_NO_INTERRUPT', 'LOAD_FAST_BORROW_LOAD_FAST_BORROW', - 'INSTRUMENTED_CALL_FUNCTION_EX']) + 'INSTRUMENTED_CALL_FUNCTION_EX', + 'ANNOTATIONS_PLACEHOLDER']) for op, opname in enumerate(dis.opname): if opname in long_opcodes or opname.startswith("INSTRUMENTED"): continue diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-03-21-08-47-36.gh-issue-130907.rGg-ge.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-21-08-47-36.gh-issue-130907.rGg-ge.rst index e9f71f3f06ae1c..587627e979adbb 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-03-21-08-47-36.gh-issue-130907.rGg-ge.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-21-08-47-36.gh-issue-130907.rGg-ge.rst @@ -1,2 +1,3 @@ -Raise an error if the ``__annotations__`` attribute is accessed on a module -that has not finished executing. +If the ``__annotations__`` of a module object are accessed while the +module is executing, return the annotations that have been defined so far, +without caching them. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 07df22c761fc1c..9aecb3f5c8c731 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2026,6 +2026,10 @@ dummy_func( } } + pseudo(ANNOTATIONS_PLACEHOLDER, (--)) = { + NOP, + }; + inst(DICT_UPDATE, (dict, unused[oparg - 1], update -- dict, unused[oparg - 1])) { PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); diff --git a/Python/codegen.c b/Python/codegen.c index 26705a59624a46..0a5570ebd0461f 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -654,6 +654,9 @@ codegen_enter_scope(compiler *c, identifier name, int scope_type, loc.lineno = 0; } ADDOP_I(c, loc, RESUME, RESUME_AT_FUNC_START); + if (scope_type == COMPILE_SCOPE_MODULE) { + ADDOP(c, loc, ANNOTATIONS_PLACEHOLDER); + } return SUCCESS; } @@ -832,9 +835,8 @@ codegen_process_deferred_annotations(compiler *c, location loc) if (nested_instr_seq != NULL) { RETURN_IF_ERROR( - _PyInstructionSequence_InjectSequence(old_instr_seq, 1, nested_instr_seq)); + _PyInstructionSequence_SetAnnotationsCode(old_instr_seq, nested_instr_seq)); _PyCompile_SetInstrSequence(c, old_instr_seq); - PyInstructionSequence_Fini(nested_instr_seq); } return SUCCESS; diff --git a/Python/flowgraph.c b/Python/flowgraph.c index a0d5690250cffb..9e714bf6c4c54d 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -3847,16 +3847,38 @@ _PyCfg_FromInstructionSequence(_PyInstructionSequence *seq) seq->s_instrs[instr->i_oparg].i_target = 1; } } + int offset = 0; for (int i = 0; i < seq->s_used; i++) { _PyInstruction *instr = &seq->s_instrs[i]; + if (instr->i_opcode == ANNOTATIONS_PLACEHOLDER) { + if (seq->s_annotations_code != NULL) { + assert(seq->s_annotations_code->s_labelmap_size == 0 + && seq->s_annotations_code->s_nested == NULL); + for (int j = 0; j < seq->s_annotations_code->s_used; j++) { + _PyInstruction *ann_instr = &seq->s_annotations_code->s_instrs[j]; + assert(!HAS_TARGET(ann_instr->i_opcode)); + if (_PyCfgBuilder_Addop(g, ann_instr->i_opcode, ann_instr->i_oparg, ann_instr->i_loc) < 0) { + goto error; + } + } + offset += seq->s_annotations_code->s_used - 1; + } + else { + offset -= 1; + } + continue; + } if (instr->i_target) { - jump_target_label lbl_ = {i}; + jump_target_label lbl_ = {i + offset}; if (_PyCfgBuilder_UseLabel(g, lbl_) < 0) { goto error; } } int opcode = instr->i_opcode; int oparg = instr->i_oparg; + if (HAS_TARGET(opcode)) { + oparg += offset; + } if (_PyCfgBuilder_Addop(g, opcode, oparg, instr->i_loc) < 0) { goto error; } diff --git a/Python/instruction_sequence.c b/Python/instruction_sequence.c index 87948a675b0bad..b068e4fa3dbf43 100644 --- a/Python/instruction_sequence.c +++ b/Python/instruction_sequence.c @@ -34,9 +34,10 @@ typedef _Py_SourceLocation location; } static int -instr_sequence_grow(instr_sequence *seq, int size) { +instr_sequence_next_inst(instr_sequence *seq) { assert(seq->s_instrs != NULL || seq->s_used == 0); + _Py_c_array_t array = { .array = (void*)seq->s_instrs, .allocated_entries = seq->s_allocated, @@ -44,19 +45,13 @@ instr_sequence_grow(instr_sequence *seq, int size) { .initial_num_entries = INITIAL_INSTR_SEQUENCE_SIZE, }; - RETURN_IF_ERROR(_Py_CArray_EnsureCapacity(&array, seq->s_used + size)); + RETURN_IF_ERROR(_Py_CArray_EnsureCapacity(&array, seq->s_used + 1)); seq->s_instrs = array.array; seq->s_allocated = array.allocated_entries; assert(seq->s_allocated >= 0); assert(seq->s_used < seq->s_allocated); - seq->s_used += size; - return seq->s_used - 1; -} - -static int -instr_sequence_next_inst(instr_sequence *seq) { - return instr_sequence_grow(seq, 1); + return seq->s_used++; } _PyJumpTargetLabel @@ -160,28 +155,11 @@ _PyInstructionSequence_InsertInstruction(instr_sequence *seq, int pos, } int -_PyInstructionSequence_InjectSequence(instr_sequence *seq, int pos, - instr_sequence *injected) +_PyInstructionSequence_SetAnnotationsCode(instr_sequence *seq, + instr_sequence *annotations) { - assert(pos >= 0 && pos <= seq->s_used); - // Merging labelmaps is not supported - assert(injected->s_labelmap_size == 0 && injected->s_nested == NULL); - if (injected->s_used == 0) { - return SUCCESS; - } - - int last_idx = instr_sequence_grow(seq, injected->s_used); - - RETURN_IF_ERROR(last_idx); - for (int i = last_idx - injected->s_used; i >= pos; i--) { - seq->s_instrs[i + injected->s_used] = seq->s_instrs[i]; - } - for (int i=0; i < injected->s_used; i++) { - seq->s_instrs[i + pos] = injected->s_instrs[i]; - } - for(int lbl=0; lbl < seq->s_labelmap_size; lbl++) { - seq->s_labelmap[lbl] += injected->s_used; - } + assert(seq->s_annotations_code == NULL); + seq->s_annotations_code = annotations; return SUCCESS; } @@ -209,6 +187,12 @@ PyInstructionSequence_Fini(instr_sequence *seq) { PyMem_Free(seq->s_instrs); seq->s_instrs = NULL; + + if (seq->s_annotations_code != NULL) { + PyInstructionSequence_Fini(seq->s_annotations_code); + Py_CLEAR(seq->s_annotations_code); + } + } /*[clinic input] @@ -231,6 +215,7 @@ inst_seq_create(void) seq->s_labelmap = NULL; seq->s_labelmap_size = 0; seq->s_nested = NULL; + seq->s_annotations_code = NULL; PyObject_GC_Track(seq); return seq; @@ -445,6 +430,7 @@ inst_seq_traverse(PyObject *op, visitproc visit, void *arg) { _PyInstructionSequence *seq = (_PyInstructionSequence *)op; Py_VISIT(seq->s_nested); + Py_VISIT((PyObject *)seq->s_annotations_code); return 0; } @@ -453,6 +439,7 @@ inst_seq_clear(PyObject *op) { _PyInstructionSequence *seq = (_PyInstructionSequence *)op; Py_CLEAR(seq->s_nested); + Py_CLEAR(seq->s_annotations_code); return 0; } From 719a2e556517de775f78f77c6d2c9c5e7c3856d0 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 21 Apr 2025 08:05:23 -0700 Subject: [PATCH 09/10] fix test --- Lib/test/test_compiler_codegen.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index cc9ecc7e38917b..d02937c84d9534 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -26,6 +26,7 @@ def test_if_expression(self): false_lbl = self.Label() expected = [ ('RESUME', 0, 0), + ('ANNOTATIONS_PLACEHOLDER', None), ('LOAD_CONST', 0, 1), ('TO_BOOL', 0, 1), ('POP_JUMP_IF_FALSE', false_lbl := self.Label(), 1), @@ -45,6 +46,7 @@ def test_for_loop(self): false_lbl = self.Label() expected = [ ('RESUME', 0, 0), + ('ANNOTATIONS_PLACEHOLDER', None), ('LOAD_NAME', 0, 1), ('GET_ITER', None, 1), loop_lbl := self.Label(), @@ -73,6 +75,7 @@ def f(x): expected = [ # Function definition ('RESUME', 0), + ('ANNOTATIONS_PLACEHOLDER', None), ('LOAD_CONST', 0), ('MAKE_FUNCTION', None), ('STORE_NAME', 0), @@ -106,6 +109,7 @@ def g(): expected = [ # Function definition ('RESUME', 0), + ('ANNOTATIONS_PLACEHOLDER', None), ('LOAD_CONST', 0), ('MAKE_FUNCTION', None), ('STORE_NAME', 0), From 862e5bf5776c7c879f6b45c1033d8c901cee2c81 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 27 Apr 2025 16:25:46 -0700 Subject: [PATCH 10/10] Let compile.c manage the instrseq --- Include/internal/pycore_compile.h | 4 ++-- Python/codegen.c | 19 +++++------------- Python/compile.c | 32 ++++++++++++++++++++++++++++--- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index a8f8ec2abcc1d8..a606c2afe0a234 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -133,8 +133,8 @@ int _PyCompile_EnterScope(struct _PyCompiler *c, identifier name, int scope_type void _PyCompile_ExitScope(struct _PyCompiler *c); Py_ssize_t _PyCompile_AddConst(struct _PyCompiler *c, PyObject *o); _PyInstructionSequence *_PyCompile_InstrSequence(struct _PyCompiler *c); -void _PyCompile_SetInstrSequence(struct _PyCompiler *c, - _PyInstructionSequence *instr_sequence); +int _PyCompile_StartAnnotationSetup(struct _PyCompiler *c); +int _PyCompile_EndAnnotationSetup(struct _PyCompiler *c); int _PyCompile_FutureFeatures(struct _PyCompiler *c); void _PyCompile_DeferredAnnotations( struct _PyCompiler *c, PyObject **deferred_annotations, diff --git a/Python/codegen.c b/Python/codegen.c index 0a5570ebd0461f..a7e412f7032c45 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -795,15 +795,12 @@ codegen_process_deferred_annotations(compiler *c, location loc) return SUCCESS; } - instr_sequence *old_instr_seq = INSTR_SEQUENCE(c); - instr_sequence *nested_instr_seq = NULL; int scope_type = SCOPE_TYPE(c); - if (scope_type == COMPILE_SCOPE_MODULE) { - nested_instr_seq = (instr_sequence *)_PyInstructionSequence_New(); - if (nested_instr_seq == NULL) { + bool need_separate_block = scope_type == COMPILE_SCOPE_MODULE; + if (need_separate_block) { + if (_PyCompile_StartAnnotationSetup(c) == ERROR) { goto error; } - _PyCompile_SetInstrSequence(c, nested_instr_seq); } // It's possible that ste_annotations_block is set but @@ -833,18 +830,12 @@ codegen_process_deferred_annotations(compiler *c, location loc) ste->ste_type == ClassBlock ? &_Py_ID(__annotate_func__) : &_Py_ID(__annotate__), Store)); - if (nested_instr_seq != NULL) { - RETURN_IF_ERROR( - _PyInstructionSequence_SetAnnotationsCode(old_instr_seq, nested_instr_seq)); - _PyCompile_SetInstrSequence(c, old_instr_seq); + if (need_separate_block) { + RETURN_IF_ERROR(_PyCompile_EndAnnotationSetup(c)); } return SUCCESS; error: - if (nested_instr_seq != NULL) { - PyInstructionSequence_Fini(nested_instr_seq); - _PyCompile_SetInstrSequence(c, old_instr_seq); - } Py_XDECREF(deferred_anno); Py_XDECREF(conditional_annotation_indices); return ERROR; diff --git a/Python/compile.c b/Python/compile.c index f3c3e6cc92a05f..15ef7214d44b9c 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -64,6 +64,7 @@ struct compiler_unit { long u_next_conditional_annotation_index; /* index of the next conditional annotation */ instr_sequence *u_instr_sequence; /* codegen output */ + instr_sequence *u_stashed_instr_sequence; /* temporarily stashed parent instruction sequence */ int u_nfblocks; int u_in_inlined_comp; @@ -178,6 +179,7 @@ static void compiler_unit_free(struct compiler_unit *u) { Py_CLEAR(u->u_instr_sequence); + Py_CLEAR(u->u_stashed_instr_sequence); Py_CLEAR(u->u_ste); Py_CLEAR(u->u_metadata.u_name); Py_CLEAR(u->u_metadata.u_qualname); @@ -681,6 +683,7 @@ _PyCompile_EnterScope(compiler *c, identifier name, int scope_type, compiler_unit_free(u); return ERROR; } + u->u_stashed_instr_sequence = NULL; /* Push the old compiler_unit on the stack. */ if (c->u) { @@ -1231,12 +1234,35 @@ _PyCompile_InstrSequence(compiler *c) return c->u->u_instr_sequence; } -void -_PyCompile_SetInstrSequence(compiler *c, instr_sequence *seq) +int +_PyCompile_StartAnnotationSetup(struct _PyCompiler *c) { - c->u->u_instr_sequence = seq; + instr_sequence *new_seq = (instr_sequence *)_PyInstructionSequence_New(); + if (new_seq == NULL) { + return ERROR; + } + assert(c->u->u_stashed_instr_sequence == NULL); + c->u->u_stashed_instr_sequence = c->u->u_instr_sequence; + c->u->u_instr_sequence = new_seq; + return SUCCESS; } +int +_PyCompile_EndAnnotationSetup(struct _PyCompiler *c) +{ + assert(c->u->u_stashed_instr_sequence != NULL); + instr_sequence *parent_seq = c->u->u_stashed_instr_sequence; + instr_sequence *anno_seq = c->u->u_instr_sequence; + c->u->u_stashed_instr_sequence = NULL; + c->u->u_instr_sequence = parent_seq; + if (_PyInstructionSequence_SetAnnotationsCode(parent_seq, anno_seq) == ERROR) { + Py_DECREF(anno_seq); + return ERROR; + } + return SUCCESS; +} + + int _PyCompile_FutureFeatures(compiler *c) {