From 0ac6ed45773e4c5347766bc28da1ee1104fec818 Mon Sep 17 00:00:00 2001 From: John Date: Fri, 28 Aug 2026 06:58:09 +0000 Subject: [PATCH 1/2] gh-143732: Specialize `__setitem__` dunder method for `STORE_SUBSCR` Add STORE_SUBSCR_PY_DUNDER, which enters a Python `__setitem__` directly instead of calling it through PyObject_SetItem, following the template the issue asks for. Unlike BINARY_OP_SUBSCR_GETITEM, the frame for the dunder cannot simply be pushed: STORE_SUBSCR must pop three operands and push nothing, but a returning frame always pushes exactly one value (see _RETURN_VALUE). So a shim frame (_Py_SetItemCleanup) is pushed underneath the `__setitem__` frame, in the same way CALL_ALLOC_AND_ENTER_INIT uses _Py_InitCleanup. `__setitem__` returns into the shim, whose EXIT_SETITEM discards the returned value and pops the shim without pushing a result, keeping the net stack effect at -3. The function is cached on the heap type's specialization cache (_spec_cache.setitem) rather than in the inline cache, mirroring _spec_cache.getitem, so STORE_SUBSCR's cache size is unchanged. This is tier 1 only; tier 2 / JIT support is left for a follow-up. Co-Authored-By: Claude Opus 5 (1M context) --- Include/cpython/object.h | 6 + Include/internal/pycore_code.h | 1 + Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 18 +- Include/internal/pycore_typeobject.h | 1 + Include/opcode_ids.h | 230 ++++++++--------- Lib/_opcode_metadata.py | 231 +++++++++--------- Lib/test/test_opcache.py | 68 ++++++ ...-08-28-11-00-00.gh-issue-143732.Kv8pQt.rst | 3 + Modules/_testinternalcapi/test_cases.c.h | 136 +++++++++++ Modules/_testinternalcapi/test_targets.h | 20 +- Objects/typeobject.c | 29 +++ Python/bytecodes.c | 78 ++++++ Python/generated_cases.c.h | 136 +++++++++++ Python/opcode_targets.h | 20 +- Python/record_functions.c.h | 1 + Python/specialize.c | 65 +++++ 17 files changed, 795 insertions(+), 251 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-08-28-11-00-00.gh-issue-143732.Kv8pQt.rst diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 4c5a677e5543ece..c5d48055d72d359 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -268,6 +268,12 @@ struct _specialization_cache { // *args nor **kwargs (as required by BINARY_OP_SUBSCR_GETITEM): PyObject *getitem; uint32_t getitem_version; + // The same contract applies to setitem, except that it is the Python + // function that PyType_Lookup(cls, "__setitem__") would return, and it is + // called with three positional arguments (as required by + // STORE_SUBSCR_PY_DUNDER): + PyObject *setitem; + uint32_t setitem_version; PyObject *init; }; diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 293c1ea4414e23e..82d6762ebc01e4b 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -549,6 +549,7 @@ extern PyObject *_PyInstrumentation_BranchesIterator(PyCodeObject *code); struct _PyCode8 _PyCode_DEF(8); PyAPI_DATA(const struct _PyCode8) _Py_InitCleanup; +PyAPI_DATA(const struct _PyCode8) _Py_SetItemCleanup; #ifdef Py_GIL_DISABLED diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index b6945f2bc5f6e0d..9fde901bd152cce 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -303,6 +303,7 @@ Known values: Python 3.16a1 3703 (Replace DELETE_GLOBAL with PUSH_NULL; STORE_GLOBAL) Python 3.16a1 3704 (Replace DELETE_ATTR with PUSH_NULL; STORE_ATTR) Python 3.16a1 3705 (Add INTRINSIC_ADD_CONDITIONAL_ANNOTATION) + Python 3.16a1 3706 (Add STORE_SUBSCR_PY_DUNDER and EXIT_SETITEM) Python 3.17 will start with 3750 @@ -312,7 +313,7 @@ Known values: */ -#define PYC_MAGIC_NUMBER 3705 +#define PYC_MAGIC_NUMBER 3706 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 38bccdd48c0f862..5fe81c452a0ee35 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -198,6 +198,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case EXIT_INIT_CHECK: return 1; + case EXIT_SETITEM: + return 1; case EXTENDED_ARG: return 0; case FORMAT_SIMPLE: @@ -482,6 +484,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 3; case STORE_SUBSCR_LIST_INT: return 3; + case STORE_SUBSCR_PY_DUNDER: + return 3; case SWAP: return 2 + (oparg-2); case TO_BOOL: @@ -693,6 +697,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case EXIT_INIT_CHECK: return 0; + case EXIT_SETITEM: + return 0; case EXTENDED_ARG: return 0; case FORMAT_SIMPLE: @@ -977,6 +983,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case STORE_SUBSCR_LIST_INT: return 0; + case STORE_SUBSCR_PY_DUNDER: + return 0; case SWAP: return 2 + (oparg-2); case TO_BOOL: @@ -1180,6 +1188,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [EXIT_SETITEM] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1312,6 +1321,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR_PY_DUNDER] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, @@ -1632,6 +1642,7 @@ const char *_PyOpcode_OpName[267] = { [END_SEND] = "END_SEND", [ENTER_EXECUTOR] = "ENTER_EXECUTOR", [EXIT_INIT_CHECK] = "EXIT_INIT_CHECK", + [EXIT_SETITEM] = "EXIT_SETITEM", [EXTENDED_ARG] = "EXTENDED_ARG", [FORMAT_SIMPLE] = "FORMAT_SIMPLE", [FORMAT_WITH_SPEC] = "FORMAT_WITH_SPEC", @@ -1774,6 +1785,7 @@ const char *_PyOpcode_OpName[267] = { [STORE_SUBSCR] = "STORE_SUBSCR", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", + [STORE_SUBSCR_PY_DUNDER] = "STORE_SUBSCR_PY_DUNDER", [SWAP] = "SWAP", [TO_BOOL] = "TO_BOOL", [TO_BOOL_ALWAYS_TRUE] = "TO_BOOL_ALWAYS_TRUE", @@ -1827,7 +1839,6 @@ const uint8_t _PyOpcode_Caches[256] = { PyAPI_DATA(const uint8_t) _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { - [117] = 117, [118] = 118, [119] = 119, [120] = 120, @@ -1838,7 +1849,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [125] = 125, [126] = 126, [127] = 127, - [219] = 219, [220] = 220, [221] = 221, [222] = 222, @@ -1932,6 +1942,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [END_SEND] = END_SEND, [ENTER_EXECUTOR] = ENTER_EXECUTOR, [EXIT_INIT_CHECK] = EXIT_INIT_CHECK, + [EXIT_SETITEM] = EXIT_SETITEM, [EXTENDED_ARG] = EXTENDED_ARG, [FORMAT_SIMPLE] = FORMAT_SIMPLE, [FORMAT_WITH_SPEC] = FORMAT_WITH_SPEC, @@ -2064,6 +2075,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [STORE_SUBSCR] = STORE_SUBSCR, [STORE_SUBSCR_DICT] = STORE_SUBSCR, [STORE_SUBSCR_LIST_INT] = STORE_SUBSCR, + [STORE_SUBSCR_PY_DUNDER] = STORE_SUBSCR, [SWAP] = SWAP, [TO_BOOL] = TO_BOOL, [TO_BOOL_ALWAYS_TRUE] = TO_BOOL, @@ -2088,7 +2100,6 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ - case 117: \ case 118: \ case 119: \ case 120: \ @@ -2099,7 +2110,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 125: \ case 126: \ case 127: \ - case 219: \ case 220: \ case 221: \ case 222: \ diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index ecb3c78a35b9165..249189d7aad03d5 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -164,6 +164,7 @@ typedef int (*_py_validate_type)(PyTypeObject *); // Exported for external JIT support int _PyType_Validate(PyTypeObject *ty, _py_validate_type validate, unsigned int *tp_version); int _PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version); +int _PyType_CacheSetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version); // Precalculates count of non-unique slots and fills wrapperbase.name_count. extern int _PyType_InitSlotDefs(PyInterpreterState *interp); diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 11342ae451b9f6c..7bffc365aca4491 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -22,111 +22,112 @@ extern "C" { #define END_FOR 9 #define END_SEND 10 #define EXIT_INIT_CHECK 11 -#define FORMAT_SIMPLE 12 -#define FORMAT_WITH_SPEC 13 -#define GET_AITER 14 -#define GET_ANEXT 15 -#define GET_LEN 16 +#define EXIT_SETITEM 12 +#define FORMAT_SIMPLE 13 +#define FORMAT_WITH_SPEC 14 +#define GET_AITER 15 +#define GET_ANEXT 16 #define RESERVED 17 -#define INTERPRETER_EXIT 18 -#define LOAD_BUILD_CLASS 19 -#define LOAD_LOCALS 20 -#define MAKE_FUNCTION 21 -#define MATCH_KEYS 22 -#define MATCH_MAPPING 23 -#define MATCH_SEQUENCE 24 -#define NOP 25 -#define NOT_TAKEN 26 -#define POP_EXCEPT 27 -#define POP_ITER 28 -#define POP_TOP 29 -#define PUSH_EXC_INFO 30 -#define PUSH_NULL 31 -#define RETURN_GENERATOR 32 -#define RETURN_VALUE 33 -#define SETUP_ANNOTATIONS 34 -#define STORE_SLICE 35 -#define STORE_SUBSCR 36 -#define TO_BOOL 37 -#define UNARY_INVERT 38 -#define UNARY_NEGATIVE 39 -#define UNARY_NOT 40 -#define WITH_EXCEPT_START 41 -#define BINARY_OP 42 -#define BUILD_INTERPOLATION 43 -#define BUILD_LIST 44 -#define BUILD_MAP 45 -#define BUILD_SET 46 -#define BUILD_SLICE 47 -#define BUILD_STRING 48 -#define BUILD_TUPLE 49 -#define CALL 50 -#define CALL_INTRINSIC_1 51 -#define CALL_INTRINSIC_2 52 -#define CALL_KW 53 -#define COMPARE_OP 54 -#define CONTAINS_OP 55 -#define CONVERT_VALUE 56 -#define COPY 57 -#define COPY_FREE_VARS 58 -#define DELETE_DEREF 59 -#define DELETE_FAST 60 -#define DICT_MERGE 61 -#define DICT_UPDATE 62 -#define END_ASYNC_FOR 63 -#define EXTENDED_ARG 64 -#define FOR_ITER 65 -#define GET_AWAITABLE 66 -#define GET_ITER 67 -#define IMPORT_FROM 68 -#define IMPORT_NAME 69 -#define IS_OP 70 -#define JUMP_BACKWARD 71 -#define JUMP_BACKWARD_NO_INTERRUPT 72 -#define JUMP_FORWARD 73 -#define LIST_APPEND 74 -#define LIST_EXTEND 75 -#define LOAD_ATTR 76 -#define LOAD_COMMON_CONSTANT 77 -#define LOAD_CONST 78 -#define LOAD_DEREF 79 -#define LOAD_FAST 80 -#define LOAD_FAST_AND_CLEAR 81 -#define LOAD_FAST_BORROW 82 -#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 83 -#define LOAD_FAST_CHECK 84 -#define LOAD_FAST_LOAD_FAST 85 -#define LOAD_FROM_DICT_OR_DEREF 86 -#define LOAD_FROM_DICT_OR_GLOBALS 87 -#define LOAD_GLOBAL 88 -#define LOAD_NAME 89 -#define LOAD_SMALL_INT 90 -#define LOAD_SPECIAL 91 -#define LOAD_SUPER_ATTR 92 -#define MAKE_CELL 93 -#define MAP_ADD 94 -#define MATCH_CLASS 95 -#define POP_JUMP_IF_FALSE 96 -#define POP_JUMP_IF_NONE 97 -#define POP_JUMP_IF_NOT_NONE 98 -#define POP_JUMP_IF_TRUE 99 -#define RAISE_VARARGS 100 -#define RERAISE 101 -#define SEND 102 -#define SET_ADD 103 -#define SET_FUNCTION_ATTRIBUTE 104 -#define SET_UPDATE 105 -#define STORE_ATTR 106 -#define STORE_DEREF 107 -#define STORE_FAST 108 -#define STORE_FAST_LOAD_FAST 109 -#define STORE_FAST_STORE_FAST 110 -#define STORE_GLOBAL 111 -#define STORE_NAME 112 -#define SWAP 113 -#define UNPACK_EX 114 -#define UNPACK_SEQUENCE 115 -#define YIELD_VALUE 116 +#define GET_LEN 18 +#define INTERPRETER_EXIT 19 +#define LOAD_BUILD_CLASS 20 +#define LOAD_LOCALS 21 +#define MAKE_FUNCTION 22 +#define MATCH_KEYS 23 +#define MATCH_MAPPING 24 +#define MATCH_SEQUENCE 25 +#define NOP 26 +#define NOT_TAKEN 27 +#define POP_EXCEPT 28 +#define POP_ITER 29 +#define POP_TOP 30 +#define PUSH_EXC_INFO 31 +#define PUSH_NULL 32 +#define RETURN_GENERATOR 33 +#define RETURN_VALUE 34 +#define SETUP_ANNOTATIONS 35 +#define STORE_SLICE 36 +#define STORE_SUBSCR 37 +#define TO_BOOL 38 +#define UNARY_INVERT 39 +#define UNARY_NEGATIVE 40 +#define UNARY_NOT 41 +#define WITH_EXCEPT_START 42 +#define BINARY_OP 43 +#define BUILD_INTERPOLATION 44 +#define BUILD_LIST 45 +#define BUILD_MAP 46 +#define BUILD_SET 47 +#define BUILD_SLICE 48 +#define BUILD_STRING 49 +#define BUILD_TUPLE 50 +#define CALL 51 +#define CALL_INTRINSIC_1 52 +#define CALL_INTRINSIC_2 53 +#define CALL_KW 54 +#define COMPARE_OP 55 +#define CONTAINS_OP 56 +#define CONVERT_VALUE 57 +#define COPY 58 +#define COPY_FREE_VARS 59 +#define DELETE_DEREF 60 +#define DELETE_FAST 61 +#define DICT_MERGE 62 +#define DICT_UPDATE 63 +#define END_ASYNC_FOR 64 +#define EXTENDED_ARG 65 +#define FOR_ITER 66 +#define GET_AWAITABLE 67 +#define GET_ITER 68 +#define IMPORT_FROM 69 +#define IMPORT_NAME 70 +#define IS_OP 71 +#define JUMP_BACKWARD 72 +#define JUMP_BACKWARD_NO_INTERRUPT 73 +#define JUMP_FORWARD 74 +#define LIST_APPEND 75 +#define LIST_EXTEND 76 +#define LOAD_ATTR 77 +#define LOAD_COMMON_CONSTANT 78 +#define LOAD_CONST 79 +#define LOAD_DEREF 80 +#define LOAD_FAST 81 +#define LOAD_FAST_AND_CLEAR 82 +#define LOAD_FAST_BORROW 83 +#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 84 +#define LOAD_FAST_CHECK 85 +#define LOAD_FAST_LOAD_FAST 86 +#define LOAD_FROM_DICT_OR_DEREF 87 +#define LOAD_FROM_DICT_OR_GLOBALS 88 +#define LOAD_GLOBAL 89 +#define LOAD_NAME 90 +#define LOAD_SMALL_INT 91 +#define LOAD_SPECIAL 92 +#define LOAD_SUPER_ATTR 93 +#define MAKE_CELL 94 +#define MAP_ADD 95 +#define MATCH_CLASS 96 +#define POP_JUMP_IF_FALSE 97 +#define POP_JUMP_IF_NONE 98 +#define POP_JUMP_IF_NOT_NONE 99 +#define POP_JUMP_IF_TRUE 100 +#define RAISE_VARARGS 101 +#define RERAISE 102 +#define SEND 103 +#define SET_ADD 104 +#define SET_FUNCTION_ATTRIBUTE 105 +#define SET_UPDATE 106 +#define STORE_ATTR 107 +#define STORE_DEREF 108 +#define STORE_FAST 109 +#define STORE_FAST_LOAD_FAST 110 +#define STORE_FAST_STORE_FAST 111 +#define STORE_GLOBAL 112 +#define STORE_NAME 113 +#define SWAP 114 +#define UNPACK_EX 115 +#define UNPACK_SEQUENCE 116 +#define YIELD_VALUE 117 #define RESUME 128 #define BINARY_OP_ADD_FLOAT 129 #define BINARY_OP_ADD_INT 130 @@ -209,15 +210,16 @@ extern "C" { #define STORE_ATTR_WITH_HINT 207 #define STORE_SUBSCR_DICT 208 #define STORE_SUBSCR_LIST_INT 209 -#define TO_BOOL_ALWAYS_TRUE 210 -#define TO_BOOL_BOOL 211 -#define TO_BOOL_INT 212 -#define TO_BOOL_LIST 213 -#define TO_BOOL_NONE 214 -#define TO_BOOL_STR 215 -#define UNPACK_SEQUENCE_LIST 216 -#define UNPACK_SEQUENCE_TUPLE 217 -#define UNPACK_SEQUENCE_TWO_TUPLE 218 +#define STORE_SUBSCR_PY_DUNDER 210 +#define TO_BOOL_ALWAYS_TRUE 211 +#define TO_BOOL_BOOL 212 +#define TO_BOOL_INT 213 +#define TO_BOOL_LIST 214 +#define TO_BOOL_NONE 215 +#define TO_BOOL_STR 216 +#define UNPACK_SEQUENCE_LIST 217 +#define UNPACK_SEQUENCE_TUPLE 218 +#define UNPACK_SEQUENCE_TWO_TUPLE 219 #define INSTRUMENTED_END_FOR 233 #define INSTRUMENTED_POP_ITER 234 #define INSTRUMENTED_END_SEND 235 @@ -253,7 +255,7 @@ extern "C" { #define SETUP_WITH 265 #define STORE_FAST_MAYBE_NULL 266 -#define HAVE_ARGUMENT 41 +#define HAVE_ARGUMENT 42 #define MIN_SPECIALIZED_OPCODE 129 #define MIN_INSTRUMENTED_OPCODE 233 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index df92eae151d2488..d7fba76f9fa3611 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -36,6 +36,7 @@ STORE_SUBSCR=( "STORE_SUBSCR_DICT", "STORE_SUBSCR_LIST_INT", + "STORE_SUBSCR_PY_DUNDER", ), SEND=( "SEND_GEN", @@ -215,15 +216,16 @@ STORE_ATTR_WITH_HINT=207, STORE_SUBSCR_DICT=208, STORE_SUBSCR_LIST_INT=209, - TO_BOOL_ALWAYS_TRUE=210, - TO_BOOL_BOOL=211, - TO_BOOL_INT=212, - TO_BOOL_LIST=213, - TO_BOOL_NONE=214, - TO_BOOL_STR=215, - UNPACK_SEQUENCE_LIST=216, - UNPACK_SEQUENCE_TUPLE=217, - UNPACK_SEQUENCE_TWO_TUPLE=218, + STORE_SUBSCR_PY_DUNDER=210, + TO_BOOL_ALWAYS_TRUE=211, + TO_BOOL_BOOL=212, + TO_BOOL_INT=213, + TO_BOOL_LIST=214, + TO_BOOL_NONE=215, + TO_BOOL_STR=216, + UNPACK_SEQUENCE_LIST=217, + UNPACK_SEQUENCE_TUPLE=218, + UNPACK_SEQUENCE_TWO_TUPLE=219, ) opmap = frozendict( @@ -243,110 +245,111 @@ END_FOR=9, END_SEND=10, EXIT_INIT_CHECK=11, - FORMAT_SIMPLE=12, - FORMAT_WITH_SPEC=13, - GET_AITER=14, - GET_ANEXT=15, - GET_LEN=16, - INTERPRETER_EXIT=18, - LOAD_BUILD_CLASS=19, - LOAD_LOCALS=20, - MAKE_FUNCTION=21, - MATCH_KEYS=22, - MATCH_MAPPING=23, - MATCH_SEQUENCE=24, - NOP=25, - NOT_TAKEN=26, - POP_EXCEPT=27, - POP_ITER=28, - POP_TOP=29, - PUSH_EXC_INFO=30, - PUSH_NULL=31, - RETURN_GENERATOR=32, - RETURN_VALUE=33, - SETUP_ANNOTATIONS=34, - STORE_SLICE=35, - STORE_SUBSCR=36, - TO_BOOL=37, - UNARY_INVERT=38, - UNARY_NEGATIVE=39, - UNARY_NOT=40, - WITH_EXCEPT_START=41, - BINARY_OP=42, - BUILD_INTERPOLATION=43, - BUILD_LIST=44, - BUILD_MAP=45, - BUILD_SET=46, - BUILD_SLICE=47, - BUILD_STRING=48, - BUILD_TUPLE=49, - CALL=50, - CALL_INTRINSIC_1=51, - CALL_INTRINSIC_2=52, - CALL_KW=53, - COMPARE_OP=54, - CONTAINS_OP=55, - CONVERT_VALUE=56, - COPY=57, - COPY_FREE_VARS=58, - DELETE_DEREF=59, - DELETE_FAST=60, - DICT_MERGE=61, - DICT_UPDATE=62, - END_ASYNC_FOR=63, - EXTENDED_ARG=64, - FOR_ITER=65, - GET_AWAITABLE=66, - GET_ITER=67, - IMPORT_FROM=68, - IMPORT_NAME=69, - IS_OP=70, - JUMP_BACKWARD=71, - JUMP_BACKWARD_NO_INTERRUPT=72, - JUMP_FORWARD=73, - LIST_APPEND=74, - LIST_EXTEND=75, - LOAD_ATTR=76, - LOAD_COMMON_CONSTANT=77, - LOAD_CONST=78, - LOAD_DEREF=79, - LOAD_FAST=80, - LOAD_FAST_AND_CLEAR=81, - LOAD_FAST_BORROW=82, - LOAD_FAST_BORROW_LOAD_FAST_BORROW=83, - LOAD_FAST_CHECK=84, - LOAD_FAST_LOAD_FAST=85, - LOAD_FROM_DICT_OR_DEREF=86, - LOAD_FROM_DICT_OR_GLOBALS=87, - LOAD_GLOBAL=88, - LOAD_NAME=89, - LOAD_SMALL_INT=90, - LOAD_SPECIAL=91, - LOAD_SUPER_ATTR=92, - MAKE_CELL=93, - MAP_ADD=94, - MATCH_CLASS=95, - POP_JUMP_IF_FALSE=96, - POP_JUMP_IF_NONE=97, - POP_JUMP_IF_NOT_NONE=98, - POP_JUMP_IF_TRUE=99, - RAISE_VARARGS=100, - RERAISE=101, - SEND=102, - SET_ADD=103, - SET_FUNCTION_ATTRIBUTE=104, - SET_UPDATE=105, - STORE_ATTR=106, - STORE_DEREF=107, - STORE_FAST=108, - STORE_FAST_LOAD_FAST=109, - STORE_FAST_STORE_FAST=110, - STORE_GLOBAL=111, - STORE_NAME=112, - SWAP=113, - UNPACK_EX=114, - UNPACK_SEQUENCE=115, - YIELD_VALUE=116, + EXIT_SETITEM=12, + FORMAT_SIMPLE=13, + FORMAT_WITH_SPEC=14, + GET_AITER=15, + GET_ANEXT=16, + GET_LEN=18, + INTERPRETER_EXIT=19, + LOAD_BUILD_CLASS=20, + LOAD_LOCALS=21, + MAKE_FUNCTION=22, + MATCH_KEYS=23, + MATCH_MAPPING=24, + MATCH_SEQUENCE=25, + NOP=26, + NOT_TAKEN=27, + POP_EXCEPT=28, + POP_ITER=29, + POP_TOP=30, + PUSH_EXC_INFO=31, + PUSH_NULL=32, + RETURN_GENERATOR=33, + RETURN_VALUE=34, + SETUP_ANNOTATIONS=35, + STORE_SLICE=36, + STORE_SUBSCR=37, + TO_BOOL=38, + UNARY_INVERT=39, + UNARY_NEGATIVE=40, + UNARY_NOT=41, + WITH_EXCEPT_START=42, + BINARY_OP=43, + BUILD_INTERPOLATION=44, + BUILD_LIST=45, + BUILD_MAP=46, + BUILD_SET=47, + BUILD_SLICE=48, + BUILD_STRING=49, + BUILD_TUPLE=50, + CALL=51, + CALL_INTRINSIC_1=52, + CALL_INTRINSIC_2=53, + CALL_KW=54, + COMPARE_OP=55, + CONTAINS_OP=56, + CONVERT_VALUE=57, + COPY=58, + COPY_FREE_VARS=59, + DELETE_DEREF=60, + DELETE_FAST=61, + DICT_MERGE=62, + DICT_UPDATE=63, + END_ASYNC_FOR=64, + EXTENDED_ARG=65, + FOR_ITER=66, + GET_AWAITABLE=67, + GET_ITER=68, + IMPORT_FROM=69, + IMPORT_NAME=70, + IS_OP=71, + JUMP_BACKWARD=72, + JUMP_BACKWARD_NO_INTERRUPT=73, + JUMP_FORWARD=74, + LIST_APPEND=75, + LIST_EXTEND=76, + LOAD_ATTR=77, + LOAD_COMMON_CONSTANT=78, + LOAD_CONST=79, + LOAD_DEREF=80, + LOAD_FAST=81, + LOAD_FAST_AND_CLEAR=82, + LOAD_FAST_BORROW=83, + LOAD_FAST_BORROW_LOAD_FAST_BORROW=84, + LOAD_FAST_CHECK=85, + LOAD_FAST_LOAD_FAST=86, + LOAD_FROM_DICT_OR_DEREF=87, + LOAD_FROM_DICT_OR_GLOBALS=88, + LOAD_GLOBAL=89, + LOAD_NAME=90, + LOAD_SMALL_INT=91, + LOAD_SPECIAL=92, + LOAD_SUPER_ATTR=93, + MAKE_CELL=94, + MAP_ADD=95, + MATCH_CLASS=96, + POP_JUMP_IF_FALSE=97, + POP_JUMP_IF_NONE=98, + POP_JUMP_IF_NOT_NONE=99, + POP_JUMP_IF_TRUE=100, + RAISE_VARARGS=101, + RERAISE=102, + SEND=103, + SET_ADD=104, + SET_FUNCTION_ATTRIBUTE=105, + SET_UPDATE=106, + STORE_ATTR=107, + STORE_DEREF=108, + STORE_FAST=109, + STORE_FAST_LOAD_FAST=110, + STORE_FAST_STORE_FAST=111, + STORE_GLOBAL=112, + STORE_NAME=113, + SWAP=114, + UNPACK_EX=115, + UNPACK_SEQUENCE=116, + YIELD_VALUE=117, INSTRUMENTED_END_FOR=233, INSTRUMENTED_POP_ITER=234, INSTRUMENTED_END_SEND=235, @@ -380,5 +383,5 @@ STORE_FAST_MAYBE_NULL=266, ) -HAVE_ARGUMENT = 41 +HAVE_ARGUMENT = 42 MIN_INSTRUMENTED_OPCODE = 233 diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 7946550ec0db637..5fd64fad9e6eda1 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -2038,6 +2038,74 @@ def __setitem__(self, key, value): store_subscr_dict_subclass_override() self.assert_no_opcode(store_subscr_dict_subclass_override, "STORE_SUBSCR_DICT") + self.assert_specialized(store_subscr_dict_subclass_override, + "STORE_SUBSCR_PY_DUNDER") + + def store_subscr_py_dunder(): + class C: + __slots__ = ("cell",) + def __setitem__(self, key, value): + self.cell = (key, value) + + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + c = C() + c[i] = i + 1 + self.assertEqual(c.cell, (i, i + 1)) + + store_subscr_py_dunder() + self.assert_specialized(store_subscr_py_dunder, "STORE_SUBSCR_PY_DUNDER") + self.assert_no_opcode(store_subscr_py_dunder, "STORE_SUBSCR") + + def store_subscr_py_dunder_returns_value(): + # Like slot_mp_ass_subscript, the return value of __setitem__ is + # ignored rather than required to be None. + class C: + def __setitem__(self, key, value): + self.stored = value + return "ignored" + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + c = C() + c[0] = 7 + self.assertEqual(c.stored, 7) + + store_subscr_py_dunder_returns_value() + self.assert_specialized(store_subscr_py_dunder_returns_value, + "STORE_SUBSCR_PY_DUNDER") + + def store_subscr_py_dunder_raises(): + class C: + def __setitem__(self, key, value): + raise ValueError(key) + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + c = C() + with self.assertRaises(ValueError): + c["boom"] = 1 + + store_subscr_py_dunder_raises() + self.assert_specialized(store_subscr_py_dunder_raises, + "STORE_SUBSCR_PY_DUNDER") + + def store_subscr_py_dunder_deopt(): + # Rebinding __setitem__ invalidates the type's specialization + # cache, so the specialized form must deopt cleanly. + class C: + def __setitem__(self, key, value): + self.stored = ("first", value) + + c = C() + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + c[0] = 1 + self.assertEqual(c.stored, ("first", 1)) + + def replacement(self, key, value): + self.stored = ("second", value) + C.__setitem__ = replacement + c[0] = 2 + self.assertEqual(c.stored, ("second", 2)) + + store_subscr_py_dunder_deopt() @cpython_only @requires_specialization diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-28-11-00-00.gh-issue-143732.Kv8pQt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-28-11-00-00.gh-issue-143732.Kv8pQt.rst new file mode 100644 index 000000000000000..a65db3b5f8eb37f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-28-11-00-00.gh-issue-143732.Kv8pQt.rst @@ -0,0 +1,3 @@ +Specialize :opcode:`STORE_SUBSCR` as ``STORE_SUBSCR_PY_DUNDER`` when the +container's ``__setitem__`` is implemented in Python, entering the method +directly instead of calling it through :c:func:`PyObject_SetItem`. diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 7a75e80298fcd82..d90317a288caf5e 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -6215,6 +6215,37 @@ DISPATCH(); } + TARGET(EXIT_SETITEM) { + #if _Py_TAIL_CALL_INTERP + int opcode = EXIT_SETITEM; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_SETITEM); + _PyStackRef retval; + retval = stack_pointer[-1]; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(retval); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + assert(STACK_LEVEL() == 0); + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(frame->return_offset); + LLTRACE_RESUME_FRAME(); + DISPATCH(); + } + TARGET(EXTENDED_ARG) { #if _Py_TAIL_CALL_INTERP int opcode = EXTENDED_ARG; @@ -12837,6 +12868,111 @@ DISPATCH(); } + TARGET(STORE_SUBSCR_PY_DUNDER) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR_PY_DUNDER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_PY_DUNDER); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef container; + _PyStackRef setitem; + _PyStackRef v; + _PyStackRef sub; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + } + // _STORE_SUBSCR_CHECK_FUNC + { + container = stack_pointer[-2]; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); + if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *setitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.setitem); + if (setitem_o == NULL) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + assert(PyFunction_Check(setitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.setitem_version); + if (((PyFunctionObject *)setitem_o)->func_version != cached_version) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + PyCodeObject *fcode = (PyCodeObject *)PyFunction_GET_CODE(setitem_o); + assert(fcode->co_argcount == 3); + if (!_PyThreadState_HasStackSpace( + tstate, fcode->co_framesize + _Py_SetItemCleanup.co_framesize)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + setitem = PyStackRef_FromPyObjectNew(setitem_o); + } + // _STORE_SUBSCR_INIT_CALL + { + sub = stack_pointer[-1]; + v = stack_pointer[-3]; + STAT_INC(STORE_SUBSCR, hit); + stack_pointer[0] = setitem; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_SetItemCleanup, 0, frame); + _PyFrame_StackPointerInvalidate(frame); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_SETITEM); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked( + tstate, setitem, 3, shim); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; + pushed_frame->localsplus[2] = v; + stack_pointer += -4; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + frame->return_offset = 2u ; + tstate->py_recursion_remaining--; + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(SWAP) { #if _Py_TAIL_CALL_INTERP int opcode = SWAP; diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h index 91b424773224e4b..96fee65f9880809 100644 --- a/Modules/_testinternalcapi/test_targets.h +++ b/Modules/_testinternalcapi/test_targets.h @@ -12,12 +12,13 @@ static void *opcode_targets_table[256] = { &&TARGET_END_FOR, &&TARGET_END_SEND, &&TARGET_EXIT_INIT_CHECK, + &&TARGET_EXIT_SETITEM, &&TARGET_FORMAT_SIMPLE, &&TARGET_FORMAT_WITH_SPEC, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, - &&TARGET_GET_LEN, &&TARGET_RESERVED, + &&TARGET_GET_LEN, &&TARGET_INTERPRETER_EXIT, &&TARGET_LOAD_BUILD_CLASS, &&TARGET_LOAD_LOCALS, @@ -127,7 +128,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -210,6 +210,7 @@ static void *opcode_targets_table[256] = { &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, + &&TARGET_STORE_SUBSCR_PY_DUNDER, &&TARGET_TO_BOOL_ALWAYS_TRUE, &&TARGET_TO_BOOL_BOOL, &&TARGET_TO_BOOL_INT, @@ -232,7 +233,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -376,7 +376,7 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -478,7 +478,7 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -611,6 +611,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXIT_SETITEM(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); @@ -743,6 +744,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_PY_DUNDER(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SWAP(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); @@ -854,6 +856,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [END_SEND] = _TAIL_CALL_END_SEND, [ENTER_EXECUTOR] = _TAIL_CALL_ENTER_EXECUTOR, [EXIT_INIT_CHECK] = _TAIL_CALL_EXIT_INIT_CHECK, + [EXIT_SETITEM] = _TAIL_CALL_EXIT_SETITEM, [EXTENDED_ARG] = _TAIL_CALL_EXTENDED_ARG, [FORMAT_SIMPLE] = _TAIL_CALL_FORMAT_SIMPLE, [FORMAT_WITH_SPEC] = _TAIL_CALL_FORMAT_WITH_SPEC, @@ -986,6 +989,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [STORE_SUBSCR] = _TAIL_CALL_STORE_SUBSCR, [STORE_SUBSCR_DICT] = _TAIL_CALL_STORE_SUBSCR_DICT, [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_STORE_SUBSCR_LIST_INT, + [STORE_SUBSCR_PY_DUNDER] = _TAIL_CALL_STORE_SUBSCR_PY_DUNDER, [SWAP] = _TAIL_CALL_SWAP, [TO_BOOL] = _TAIL_CALL_TO_BOOL, [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TO_BOOL_ALWAYS_TRUE, @@ -1005,7 +1009,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1016,7 +1019,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, [220] = _TAIL_CALL_UNKNOWN_OPCODE, [221] = _TAIL_CALL_UNKNOWN_OPCODE, [222] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1112,6 +1114,7 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [END_SEND] = _TAIL_CALL_TRACE_RECORD, [ENTER_EXECUTOR] = _TAIL_CALL_TRACE_RECORD, [EXIT_INIT_CHECK] = _TAIL_CALL_TRACE_RECORD, + [EXIT_SETITEM] = _TAIL_CALL_TRACE_RECORD, [EXTENDED_ARG] = _TAIL_CALL_TRACE_RECORD, [FORMAT_SIMPLE] = _TAIL_CALL_TRACE_RECORD, [FORMAT_WITH_SPEC] = _TAIL_CALL_TRACE_RECORD, @@ -1244,6 +1247,7 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [STORE_SUBSCR] = _TAIL_CALL_TRACE_RECORD, [STORE_SUBSCR_DICT] = _TAIL_CALL_TRACE_RECORD, [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR_PY_DUNDER] = _TAIL_CALL_TRACE_RECORD, [SWAP] = _TAIL_CALL_TRACE_RECORD, [TO_BOOL] = _TAIL_CALL_TRACE_RECORD, [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TRACE_RECORD, @@ -1263,7 +1267,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1274,7 +1277,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, [220] = _TAIL_CALL_UNKNOWN_OPCODE, [221] = _TAIL_CALL_UNKNOWN_OPCODE, [222] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 572e302df8d80d5..a6812195a33fb30 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1201,6 +1201,8 @@ _PyType_Modified_Unlocked(PyTypeObject *type) // comment on struct _specialization_cache): FT_ATOMIC_STORE_PTR_RELAXED( ((PyHeapTypeObject *)type)->_spec_cache.getitem, NULL); + FT_ATOMIC_STORE_PTR_RELAXED( + ((PyHeapTypeObject *)type)->_spec_cache.setitem, NULL); } } @@ -1286,6 +1288,8 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) // comment on struct _specialization_cache): FT_ATOMIC_STORE_PTR_RELAXED( ((PyHeapTypeObject *)type)->_spec_cache.getitem, NULL); + FT_ATOMIC_STORE_PTR_RELAXED( + ((PyHeapTypeObject *)type)->_spec_cache.setitem, NULL); } } @@ -6320,6 +6324,31 @@ _PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor return can_cache; } +int +_PyType_CacheSetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version) +{ + if (!descriptor || !tp_version) { + return 0; + } + int can_cache; + BEGIN_TYPE_LOCK(); + can_cache = ((PyTypeObject*)ht)->tp_version_tag == tp_version; + // This pointer is invalidated by PyType_Modified (see the comment on + // struct _specialization_cache): + PyFunctionObject *func = (PyFunctionObject *)descriptor; + uint32_t version = _PyFunction_GetVersionForCurrentState(func); + can_cache = can_cache && _PyFunction_IsVersionValid(version); +#ifdef Py_GIL_DISABLED + can_cache = can_cache && _PyObject_HasDeferredRefcount(descriptor); +#endif + if (can_cache) { + FT_ATOMIC_STORE_PTR_RELEASE(ht->_spec_cache.setitem, descriptor); + FT_ATOMIC_STORE_UINT32_RELAXED(ht->_spec_cache.setitem_version, version); + } + END_TYPE_LOCK(); + return can_cache; +} + void _PyType_SetFlags(PyTypeObject *self, unsigned long mask, unsigned long flags) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fb0cdf4d65e060d..d2f015b1633a61c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1393,6 +1393,7 @@ dummy_func( family(STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR) = { STORE_SUBSCR_DICT, STORE_SUBSCR_LIST_INT, + STORE_SUBSCR_PY_DUNDER, }; specializing op(_SPECIALIZE_STORE_SUBSCR, (counter/1, container, sub -- container, sub)) { @@ -1416,6 +1417,83 @@ dummy_func( macro(STORE_SUBSCR) = _SPECIALIZE_STORE_SUBSCR + _STORE_SUBSCR; + /* Specialization for a `__setitem__` implemented in Python. + * + * Unlike BINARY_OP_SUBSCR_GETITEM, we cannot simply push the frame for + * the dunder and let its return value land on the stack: STORE_SUBSCR + * has to pop three operands and push nothing, but a returning frame + * always pushes exactly one value (see _RETURN_VALUE). + * + * So we push a shim frame (_Py_SetItemCleanup) underneath the + * `__setitem__` frame, in the same way CALL_ALLOC_AND_ENTER_INIT uses + * _Py_InitCleanup. `__setitem__` returns into the shim rather than into + * this frame, and the shim's EXIT_SETITEM discards that value and pops + * the shim without pushing anything, leaving the net effect at -3. + */ + tier1 op(_STORE_SUBSCR_CHECK_FUNC, (v, container, sub -- v, container, sub, setitem)) { + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)); + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *setitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.setitem); + DEOPT_IF(setitem_o == NULL); + assert(PyFunction_Check(setitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.setitem_version); + DEOPT_IF(((PyFunctionObject *)setitem_o)->func_version != cached_version); + PyCodeObject *fcode = (PyCodeObject *)PyFunction_GET_CODE(setitem_o); + assert(fcode->co_argcount == 3); + DEOPT_IF(!_PyThreadState_HasStackSpace( + tstate, fcode->co_framesize + _Py_SetItemCleanup.co_framesize)); + setitem = PyStackRef_FromPyObjectNew(setitem_o); + } + + tier1 op(_STORE_SUBSCR_INIT_CALL, (v, container, sub, setitem -- new_frame)) { + STAT_INC(STORE_SUBSCR, hit); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_SetItemCleanup, 0, frame); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_SETITEM); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked( + tstate, setitem, 3, shim); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; + pushed_frame->localsplus[2] = v; + DEAD(container); + DEAD(sub); + DEAD(v); + DEAD(setitem); + SYNC_SP(); + frame->return_offset = INSTRUCTION_SIZE; + /* Account for pushing the extra shim frame. + * We don't check recursion depth here, + * as it will be checked after start_frame */ + tstate->py_recursion_remaining--; + new_frame = PyStackRef_Wrap(pushed_frame); + } + + macro(STORE_SUBSCR_PY_DUNDER) = + unused/1 + + _CHECK_PEP_523 + + _STORE_SUBSCR_CHECK_FUNC + + _STORE_SUBSCR_INIT_CALL + + _PUSH_FRAME; + + /* Only ever executed as the sole instruction of the _Py_SetItemCleanup + * shim frame. Discards whatever `__setitem__` returned (slot_mp_ass_subscript + * ignores it too) and pops the shim frame without pushing a result. */ + tier1 inst(EXIT_SETITEM, (retval -- )) { + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyStackRef_CLOSE(retval); + SAVE_STACK(); + assert(STACK_LEVEL() == 0); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + RELOAD_STACK(); + LOAD_IP(frame->return_offset); + LLTRACE_RESUME_FRAME(); + } + macro(STORE_SUBSCR_LIST_INT) = _GUARD_TOS_INT + _GUARD_NOS_LIST + unused/1 + _STORE_SUBSCR_LIST_INT + _POP_TOP_INT + POP_TOP; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 77c18b3d61fefc7..7b5a4f49658db82 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -6215,6 +6215,37 @@ DISPATCH(); } + TARGET(EXIT_SETITEM) { + #if _Py_TAIL_CALL_INTERP + int opcode = EXIT_SETITEM; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_SETITEM); + _PyStackRef retval; + retval = stack_pointer[-1]; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(retval); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + assert(STACK_LEVEL() == 0); + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(frame->return_offset); + LLTRACE_RESUME_FRAME(); + DISPATCH(); + } + TARGET(EXTENDED_ARG) { #if _Py_TAIL_CALL_INTERP int opcode = EXTENDED_ARG; @@ -12834,6 +12865,111 @@ DISPATCH(); } + TARGET(STORE_SUBSCR_PY_DUNDER) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR_PY_DUNDER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_PY_DUNDER); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef container; + _PyStackRef setitem; + _PyStackRef v; + _PyStackRef sub; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + } + // _STORE_SUBSCR_CHECK_FUNC + { + container = stack_pointer[-2]; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); + if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *setitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.setitem); + if (setitem_o == NULL) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + assert(PyFunction_Check(setitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.setitem_version); + if (((PyFunctionObject *)setitem_o)->func_version != cached_version) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + PyCodeObject *fcode = (PyCodeObject *)PyFunction_GET_CODE(setitem_o); + assert(fcode->co_argcount == 3); + if (!_PyThreadState_HasStackSpace( + tstate, fcode->co_framesize + _Py_SetItemCleanup.co_framesize)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + setitem = PyStackRef_FromPyObjectNew(setitem_o); + } + // _STORE_SUBSCR_INIT_CALL + { + sub = stack_pointer[-1]; + v = stack_pointer[-3]; + STAT_INC(STORE_SUBSCR, hit); + stack_pointer[0] = setitem; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_SetItemCleanup, 0, frame); + _PyFrame_StackPointerInvalidate(frame); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_SETITEM); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked( + tstate, setitem, 3, shim); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; + pushed_frame->localsplus[2] = v; + stack_pointer += -4; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + frame->return_offset = 2u ; + tstate->py_recursion_remaining--; + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(SWAP) { #if _Py_TAIL_CALL_INTERP int opcode = SWAP; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 91b424773224e4b..96fee65f9880809 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -12,12 +12,13 @@ static void *opcode_targets_table[256] = { &&TARGET_END_FOR, &&TARGET_END_SEND, &&TARGET_EXIT_INIT_CHECK, + &&TARGET_EXIT_SETITEM, &&TARGET_FORMAT_SIMPLE, &&TARGET_FORMAT_WITH_SPEC, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, - &&TARGET_GET_LEN, &&TARGET_RESERVED, + &&TARGET_GET_LEN, &&TARGET_INTERPRETER_EXIT, &&TARGET_LOAD_BUILD_CLASS, &&TARGET_LOAD_LOCALS, @@ -127,7 +128,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -210,6 +210,7 @@ static void *opcode_targets_table[256] = { &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, + &&TARGET_STORE_SUBSCR_PY_DUNDER, &&TARGET_TO_BOOL_ALWAYS_TRUE, &&TARGET_TO_BOOL_BOOL, &&TARGET_TO_BOOL_INT, @@ -232,7 +233,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -376,7 +376,7 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -478,7 +478,7 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -611,6 +611,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXIT_SETITEM(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); @@ -743,6 +744,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_PY_DUNDER(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SWAP(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); @@ -854,6 +856,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [END_SEND] = _TAIL_CALL_END_SEND, [ENTER_EXECUTOR] = _TAIL_CALL_ENTER_EXECUTOR, [EXIT_INIT_CHECK] = _TAIL_CALL_EXIT_INIT_CHECK, + [EXIT_SETITEM] = _TAIL_CALL_EXIT_SETITEM, [EXTENDED_ARG] = _TAIL_CALL_EXTENDED_ARG, [FORMAT_SIMPLE] = _TAIL_CALL_FORMAT_SIMPLE, [FORMAT_WITH_SPEC] = _TAIL_CALL_FORMAT_WITH_SPEC, @@ -986,6 +989,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [STORE_SUBSCR] = _TAIL_CALL_STORE_SUBSCR, [STORE_SUBSCR_DICT] = _TAIL_CALL_STORE_SUBSCR_DICT, [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_STORE_SUBSCR_LIST_INT, + [STORE_SUBSCR_PY_DUNDER] = _TAIL_CALL_STORE_SUBSCR_PY_DUNDER, [SWAP] = _TAIL_CALL_SWAP, [TO_BOOL] = _TAIL_CALL_TO_BOOL, [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TO_BOOL_ALWAYS_TRUE, @@ -1005,7 +1009,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1016,7 +1019,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, [220] = _TAIL_CALL_UNKNOWN_OPCODE, [221] = _TAIL_CALL_UNKNOWN_OPCODE, [222] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1112,6 +1114,7 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [END_SEND] = _TAIL_CALL_TRACE_RECORD, [ENTER_EXECUTOR] = _TAIL_CALL_TRACE_RECORD, [EXIT_INIT_CHECK] = _TAIL_CALL_TRACE_RECORD, + [EXIT_SETITEM] = _TAIL_CALL_TRACE_RECORD, [EXTENDED_ARG] = _TAIL_CALL_TRACE_RECORD, [FORMAT_SIMPLE] = _TAIL_CALL_TRACE_RECORD, [FORMAT_WITH_SPEC] = _TAIL_CALL_TRACE_RECORD, @@ -1244,6 +1247,7 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [STORE_SUBSCR] = _TAIL_CALL_TRACE_RECORD, [STORE_SUBSCR_DICT] = _TAIL_CALL_TRACE_RECORD, [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR_PY_DUNDER] = _TAIL_CALL_TRACE_RECORD, [SWAP] = _TAIL_CALL_TRACE_RECORD, [TO_BOOL] = _TAIL_CALL_TRACE_RECORD, [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TRACE_RECORD, @@ -1263,7 +1267,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1274,7 +1277,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, [220] = _TAIL_CALL_UNKNOWN_OPCODE, [221] = _TAIL_CALL_UNKNOWN_OPCODE, [222] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Python/record_functions.c.h b/Python/record_functions.c.h index 98abe3d0505e202..bead250e8ad3ed9 100644 --- a/Python/record_functions.c.h +++ b/Python/record_functions.c.h @@ -133,6 +133,7 @@ const _PyOpcodeRecordEntry _PyOpcode_RecordEntries[256] = { [BINARY_OP_SUBSCR_DICT] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, [BINARY_OP_SUBSCR_GETITEM] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, [STORE_SUBSCR] = {1, {_RECORD_NOS_TYPE_INDEX}}, + [STORE_SUBSCR_PY_DUNDER] = {1, {_RECORD_NOS_TYPE_INDEX}}, [STORE_SUBSCR_LIST_INT] = {1, {_RECORD_NOS_TYPE_INDEX}}, [STORE_SUBSCR_DICT] = {1, {_RECORD_NOS_TYPE_INDEX}}, [SEND] = {1, {_RECORD_3OS_GEN_FUNC_INDEX}}, diff --git a/Python/specialize.c b/Python/specialize.c index 05cb76ff015ff40..a31f9ef5584057b 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1619,6 +1619,26 @@ _Py_Specialize_StoreSubscr(_PyStackRef container_st, _PyStackRef sub_st, _Py_COD specialize(instr, STORE_SUBSCR_DICT); return; } + unsigned int tp_version; + PyObject *descriptor = _PyType_LookupRefAndVersion(container_type, &_Py_ID(__setitem__), &tp_version); + if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type && + container_type->tp_flags & Py_TPFLAGS_HEAPTYPE) + { + PyFunctionObject *func = (PyFunctionObject *)descriptor; + PyCodeObject *fcode = (PyCodeObject *)func->func_code; + int kind = function_kind(fcode); + PyHeapTypeObject *ht = (PyHeapTypeObject *)container_type; + if (kind == SIMPLE_FUNCTION && + fcode->co_argcount == 3 && + _PyInterpreterState_IsSpecializationEnabled(_PyInterpreterState_GET()) && /* Don't specialize if PEP 523 is active */ + _PyType_CacheSetItemForSpecialization(ht, descriptor, (uint32_t)tp_version)) + { + specialize(instr, STORE_SUBSCR_PY_DUNDER); + Py_DECREF(descriptor); + return; + } + } + Py_XDECREF(descriptor); SPECIALIZATION_FAIL(STORE_SUBSCR, store_subscr_fail_kind(container, sub)); unspecialize(instr); } @@ -3077,3 +3097,48 @@ const struct _PyCode8 _Py_InitCleanup = { CACHE, 0, /* RESUME's cache */ } }; + +/* Store subscript cleanup. + * STORE_SUBSCR_PY_DUNDER pushes this shim underneath the frame for a Python + * `__setitem__`, so that the dunder returns into the shim rather than into the + * frame executing STORE_SUBSCR. EXIT_SETITEM then discards the returned value + * and pops the shim without pushing anything, which is what keeps the net + * stack effect of STORE_SUBSCR_PY_DUNDER at -3. + * + * The same constraints as _Py_InitCleanup apply: it is used as a plain code + * object rather than a function, so it must not access globals or builtins, + * the trailing RESUME must never be executed, and it must contain no + * specializable instructions. + */ +#ifdef Py_GIL_DISABLED +static _PyCodeArray setitem_cleanup_tlbc = { + .size = 1, + .entries = {(char*) &_Py_SetItemCleanup.co_code_adaptive}, +}; +#endif + +const struct _PyCode8 _Py_SetItemCleanup = { + _PyVarObject_HEAD_INIT(&PyCode_Type, 2), + .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty), + .co_names = (PyObject *)&_Py_SINGLETON(tuple_empty), + .co_exceptiontable = (PyObject *)&_Py_SINGLETON(bytes_empty), + .co_flags = CO_OPTIMIZED | CO_NO_MONITORING_EVENTS, + .co_localsplusnames = (PyObject *)&_Py_SINGLETON(tuple_empty), + .co_localspluskinds = (PyObject *)&_Py_SINGLETON(bytes_empty), + .co_filename = &_Py_ID(__setitem__), + .co_name = &_Py_ID(__setitem__), + .co_qualname = &_Py_ID(__setitem__), + .co_linetable = (PyObject *)&no_location, + ._co_firsttraceable = 3, + .co_stacksize = 1, + .co_framesize = 1 + FRAME_SPECIALS_SIZE, +#ifdef Py_GIL_DISABLED + .co_tlbc = &setitem_cleanup_tlbc, +#endif + .co_code_adaptive = { + EXIT_SETITEM, 0, + RESUME, RESUME_AT_FUNC_START, + CACHE, 0, /* RESUME's cache */ + CACHE, 0, /* padding */ + } +}; From 85ff74667e10d0db234e0ef2a286c6dba8e46e0f Mon Sep 17 00:00:00 2001 From: John Date: Fri, 28 Aug 2026 07:57:53 +0000 Subject: [PATCH 2/2] gh-143732: Keep the cached `__setitem__` off the value stack Splitting the guard and the frame push into two uops (mirroring _BINARY_OP_SUBSCR_CHECK_FUNC) pushed the looked-up function onto the caller's value stack as a temporary. STORE_SUBSCR already occupies three stack slots, so the extra slot overran the frame's co_stacksize and tripped ASSERT_WITHIN_STACK_BOUNDS in a debug build. BINARY_OP_SUBSCR_GETITEM gets away with this because BINARY_OP only occupies two slots. Merge the guard and the frame push into a single uop so the function stays in a C local. This is fine here because the specialization is tier 1 only and does not need a separately traceable guard uop. Co-Authored-By: Claude Opus 5 (1M context) --- Modules/_testinternalcapi/test_cases.c.h | 20 ++++++-------------- Python/bytecodes.c | 15 ++++++--------- Python/generated_cases.c.h | 20 ++++++-------------- 3 files changed, 18 insertions(+), 37 deletions(-) diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index d90317a288caf5e..ad34eb9d292be8f 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -12879,9 +12879,8 @@ next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_PY_DUNDER); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef container; - _PyStackRef setitem; _PyStackRef v; + _PyStackRef container; _PyStackRef sub; _PyStackRef new_frame; /* Skip 1 cache entry */ @@ -12893,9 +12892,11 @@ JUMP_TO_PREDICTED(STORE_SUBSCR); } } - // _STORE_SUBSCR_CHECK_FUNC + // _STORE_SUBSCR_PY_DUNDER_FRAME { + sub = stack_pointer[-1]; container = stack_pointer[-2]; + v = stack_pointer[-3]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { UPDATE_MISS_STATS(STORE_SUBSCR); @@ -12924,16 +12925,7 @@ assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); JUMP_TO_PREDICTED(STORE_SUBSCR); } - setitem = PyStackRef_FromPyObjectNew(setitem_o); - } - // _STORE_SUBSCR_INIT_CALL - { - sub = stack_pointer[-1]; - v = stack_pointer[-3]; STAT_INC(STORE_SUBSCR, hit); - stack_pointer[0] = setitem; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( @@ -12941,11 +12933,11 @@ _PyFrame_StackPointerInvalidate(frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_SETITEM); _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked( - tstate, setitem, 3, shim); + tstate, PyStackRef_FromPyObjectNew(setitem_o), 3, shim); pushed_frame->localsplus[0] = container; pushed_frame->localsplus[1] = sub; pushed_frame->localsplus[2] = v; - stack_pointer += -4; + stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); frame->return_offset = 2u ; tstate->py_recursion_remaining--; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d2f015b1633a61c..b8bd593b04fcc89 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1430,7 +1430,7 @@ dummy_func( * this frame, and the shim's EXIT_SETITEM discards that value and pops * the shim without pushing anything, leaving the net effect at -3. */ - tier1 op(_STORE_SUBSCR_CHECK_FUNC, (v, container, sub -- v, container, sub, setitem)) { + tier1 op(_STORE_SUBSCR_PY_DUNDER_FRAME, (v, container, sub -- new_frame)) { PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; @@ -1443,23 +1443,21 @@ dummy_func( assert(fcode->co_argcount == 3); DEOPT_IF(!_PyThreadState_HasStackSpace( tstate, fcode->co_framesize + _Py_SetItemCleanup.co_framesize)); - setitem = PyStackRef_FromPyObjectNew(setitem_o); - } - - tier1 op(_STORE_SUBSCR_INIT_CALL, (v, container, sub, setitem -- new_frame)) { STAT_INC(STORE_SUBSCR, hit); + /* `setitem` is deliberately kept in a C local rather than pushed as + * a stack output: STORE_SUBSCR already occupies three stack slots, + * and one more would exceed the frame's co_stacksize. */ _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_SetItemCleanup, 0, frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_SETITEM); _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked( - tstate, setitem, 3, shim); + tstate, PyStackRef_FromPyObjectNew(setitem_o), 3, shim); pushed_frame->localsplus[0] = container; pushed_frame->localsplus[1] = sub; pushed_frame->localsplus[2] = v; DEAD(container); DEAD(sub); DEAD(v); - DEAD(setitem); SYNC_SP(); frame->return_offset = INSTRUCTION_SIZE; /* Account for pushing the extra shim frame. @@ -1472,8 +1470,7 @@ dummy_func( macro(STORE_SUBSCR_PY_DUNDER) = unused/1 + _CHECK_PEP_523 + - _STORE_SUBSCR_CHECK_FUNC + - _STORE_SUBSCR_INIT_CALL + + _STORE_SUBSCR_PY_DUNDER_FRAME + _PUSH_FRAME; /* Only ever executed as the sole instruction of the _Py_SetItemCleanup diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7b5a4f49658db82..e14def72d5281db 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -12876,9 +12876,8 @@ next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_PY_DUNDER); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef container; - _PyStackRef setitem; _PyStackRef v; + _PyStackRef container; _PyStackRef sub; _PyStackRef new_frame; /* Skip 1 cache entry */ @@ -12890,9 +12889,11 @@ JUMP_TO_PREDICTED(STORE_SUBSCR); } } - // _STORE_SUBSCR_CHECK_FUNC + // _STORE_SUBSCR_PY_DUNDER_FRAME { + sub = stack_pointer[-1]; container = stack_pointer[-2]; + v = stack_pointer[-3]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { UPDATE_MISS_STATS(STORE_SUBSCR); @@ -12921,16 +12922,7 @@ assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); JUMP_TO_PREDICTED(STORE_SUBSCR); } - setitem = PyStackRef_FromPyObjectNew(setitem_o); - } - // _STORE_SUBSCR_INIT_CALL - { - sub = stack_pointer[-1]; - v = stack_pointer[-3]; STAT_INC(STORE_SUBSCR, hit); - stack_pointer[0] = setitem; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( @@ -12938,11 +12930,11 @@ _PyFrame_StackPointerInvalidate(frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_SETITEM); _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked( - tstate, setitem, 3, shim); + tstate, PyStackRef_FromPyObjectNew(setitem_o), 3, shim); pushed_frame->localsplus[0] = container; pushed_frame->localsplus[1] = sub; pushed_frame->localsplus[2] = v; - stack_pointer += -4; + stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); frame->return_offset = 2u ; tstate->py_recursion_remaining--;