From 9a4879d75cad85d2d68db46d3f5c360499ef39b9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 12 Mar 2024 11:10:27 +0000 Subject: [PATCH 01/24] Subdivide uop instruction into deopt, exit and error targets --- Include/cpython/optimizer.h | 39 +++- Python/ceval.c | 5 +- Python/executor_cases.c.h | 228 +++++++++++------------ Python/optimizer.c | 4 +- Tools/cases_generator/tier2_generator.py | 2 +- 5 files changed, 156 insertions(+), 122 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 6d7b8bc3c1433a..c072dfb60d32e7 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -30,16 +30,51 @@ typedef struct { PyCodeObject *code; // Weak (NULL if no corresponding ENTER_EXECUTOR). } _PyVMData; +#define UOP_FORMAT_TARGET 0 +#define UOP_FORMAT_EXIT 1 +#define UOP_FORMAT_DEOPT 2 + typedef struct { - uint16_t opcode; + uint16_t opcode:14; + uint16_t format:2; uint16_t oparg; union { uint32_t target; - uint32_t exit_index; + struct { + union { + uint16_t exit_index; + uint16_t deopt_target; + }; + uint16_t error_target; + }; }; uint64_t operand; // A cache entry } _PyUOpInstruction; +static inline uint32_t uop_get_target(_PyUOpInstruction *inst) +{ + assert(inst->format == UOP_FORMAT_TARGET); + return inst->target; +} + +static inline uint16_t uop_get_exit_index(_PyUOpInstruction *inst) +{ + assert(inst->format == UOP_FORMAT_EXIT); + return inst->exit_index; +} + +static inline uint16_t uop_get_deopt_target(_PyUOpInstruction *inst) +{ + assert(inst->format == UOP_FORMAT_DEOPT); + return inst->deopt_target; +} + +static inline uint16_t uop_get_error_target(_PyUOpInstruction *inst) +{ + assert(inst->format != UOP_FORMAT_TARGET); + return inst->error_target; +} + typedef struct _exit_data { uint32_t target; int16_t temperature; diff --git a/Python/ceval.c b/Python/ceval.c index f817f288903694..79aa04f9ea8cbd 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -981,10 +981,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #define GOTO_ERROR(LABEL) goto LABEL ## _tier_two #undef DEOPT_IF -#define DEOPT_IF(COND, INSTNAME) \ - if ((COND)) { \ - goto deoptimize;\ - } +#define DEOPTIMIZE goto deoptimize #ifdef Py_STATS // Disable these macros that apply to Tier 1 stats when we are in Tier 2 diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 42e884c20ba04f..4108fe38d2d144 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -14,13 +14,13 @@ case _RESUME_CHECK: { #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) goto deoptimize; + if (_Py_emscripten_signal_clock == 0) DEOPTIMIZE; _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) goto deoptimize; + if (eval_breaker != version) DEOPTIMIZE; break; } @@ -617,12 +617,12 @@ PyObject *res; sub = stack_pointer[-1]; list = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) goto deoptimize; - if (!PyList_CheckExact(list)) goto deoptimize; + if (!PyLong_CheckExact(sub)) DEOPTIMIZE; + if (!PyList_CheckExact(list)) DEOPTIMIZE; // Deopt unless 0 <= sub < PyList_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) DEOPTIMIZE; Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyList_GET_SIZE(list)) goto deoptimize; + if (index >= PyList_GET_SIZE(list)) DEOPTIMIZE; STAT_INC(BINARY_SUBSCR, hit); res = PyList_GET_ITEM(list, index); assert(res != NULL); @@ -640,14 +640,14 @@ PyObject *res; sub = stack_pointer[-1]; str = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) goto deoptimize; - if (!PyUnicode_CheckExact(str)) goto deoptimize; - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; + if (!PyLong_CheckExact(sub)) DEOPTIMIZE; + if (!PyUnicode_CheckExact(str)) DEOPTIMIZE; + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) DEOPTIMIZE; Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) goto deoptimize; + if (PyUnicode_GET_LENGTH(str) <= index) DEOPTIMIZE; // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) goto deoptimize; + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) DEOPTIMIZE; STAT_INC(BINARY_SUBSCR, hit); res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); @@ -663,12 +663,12 @@ PyObject *res; sub = stack_pointer[-1]; tuple = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) goto deoptimize; - if (!PyTuple_CheckExact(tuple)) goto deoptimize; + if (!PyLong_CheckExact(sub)) DEOPTIMIZE; + if (!PyTuple_CheckExact(tuple)) DEOPTIMIZE; // Deopt unless 0 <= sub < PyTuple_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) DEOPTIMIZE; Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) goto deoptimize; + if (index >= PyTuple_GET_SIZE(tuple)) DEOPTIMIZE; STAT_INC(BINARY_SUBSCR, hit); res = PyTuple_GET_ITEM(tuple, index); assert(res != NULL); @@ -686,7 +686,7 @@ PyObject *res; sub = stack_pointer[-1]; dict = stack_pointer[-2]; - if (!PyDict_CheckExact(dict)) goto deoptimize; + if (!PyDict_CheckExact(dict)) DEOPTIMIZE; STAT_INC(BINARY_SUBSCR, hit); int rc = PyDict_GetItemRef(dict, sub, &res); if (rc == 0) { @@ -751,13 +751,13 @@ sub = stack_pointer[-1]; list = stack_pointer[-2]; value = stack_pointer[-3]; - if (!PyLong_CheckExact(sub)) goto deoptimize; - if (!PyList_CheckExact(list)) goto deoptimize; + if (!PyLong_CheckExact(sub)) DEOPTIMIZE; + if (!PyList_CheckExact(list)) DEOPTIMIZE; // Ensure nonnegative, zero-or-one-digit ints. - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) DEOPTIMIZE; Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; // Ensure index < len(list) - if (index >= PyList_GET_SIZE(list)) goto deoptimize; + if (index >= PyList_GET_SIZE(list)) DEOPTIMIZE; STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); @@ -776,7 +776,7 @@ sub = stack_pointer[-1]; dict = stack_pointer[-2]; value = stack_pointer[-3]; - if (!PyDict_CheckExact(dict)) goto deoptimize; + if (!PyDict_CheckExact(dict)) DEOPTIMIZE; STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); @@ -1066,8 +1066,8 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; assert(oparg == 2); - if (!PyTuple_CheckExact(seq)) goto deoptimize; - if (PyTuple_GET_SIZE(seq) != 2) goto deoptimize; + if (!PyTuple_CheckExact(seq)) DEOPTIMIZE; + if (PyTuple_GET_SIZE(seq) != 2) DEOPTIMIZE; STAT_INC(UNPACK_SEQUENCE, hit); val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); @@ -1084,8 +1084,8 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; values = &stack_pointer[-1]; - if (!PyTuple_CheckExact(seq)) goto deoptimize; - if (PyTuple_GET_SIZE(seq) != oparg) goto deoptimize; + if (!PyTuple_CheckExact(seq)) DEOPTIMIZE; + if (PyTuple_GET_SIZE(seq) != oparg) DEOPTIMIZE; STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -1102,8 +1102,8 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; values = &stack_pointer[-1]; - if (!PyList_CheckExact(seq)) goto deoptimize; - if (PyList_GET_SIZE(seq) != oparg) goto deoptimize; + if (!PyList_CheckExact(seq)) DEOPTIMIZE; + if (PyList_GET_SIZE(seq) != oparg) DEOPTIMIZE; STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -1307,8 +1307,8 @@ case _GUARD_GLOBALS_VERSION: { uint16_t version = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) goto deoptimize; - if (dict->ma_keys->dk_version != version) goto deoptimize; + if (!PyDict_CheckExact(dict)) DEOPTIMIZE; + if (dict->ma_keys->dk_version != version) DEOPTIMIZE; assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1316,8 +1316,8 @@ case _GUARD_BUILTINS_VERSION: { uint16_t version = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)BUILTINS(); - if (!PyDict_CheckExact(dict)) goto deoptimize; - if (dict->ma_keys->dk_version != version) goto deoptimize; + if (!PyDict_CheckExact(dict)) DEOPTIMIZE; + if (dict->ma_keys->dk_version != version) DEOPTIMIZE; assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1330,7 +1330,7 @@ PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; - if (res == NULL) goto deoptimize; + if (res == NULL) DEOPTIMIZE; Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1348,7 +1348,7 @@ PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; - if (res == NULL) goto deoptimize; + if (res == NULL) DEOPTIMIZE; Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1697,8 +1697,8 @@ class = stack_pointer[-2]; global_super = stack_pointer[-3]; assert(!(oparg & 1)); - if (global_super != (PyObject *)&PySuper_Type) goto deoptimize; - if (!PyType_Check(class)) goto deoptimize; + if (global_super != (PyObject *)&PySuper_Type) DEOPTIMIZE; + if (!PyType_Check(class)) DEOPTIMIZE; STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); @@ -1722,8 +1722,8 @@ class = stack_pointer[-2]; global_super = stack_pointer[-3]; assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) goto deoptimize; - if (!PyType_Check(class)) goto deoptimize; + if (global_super != (PyObject *)&PySuper_Type) DEOPTIMIZE; + if (!PyType_Check(class)) DEOPTIMIZE; STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -1806,7 +1806,7 @@ assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) goto deoptimize; + if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) DEOPTIMIZE; break; } @@ -1819,7 +1819,7 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - if (attr == NULL) goto deoptimize; + if (attr == NULL) DEOPTIMIZE; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1837,7 +1837,7 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - if (attr == NULL) goto deoptimize; + if (attr == NULL) DEOPTIMIZE; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1854,10 +1854,10 @@ PyObject *owner; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)CURRENT_OPERAND(); - if (!PyModule_CheckExact(owner)) goto deoptimize; + if (!PyModule_CheckExact(owner)) DEOPTIMIZE; PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - if (dict->ma_keys->dk_version != dict_version) goto deoptimize; + if (dict->ma_keys->dk_version != dict_version) DEOPTIMIZE; break; } @@ -1873,7 +1873,7 @@ assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; attr = ep->me_value; - if (attr == NULL) goto deoptimize; + if (attr == NULL) DEOPTIMIZE; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1889,9 +1889,9 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - if (_PyDictOrValues_IsValues(dorv)) goto deoptimize; + if (_PyDictOrValues_IsValues(dorv)) DEOPTIMIZE; PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - if (dict == NULL) goto deoptimize; + if (dict == NULL) DEOPTIMIZE; assert(PyDict_CheckExact((PyObject *)dict)); break; } @@ -1905,19 +1905,19 @@ uint16_t hint = (uint16_t)CURRENT_OPERAND(); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - if (hint >= (size_t)dict->ma_keys->dk_nentries) goto deoptimize; + if (hint >= (size_t)dict->ma_keys->dk_nentries) DEOPTIMIZE; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) goto deoptimize; + if (ep->me_key != name) DEOPTIMIZE; attr = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) goto deoptimize; + if (ep->me_key != name) DEOPTIMIZE; attr = ep->me_value; } - if (attr == NULL) goto deoptimize; + if (attr == NULL) DEOPTIMIZE; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1937,7 +1937,7 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); char *addr = (char *)owner + index; attr = *(PyObject **)addr; - if (attr == NULL) goto deoptimize; + if (attr == NULL) DEOPTIMIZE; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1955,7 +1955,7 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); char *addr = (char *)owner + index; attr = *(PyObject **)addr; - if (attr == NULL) goto deoptimize; + if (attr == NULL) DEOPTIMIZE; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1972,9 +1972,9 @@ PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)CURRENT_OPERAND(); - if (!PyType_Check(owner)) goto deoptimize; + if (!PyType_Check(owner)) DEOPTIMIZE; assert(type_version != 0); - if (((PyTypeObject *)owner)->tp_version_tag != type_version) goto deoptimize; + if (((PyTypeObject *)owner)->tp_version_tag != type_version) DEOPTIMIZE; break; } @@ -2023,7 +2023,7 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(dorv)) goto deoptimize; + if (!_PyDictOrValues_IsValues(dorv)) DEOPTIMIZE; break; } @@ -2118,8 +2118,8 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!_PyLong_IsCompact((PyLongObject *)left)) goto deoptimize; - if (!_PyLong_IsCompact((PyLongObject *)right)) goto deoptimize; + if (!_PyLong_IsCompact((PyLongObject *)left)) DEOPTIMIZE; + if (!_PyLong_IsCompact((PyLongObject *)right)) DEOPTIMIZE; STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && _PyLong_DigitCount((PyLongObject *)right) <= 1); @@ -2198,7 +2198,7 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))) goto deoptimize; + if (!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))) DEOPTIMIZE; STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! int res = _PySet_Contains((PySetObject *)right, left); @@ -2218,7 +2218,7 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyDict_CheckExact(right)) goto deoptimize; + if (!PyDict_CheckExact(right)) DEOPTIMIZE; STAT_INC(CONTAINS_OP, hit); int res = PyDict_Contains(right, left); Py_DECREF(left); @@ -2438,7 +2438,7 @@ Py_DECREF(iter); STACK_SHRINK(1); /* The translator sets the deopt target just past END_FOR */ - if (true) goto deoptimize; + if (true) DEOPTIMIZE; } // Common case: no jump, leave it to the code generator stack_pointer[0] = next; @@ -2451,7 +2451,7 @@ case _ITER_CHECK_LIST: { PyObject *iter; iter = stack_pointer[-1]; - if (Py_TYPE(iter) != &PyListIter_Type) goto deoptimize; + if (Py_TYPE(iter) != &PyListIter_Type) DEOPTIMIZE; break; } @@ -2463,8 +2463,8 @@ _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; - if (seq == NULL) goto deoptimize; - if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) goto deoptimize; + if (seq == NULL) DEOPTIMIZE; + if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) DEOPTIMIZE; break; } @@ -2486,7 +2486,7 @@ case _ITER_CHECK_TUPLE: { PyObject *iter; iter = stack_pointer[-1]; - if (Py_TYPE(iter) != &PyTupleIter_Type) goto deoptimize; + if (Py_TYPE(iter) != &PyTupleIter_Type) DEOPTIMIZE; break; } @@ -2498,8 +2498,8 @@ _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; - if (seq == NULL) goto deoptimize; - if (it->it_index >= PyTuple_GET_SIZE(seq)) goto deoptimize; + if (seq == NULL) DEOPTIMIZE; + if (it->it_index >= PyTuple_GET_SIZE(seq)) DEOPTIMIZE; break; } @@ -2522,7 +2522,7 @@ PyObject *iter; iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - if (Py_TYPE(r) != &PyRangeIter_Type) goto deoptimize; + if (Py_TYPE(r) != &PyRangeIter_Type) DEOPTIMIZE; break; } @@ -2533,7 +2533,7 @@ iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); - if (r->len <= 0) goto deoptimize; + if (r->len <= 0) DEOPTIMIZE; break; } @@ -2700,7 +2700,7 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) goto deoptimize; + if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) DEOPTIMIZE; break; } @@ -2710,7 +2710,7 @@ uint32_t keys_version = (uint32_t)CURRENT_OPERAND(); PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - if (owner_heap_type->ht_cached_keys->dk_version != keys_version) goto deoptimize; + if (owner_heap_type->ht_cached_keys->dk_version != keys_version) DEOPTIMIZE; break; } @@ -2792,7 +2792,7 @@ assert(dictoffset > 0); PyObject *dict = *(PyObject **)((char *)owner + dictoffset); /* This object has a __dict__, just not yet created */ - if (dict != NULL) goto deoptimize; + if (dict != NULL) DEOPTIMIZE; break; } @@ -2825,8 +2825,8 @@ oparg = CURRENT_OPARG(); null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - if (null != NULL) goto deoptimize; - if (Py_TYPE(callable) != &PyMethod_Type) goto deoptimize; + if (null != NULL) DEOPTIMIZE; + if (Py_TYPE(callable) != &PyMethod_Type) DEOPTIMIZE; break; } @@ -2848,7 +2848,7 @@ } case _CHECK_PEP_523: { - if (tstate->interp->eval_frame) goto deoptimize; + if (tstate->interp->eval_frame) DEOPTIMIZE; break; } @@ -2859,11 +2859,11 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND(); - if (!PyFunction_Check(callable)) goto deoptimize; + if (!PyFunction_Check(callable)) DEOPTIMIZE; PyFunctionObject *func = (PyFunctionObject *)callable; - if (func->func_version != func_version) goto deoptimize; + if (func->func_version != func_version) DEOPTIMIZE; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (self_or_null != NULL)) goto deoptimize; + if (code->co_argcount != oparg + (self_or_null != NULL)) DEOPTIMIZE; break; } @@ -2873,8 +2873,8 @@ callable = stack_pointer[-2 - oparg]; PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) goto deoptimize; - if (tstate->py_recursion_remaining <= 1) goto deoptimize; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) DEOPTIMIZE; + if (tstate->py_recursion_remaining <= 1) DEOPTIMIZE; break; } @@ -3068,8 +3068,8 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; assert(oparg == 1); - if (null != NULL) goto deoptimize; - if (callable != (PyObject *)&PyType_Type) goto deoptimize; + if (null != NULL) DEOPTIMIZE; + if (callable != (PyObject *)&PyType_Type) DEOPTIMIZE; STAT_INC(CALL, hit); res = Py_NewRef(Py_TYPE(arg)); Py_DECREF(arg); @@ -3088,8 +3088,8 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; assert(oparg == 1); - if (null != NULL) goto deoptimize; - if (callable != (PyObject *)&PyUnicode_Type) goto deoptimize; + if (null != NULL) DEOPTIMIZE; + if (callable != (PyObject *)&PyUnicode_Type) DEOPTIMIZE; STAT_INC(CALL, hit); res = PyObject_Str(arg); Py_DECREF(arg); @@ -3110,8 +3110,8 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; assert(oparg == 1); - if (null != NULL) goto deoptimize; - if (callable != (PyObject *)&PyTuple_Type) goto deoptimize; + if (null != NULL) DEOPTIMIZE; + if (callable != (PyObject *)&PyTuple_Type) DEOPTIMIZE; STAT_INC(CALL, hit); res = PySequence_Tuple(arg); Py_DECREF(arg); @@ -3152,9 +3152,9 @@ args--; total_args++; } - if (!PyType_Check(callable)) goto deoptimize; + if (!PyType_Check(callable)) DEOPTIMIZE; PyTypeObject *tp = (PyTypeObject *)callable; - if (tp->tp_vectorcall == NULL) goto deoptimize; + if (tp->tp_vectorcall == NULL) DEOPTIMIZE; STAT_INC(CALL, hit); res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); /* Free the arguments. */ @@ -3184,9 +3184,9 @@ args--; total_args++; } - if (total_args != 1) goto deoptimize; - if (!PyCFunction_CheckExact(callable)) goto deoptimize; - if (PyCFunction_GET_FLAGS(callable) != METH_O) goto deoptimize; + if (total_args != 1) DEOPTIMIZE; + if (!PyCFunction_CheckExact(callable)) DEOPTIMIZE; + if (PyCFunction_GET_FLAGS(callable) != METH_O) DEOPTIMIZE; STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); // This is slower but CPython promises to check all non-vectorcall @@ -3222,8 +3222,8 @@ args--; total_args++; } - if (!PyCFunction_CheckExact(callable)) goto deoptimize; - if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) goto deoptimize; + if (!PyCFunction_CheckExact(callable)) DEOPTIMIZE; + if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) DEOPTIMIZE; STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ @@ -3264,8 +3264,8 @@ args--; total_args++; } - if (!PyCFunction_CheckExact(callable)) goto deoptimize; - if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) goto deoptimize; + if (!PyCFunction_CheckExact(callable)) DEOPTIMIZE; + if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) DEOPTIMIZE; STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ PyCFunctionFastWithKeywords cfunc = @@ -3300,9 +3300,9 @@ args--; total_args++; } - if (total_args != 1) goto deoptimize; + if (total_args != 1) DEOPTIMIZE; PyInterpreterState *interp = tstate->interp; - if (callable != interp->callable_cache.len) goto deoptimize; + if (callable != interp->callable_cache.len) DEOPTIMIZE; STAT_INC(CALL, hit); PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); @@ -3334,9 +3334,9 @@ args--; total_args++; } - if (total_args != 2) goto deoptimize; + if (total_args != 2) DEOPTIMIZE; PyInterpreterState *interp = tstate->interp; - if (callable != interp->callable_cache.isinstance) goto deoptimize; + if (callable != interp->callable_cache.isinstance) DEOPTIMIZE; STAT_INC(CALL, hit); PyObject *cls = args[1]; PyObject *inst = args[0]; @@ -3370,13 +3370,13 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (total_args != 2) goto deoptimize; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; + if (total_args != 2) DEOPTIMIZE; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) DEOPTIMIZE; PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_O) goto deoptimize; + if (meth->ml_flags != METH_O) DEOPTIMIZE; PyObject *arg = args[1]; PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; + if (!Py_IS_TYPE(self, method->d_common.d_type)) DEOPTIMIZE; STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -3412,12 +3412,12 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) DEOPTIMIZE; PyMethodDef *meth = method->d_method; - if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) goto deoptimize; + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) DEOPTIMIZE; PyTypeObject *d_type = method->d_common.d_type; PyObject *self = args[0]; - if (!Py_IS_TYPE(self, d_type)) goto deoptimize; + if (!Py_IS_TYPE(self, d_type)) DEOPTIMIZE; STAT_INC(CALL, hit); int nargs = total_args - 1; PyCFunctionFastWithKeywords cfunc = @@ -3451,13 +3451,13 @@ args--; total_args++; } - if (total_args != 1) goto deoptimize; + if (total_args != 1) DEOPTIMIZE; PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) DEOPTIMIZE; PyMethodDef *meth = method->d_method; PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; - if (meth->ml_flags != METH_NOARGS) goto deoptimize; + if (!Py_IS_TYPE(self, method->d_common.d_type)) DEOPTIMIZE; + if (meth->ml_flags != METH_NOARGS) DEOPTIMIZE; STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -3493,11 +3493,11 @@ } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; /* Builtin METH_FASTCALL methods, without keywords */ - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) DEOPTIMIZE; PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_FASTCALL) goto deoptimize; + if (meth->ml_flags != METH_FASTCALL) DEOPTIMIZE; PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; + if (!Py_IS_TYPE(self, method->d_common.d_type)) DEOPTIMIZE; STAT_INC(CALL, hit); PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; @@ -3767,7 +3767,7 @@ } case _CHECK_VALIDITY: { - if (!current_executor->vm_data.valid) goto deoptimize; + if (!current_executor->vm_data.valid) DEOPTIMIZE; break; } @@ -3827,7 +3827,7 @@ case _CHECK_FUNCTION: { uint32_t func_version = (uint32_t)CURRENT_OPERAND(); assert(PyFunction_Check(frame->f_funcobj)); - if (((PyFunctionObject *)frame->f_funcobj)->func_version != func_version) goto deoptimize; + if (((PyFunctionObject *)frame->f_funcobj)->func_version != func_version) DEOPTIMIZE; break; } @@ -3893,7 +3893,7 @@ case _CHECK_VALIDITY_AND_SET_IP: { PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND(); - if (!current_executor->vm_data.valid) goto deoptimize; + if (!current_executor->vm_data.valid) DEOPTIMIZE; frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; break; } diff --git a/Python/optimizer.c b/Python/optimizer.c index aaf75b2339cd2e..5bcd501238d7da 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -437,6 +437,7 @@ BRANCH_TO_GUARD[4][2] = { #define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ assert(trace_length < max_length); \ trace[trace_length].opcode = (OPCODE); \ + trace[trace_length].format = UOP_FORMAT_TARGET; \ trace[trace_length].oparg = (OPARG); \ trace[trace_length].target = (TARGET); \ trace[trace_length].operand = (OPERAND); \ @@ -450,6 +451,7 @@ BRANCH_TO_GUARD[4][2] = { #define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ assert(trace_length < max_length); \ trace[trace_length].opcode = (OPCODE); \ + trace[trace_length].format = UOP_FORMAT_TARGET; \ trace[trace_length].oparg = (OPARG); \ trace[trace_length].target = (TARGET); \ trace[trace_length].operand = (OPERAND); \ @@ -1117,7 +1119,7 @@ counter_optimize( _PyUOpInstruction buffer[3] = { { .opcode = _LOAD_CONST_INLINE_BORROW, .operand = (uintptr_t)self }, { .opcode = _INTERNAL_INCREMENT_OPT_COUNTER }, - { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)) } + { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)), .format=UOP_FORMAT_TARGET } }; _PyExecutorObject *executor = make_executor_from_uops(buffer, &EMPTY_FILTER); if (executor == NULL) { diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index d8eed1078b0914..a2d62f4bf026c5 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -100,7 +100,7 @@ def tier2_replace_deopt( out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "RPAREN") next(tkn_iter) # Semi colon - out.emit(") goto deoptimize;\n") + out.emit(") DEOPTIMIZE;\n") def tier2_replace_exit_if( From b6b6426ac44b46cff993aba363356d7ad3bee99d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 12 Mar 2024 11:39:35 +0000 Subject: [PATCH 02/24] Pass length of trace around. --- Include/cpython/optimizer.h | 8 ++++---- Python/ceval.c | 7 +++++++ Python/optimizer.c | 32 ++++++++++++++++++-------------- Python/optimizer_analysis.c | 18 +++++++++--------- 4 files changed, 38 insertions(+), 27 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index c072dfb60d32e7..e0b4c4842b7623 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -51,25 +51,25 @@ typedef struct { uint64_t operand; // A cache entry } _PyUOpInstruction; -static inline uint32_t uop_get_target(_PyUOpInstruction *inst) +static inline uint32_t uop_get_target(const _PyUOpInstruction *inst) { assert(inst->format == UOP_FORMAT_TARGET); return inst->target; } -static inline uint16_t uop_get_exit_index(_PyUOpInstruction *inst) +static inline uint16_t uop_get_exit_index(const _PyUOpInstruction *inst) { assert(inst->format == UOP_FORMAT_EXIT); return inst->exit_index; } -static inline uint16_t uop_get_deopt_target(_PyUOpInstruction *inst) +static inline uint16_t uop_get_deopt_target(const _PyUOpInstruction *inst) { assert(inst->format == UOP_FORMAT_DEOPT); return inst->deopt_target; } -static inline uint16_t uop_get_error_target(_PyUOpInstruction *inst) +static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) { assert(inst->format != UOP_FORMAT_TARGET); return inst->error_target; diff --git a/Python/ceval.c b/Python/ceval.c index 79aa04f9ea8cbd..c425cae56d463a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1010,6 +1010,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); +tier2_dispatch: for (;;) { uopcode = next_uop->opcode; #ifdef Py_DEBUG @@ -1082,6 +1083,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int // Jump here from DEOPT_IF() deoptimize: + if (next_uop[-1].format == UOP_FORMAT_DEOPT) { + uint16_t deopt_target = uop_get_deopt_target(&next_uop[-1]); + next_uop = current_executor->trace + deopt_target; + goto tier2_dispatch; + } + assert(next_uop[-1].format == UOP_FORMAT_TARGET); next_instr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); #ifdef Py_DEBUG if (lltrace >= 2) { diff --git a/Python/optimizer.c b/Python/optimizer.c index 5bcd501238d7da..ce9f09bb6f08d5 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -494,7 +494,7 @@ BRANCH_TO_GUARD[4][2] = { code = (PyCodeObject *)trace_stack[trace_stack_depth].func->func_code; \ instr = trace_stack[trace_stack_depth].instr; -/* Returns 1 on success, +/* Returns the length of the trace on success, * 0 if it failed to produce a worthwhile trace, * and -1 on an error. */ @@ -514,7 +514,7 @@ translate_bytecode_to_trace( _Py_BloomFilter_Add(dependencies, initial_code); _Py_CODEUNIT *initial_instr = instr; int trace_length = 0; - int max_length = buffer_size; + int max_length = buffer_size-1; struct { PyFunctionObject *func; _Py_CODEUNIT *instr; @@ -587,6 +587,10 @@ translate_bytecode_to_trace( } } + if (OPCODE_HAS_DEOPT(opcode) || OPCODE_HAS_ERROR(opcode)) { + // Make space for exit code + max_length--; + } switch (opcode) { case POP_JUMP_IF_NONE: case POP_JUMP_IF_NOT_NONE: @@ -622,10 +626,10 @@ translate_bytecode_to_trace( DPRINTF(2, "Jump likely (%x = %d bits), continue at byte offset %d\n", instr[1].cache, bitcount, 2 * INSTR_IP(target_instr, code)); instr = target_instr; - ADD_TO_TRACE(uopcode, max_length, 0, INSTR_IP(next_instr, code)); + ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(next_instr, code)); goto top; } - ADD_TO_TRACE(uopcode, max_length, 0, INSTR_IP(target_instr, code)); + ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(target_instr, code)); break; } @@ -821,8 +825,8 @@ translate_bytecode_to_trace( code->co_firstlineno, 2 * INSTR_IP(initial_instr, code), trace_length); - OPT_HIST(trace_length + buffer_size - max_length, trace_length_hist); - return 1; + OPT_HIST(trace_length, trace_length_hist); + return trace_length; } #undef RESERVE @@ -1005,24 +1009,24 @@ uop_optimize( _PyBloomFilter dependencies; _Py_BloomFilter_Init(&dependencies); _PyUOpInstruction buffer[UOP_MAX_TRACE_LENGTH]; - int err = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies); - if (err <= 0) { + int length = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies); + if (length <= 0) { // Error or nothing translated - return err; + return length; } OPT_STAT_INC(traces_created); char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); if (env_var == NULL || *env_var == '\0' || *env_var > '0') { - err = _Py_uop_analyze_and_optimize(frame, buffer, + length = _Py_uop_analyze_and_optimize(frame, buffer, UOP_MAX_TRACE_LENGTH, curr_stackentries, &dependencies); - if (err <= 0) { - return err; + if (length <= 0) { + return length; } } - assert(err == 1); + assert(length >= 1); /* Fix up */ - for (int pc = 0; pc < UOP_MAX_TRACE_LENGTH; pc++) { + for (int pc = 0; pc < length; pc++) { int opcode = buffer[pc].opcode; int oparg = buffer[pc].oparg; if (_PyUop_Flags[opcode] & HAS_OPARG_AND_1_FLAG) { diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 9fd4b1967ecc3b..cbd782b445cbce 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -394,7 +394,7 @@ optimize_uops( } _Py_uop_abstractcontext_fini(ctx); - return 1; + return trace_len; out_of_space: DPRINTF(1, "Out of space in abstract interpreter\n"); @@ -543,19 +543,19 @@ peephole_opt(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, int buffer_s // 0 - failure, no error raised, just fall back to Tier 1 // -1 - failure, and raise error -// 1 - optimizer success +// > 0 - length of optimized trace int _Py_uop_analyze_and_optimize( _PyInterpreterFrame *frame, _PyUOpInstruction *buffer, - int buffer_size, + int length, int curr_stacklen, _PyBloomFilter *dependencies ) { OPT_STAT_INC(optimizer_attempts); - int err = remove_globals(frame, buffer, buffer_size, dependencies); + int err = remove_globals(frame, buffer, length, dependencies); if (err == 0) { goto not_ready; } @@ -563,21 +563,21 @@ _Py_uop_analyze_and_optimize( goto error; } - peephole_opt(frame, buffer, buffer_size); + peephole_opt(frame, buffer, length); - err = optimize_uops( + length = optimize_uops( (PyCodeObject *)frame->f_executable, buffer, - buffer_size, curr_stacklen, dependencies); + length, curr_stacklen, dependencies); if (err == 0) { goto not_ready; } assert(err == 1); - remove_unneeded_uops(buffer, buffer_size); + remove_unneeded_uops(buffer, length); OPT_STAT_INC(optimizer_successes); - return 1; + return length; not_ready: return 0; error: From 20a7afe6dd9e1605086ab1146fe48b165dc2a12d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 12 Mar 2024 15:21:19 +0000 Subject: [PATCH 03/24] Hot cold splitting. Work in progress --- Include/internal/pycore_uop_ids.h | 193 +++++++++++++------------ Include/internal/pycore_uop_metadata.h | 2 + Python/bytecodes.c | 4 + Python/executor_cases.c.h | 5 + Python/optimizer.c | 143 +++++++++++------- Python/optimizer_cases.c.h | 4 + 6 files changed, 205 insertions(+), 146 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 1209d736abe696..f31198be07d50a 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -86,45 +86,46 @@ extern "C" { #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR +#define _DEOPT 330 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE #define _END_SEND END_SEND #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _FATAL_ERROR 330 +#define _FATAL_ERROR 331 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 331 +#define _FOR_ITER 332 #define _FOR_ITER_GEN FOR_ITER_GEN -#define _FOR_ITER_TIER_TWO 332 +#define _FOR_ITER_TIER_TWO 333 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 333 -#define _GUARD_BOTH_INT 334 -#define _GUARD_BOTH_UNICODE 335 -#define _GUARD_BUILTINS_VERSION 336 -#define _GUARD_DORV_VALUES 337 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 338 -#define _GUARD_GLOBALS_VERSION 339 -#define _GUARD_IS_FALSE_POP 340 -#define _GUARD_IS_NONE_POP 341 -#define _GUARD_IS_NOT_NONE_POP 342 -#define _GUARD_IS_TRUE_POP 343 -#define _GUARD_KEYS_VERSION 344 -#define _GUARD_NOT_EXHAUSTED_LIST 345 -#define _GUARD_NOT_EXHAUSTED_RANGE 346 -#define _GUARD_NOT_EXHAUSTED_TUPLE 347 -#define _GUARD_TYPE_VERSION 348 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 349 -#define _INIT_CALL_PY_EXACT_ARGS 350 -#define _INIT_CALL_PY_EXACT_ARGS_0 351 -#define _INIT_CALL_PY_EXACT_ARGS_1 352 -#define _INIT_CALL_PY_EXACT_ARGS_2 353 -#define _INIT_CALL_PY_EXACT_ARGS_3 354 -#define _INIT_CALL_PY_EXACT_ARGS_4 355 +#define _GUARD_BOTH_FLOAT 334 +#define _GUARD_BOTH_INT 335 +#define _GUARD_BOTH_UNICODE 336 +#define _GUARD_BUILTINS_VERSION 337 +#define _GUARD_DORV_VALUES 338 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 339 +#define _GUARD_GLOBALS_VERSION 340 +#define _GUARD_IS_FALSE_POP 341 +#define _GUARD_IS_NONE_POP 342 +#define _GUARD_IS_NOT_NONE_POP 343 +#define _GUARD_IS_TRUE_POP 344 +#define _GUARD_KEYS_VERSION 345 +#define _GUARD_NOT_EXHAUSTED_LIST 346 +#define _GUARD_NOT_EXHAUSTED_RANGE 347 +#define _GUARD_NOT_EXHAUSTED_TUPLE 348 +#define _GUARD_TYPE_VERSION 349 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 350 +#define _INIT_CALL_PY_EXACT_ARGS 351 +#define _INIT_CALL_PY_EXACT_ARGS_0 352 +#define _INIT_CALL_PY_EXACT_ARGS_1 353 +#define _INIT_CALL_PY_EXACT_ARGS_2 354 +#define _INIT_CALL_PY_EXACT_ARGS_3 355 +#define _INIT_CALL_PY_EXACT_ARGS_4 356 #define _INSTRUMENTED_CALL INSTRUMENTED_CALL #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW @@ -141,65 +142,65 @@ extern "C" { #define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST #define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE #define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 356 -#define _IS_NONE 357 +#define _INTERNAL_INCREMENT_OPT_COUNTER 357 +#define _IS_NONE 358 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 358 -#define _ITER_CHECK_RANGE 359 -#define _ITER_CHECK_TUPLE 360 -#define _ITER_JUMP_LIST 361 -#define _ITER_JUMP_RANGE 362 -#define _ITER_JUMP_TUPLE 363 -#define _ITER_NEXT_LIST 364 -#define _ITER_NEXT_RANGE 365 -#define _ITER_NEXT_TUPLE 366 -#define _JUMP_TO_TOP 367 +#define _ITER_CHECK_LIST 359 +#define _ITER_CHECK_RANGE 360 +#define _ITER_CHECK_TUPLE 361 +#define _ITER_JUMP_LIST 362 +#define _ITER_JUMP_RANGE 363 +#define _ITER_JUMP_TUPLE 364 +#define _ITER_NEXT_LIST 365 +#define _ITER_NEXT_RANGE 366 +#define _ITER_NEXT_TUPLE 367 +#define _JUMP_TO_TOP 368 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND #define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR -#define _LOAD_ATTR 368 -#define _LOAD_ATTR_CLASS 369 -#define _LOAD_ATTR_CLASS_0 370 -#define _LOAD_ATTR_CLASS_1 371 +#define _LOAD_ATTR 369 +#define _LOAD_ATTR_CLASS 370 +#define _LOAD_ATTR_CLASS_0 371 +#define _LOAD_ATTR_CLASS_1 372 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 372 -#define _LOAD_ATTR_INSTANCE_VALUE_0 373 -#define _LOAD_ATTR_INSTANCE_VALUE_1 374 -#define _LOAD_ATTR_METHOD_LAZY_DICT 375 -#define _LOAD_ATTR_METHOD_NO_DICT 376 -#define _LOAD_ATTR_METHOD_WITH_VALUES 377 -#define _LOAD_ATTR_MODULE 378 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 379 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 380 +#define _LOAD_ATTR_INSTANCE_VALUE 373 +#define _LOAD_ATTR_INSTANCE_VALUE_0 374 +#define _LOAD_ATTR_INSTANCE_VALUE_1 375 +#define _LOAD_ATTR_METHOD_LAZY_DICT 376 +#define _LOAD_ATTR_METHOD_NO_DICT 377 +#define _LOAD_ATTR_METHOD_WITH_VALUES 378 +#define _LOAD_ATTR_MODULE 379 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 380 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 381 #define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_SLOT 381 -#define _LOAD_ATTR_SLOT_0 382 -#define _LOAD_ATTR_SLOT_1 383 -#define _LOAD_ATTR_WITH_HINT 384 +#define _LOAD_ATTR_SLOT 382 +#define _LOAD_ATTR_SLOT_0 383 +#define _LOAD_ATTR_SLOT_1 384 +#define _LOAD_ATTR_WITH_HINT 385 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 385 -#define _LOAD_CONST_INLINE_BORROW 386 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 387 -#define _LOAD_CONST_INLINE_WITH_NULL 388 +#define _LOAD_CONST_INLINE 386 +#define _LOAD_CONST_INLINE_BORROW 387 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 388 +#define _LOAD_CONST_INLINE_WITH_NULL 389 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 389 -#define _LOAD_FAST_0 390 -#define _LOAD_FAST_1 391 -#define _LOAD_FAST_2 392 -#define _LOAD_FAST_3 393 -#define _LOAD_FAST_4 394 -#define _LOAD_FAST_5 395 -#define _LOAD_FAST_6 396 -#define _LOAD_FAST_7 397 +#define _LOAD_FAST 390 +#define _LOAD_FAST_0 391 +#define _LOAD_FAST_1 392 +#define _LOAD_FAST_2 393 +#define _LOAD_FAST_3 394 +#define _LOAD_FAST_4 395 +#define _LOAD_FAST_5 396 +#define _LOAD_FAST_6 397 +#define _LOAD_FAST_7 398 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 398 -#define _LOAD_GLOBAL_BUILTINS 399 -#define _LOAD_GLOBAL_MODULE 400 +#define _LOAD_GLOBAL 399 +#define _LOAD_GLOBAL_BUILTINS 400 +#define _LOAD_GLOBAL_MODULE 401 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR @@ -213,48 +214,48 @@ extern "C" { #define _MATCH_SEQUENCE MATCH_SEQUENCE #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_FRAME 401 -#define _POP_JUMP_IF_FALSE 402 -#define _POP_JUMP_IF_TRUE 403 +#define _POP_FRAME 402 +#define _POP_JUMP_IF_FALSE 403 +#define _POP_JUMP_IF_TRUE 404 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 404 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 405 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 405 +#define _PUSH_FRAME 406 #define _PUSH_NULL PUSH_NULL -#define _REPLACE_WITH_TRUE 406 +#define _REPLACE_WITH_TRUE 407 #define _RESUME_CHECK RESUME_CHECK -#define _SAVE_RETURN_OFFSET 407 -#define _SEND 408 +#define _SAVE_RETURN_OFFSET 408 +#define _SEND 409 #define _SEND_GEN SEND_GEN #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 409 -#define _STORE_ATTR 410 -#define _STORE_ATTR_INSTANCE_VALUE 411 -#define _STORE_ATTR_SLOT 412 +#define _START_EXECUTOR 410 +#define _STORE_ATTR 411 +#define _STORE_ATTR_INSTANCE_VALUE 412 +#define _STORE_ATTR_SLOT 413 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 413 -#define _STORE_FAST_0 414 -#define _STORE_FAST_1 415 -#define _STORE_FAST_2 416 -#define _STORE_FAST_3 417 -#define _STORE_FAST_4 418 -#define _STORE_FAST_5 419 -#define _STORE_FAST_6 420 -#define _STORE_FAST_7 421 +#define _STORE_FAST 414 +#define _STORE_FAST_0 415 +#define _STORE_FAST_1 416 +#define _STORE_FAST_2 417 +#define _STORE_FAST_3 418 +#define _STORE_FAST_4 419 +#define _STORE_FAST_5 420 +#define _STORE_FAST_6 421 +#define _STORE_FAST_7 422 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 422 +#define _STORE_SUBSCR 423 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TO_BOOL 423 +#define _TO_BOOL 424 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -264,12 +265,12 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 424 +#define _UNPACK_SEQUENCE 425 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define MAX_UOP_ID 424 +#define MAX_UOP_ID 425 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 62405a362fd7ab..0b5a96b378d542 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -243,6 +243,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_START_EXECUTOR] = 0, [_FATAL_ERROR] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, + [_DEOPT] = HAS_DEOPT_FLAG, }; const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { @@ -321,6 +322,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_DELETE_GLOBAL] = "_DELETE_GLOBAL", [_DELETE_NAME] = "_DELETE_NAME", [_DELETE_SUBSCR] = "_DELETE_SUBSCR", + [_DEOPT] = "_DEOPT", [_DICT_MERGE] = "_DICT_MERGE", [_DICT_UPDATE] = "_DICT_UPDATE", [_END_SEND] = "_END_SEND", diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 03e5f4e330bdd8..61e1995d84db57 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4157,6 +4157,10 @@ dummy_func( frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; } + tier2 op(_DEOPT, (--)) { + DEOPT_IF(1); + } + // END BYTECODES // } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 4108fe38d2d144..0f1eb3cce4054a 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3898,4 +3898,9 @@ break; } + case _DEOPT: { + if (1) DEOPTIMIZE; + break; + } + #undef TIER_TWO diff --git a/Python/optimizer.c b/Python/optimizer.c index ce9f09bb6f08d5..620559ee7e89f1 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -154,7 +154,7 @@ PyUnstable_GetOptimizer(void) } static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies); +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies); static int init_cold_exit_executor(_PyExecutorObject *executor, int oparg); @@ -839,43 +839,62 @@ translate_bytecode_to_trace( #define SET_BIT(array, bit) (array[(bit)>>5] |= (1<<((bit)&31))) #define BIT_IS_SET(array, bit) (array[(bit)>>5] & (1<<((bit)&31))) -/* Count the number of used uops, and mark them in the bit vector `used`. - * This can be done in a single pass using simple reachability analysis, - * as there are no backward jumps. - * NOPs are excluded from the count. +/* Count the number of unused uops and exits */ static int -compute_used(_PyUOpInstruction *buffer, uint32_t *used, int *exit_count_ptr) +count_exits_and_nops(_PyUOpInstruction *buffer, int length, int *exit_count_ptr) { - int count = 0; int exit_count = 0; - SET_BIT(used, 0); - for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { - if (!BIT_IS_SET(used, i)) { - continue; - } - count++; + int nop_count = 0; + for (int i = 0; i < length; i++) { int opcode = buffer[i].opcode; - if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { - exit_count++; + if (opcode == _NOP) { + nop_count++; } - if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE) { - continue; + if (opcode == _SIDE_EXIT) { + exit_count++; } - /* All other micro-ops fall through, so i+1 is reachable */ - SET_BIT(used, i+1); - assert(opcode <= MAX_UOP_ID); - if (_PyUop_Flags[opcode] & HAS_JUMP_FLAG) { - /* Mark target as reachable */ - SET_BIT(used, buffer[i].oparg); + } + *exit_count_ptr = exit_count; + return nop_count; +} + +/* Convert implicit exits, errors and deopts + * into explicit ones. */ +static int +prepare_for_execution(_PyUOpInstruction *buffer, int length) +{ + int next_exit = length; + for (int i = 0; i < length; i++) { + _PyUOpInstruction *inst = &buffer[i]; + int opcode = inst->opcode; + int current_exit = -1; + int current_exit_target = -1; + int current_error_target = -1; + int current_error = -1; + uint32_t target = uop_get_target(inst); + if (_PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) { + if (target != current_exit_target) { + current_exit_target = target; + buffer[next_exit].opcode = _PyUop_Flags[opcode] & HAS_EXIT_FLAG ? _SIDE_EXIT : _DEOPT; + buffer[next_exit].target = target; + current_exit = next_exit; + next_exit++; + } + buffer[i].deopt_target = current_exit; } - if (opcode == NOP) { - count--; - UNSET_BIT(used, i); + if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) { + if (target != current_error_target) { + current_error_target = target; + buffer[next_exit].opcode = _PyUop_Flags[opcode] & HAS_EXIT_FLAG ? _ERROR; + buffer[next_exit].target = target; + current_error = next_exit; + next_exit++; + } + buffer[i].error_target = current_error; } } - *exit_count_ptr = exit_count; - return count; + return next_exit; } /* Executor side exits */ @@ -900,13 +919,12 @@ allocate_executor(int exit_count, int length) * and not a NOP. */ static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies) +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies) { - uint32_t used[(UOP_MAX_TRACE_LENGTH + 31)/32] = { 0 }; int exit_count; - int length = compute_used(buffer, used, &exit_count); - length += 1; // For _START_EXECUTOR - _PyExecutorObject *executor = allocate_executor(exit_count, length); + int nop_count = count_exits_and_nops(buffer, length, &exit_count); + int executor_length = length-nop_count+1; // 1 for _START_EXECUTOR + _PyExecutorObject *executor = allocate_executor(exit_count, executor_length); if (executor == NULL) { return NULL; } @@ -916,29 +934,23 @@ make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *depende executor->exits[i].temperature = 0; } int next_exit = exit_count-1; - _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length-1]; + _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[executor_length-1]; /* Scan backwards, so that we see the destinations of jumps before the jumps themselves. */ - for (int i = UOP_MAX_TRACE_LENGTH-1; i >= 0; i--) { - if (!BIT_IS_SET(used, i)) { + for (int i = length-1; i >= 0; i--) { + int opcode = buffer[i].opcode; + if (opcode == NOP) { continue; } *dest = buffer[i]; - int opcode = buffer[i].opcode; - if (opcode == _POP_JUMP_IF_FALSE || - opcode == _POP_JUMP_IF_TRUE) - { - /* The oparg of the target will already have been set to its new offset */ - int oparg = dest->oparg; - dest->oparg = buffer[oparg].oparg; - } + assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE); if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { executor->exits[next_exit].target = buffer[i].target; dest->exit_index = next_exit; next_exit--; } - /* Set the oparg to be the destination offset, - * so that we can set the oparg of earlier jumps correctly. */ - buffer[i].oparg = (uint16_t)(dest - executor->trace); + if (buffer[i].format == UOP_FORMAT_DEOPT) { + dest->deopt_target -= nop_count; + } dest--; } assert(next_exit == -1); @@ -998,6 +1010,33 @@ init_cold_exit_executor(_PyExecutorObject *executor, int oparg) return 0; } +static int +insert_exits_and_deopts( + _PyUOpInstruction *buffer, + int length) +{ + assert(length < UOP_MAX_TRACE_LENGTH); + int last_target = -1; + int deopt_target = length-1; + int next_exit = 0; + for (int pc = 0; pc < length; pc++) { + int opcode = buffer[pc].opcode; + if (_PyUop_Flags[opcode] & HAS_DEOPT_FLAG) { + uint16_t target = uop_get_target(&buffer[pc]); + if (last_target != target) { + last_target = target; + deopt_target++; + buffer[deopt_target].opcode = _DEOPT; + buffer[deopt_target].target = target; + buffer[deopt_target].format = UOP_FORMAT_TARGET; + } + buffer[pc].format = UOP_FORMAT_DEOPT; + buffer[pc].deopt_target = deopt_target; + } + } + return deopt_target + 1; +} + static int uop_optimize( _PyOptimizerObject *self, @@ -1014,16 +1053,18 @@ uop_optimize( // Error or nothing translated return length; } + assert(length < UOP_MAX_TRACE_LENGTH); OPT_STAT_INC(traces_created); char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); if (env_var == NULL || *env_var == '\0' || *env_var > '0') { length = _Py_uop_analyze_and_optimize(frame, buffer, - UOP_MAX_TRACE_LENGTH, + length, curr_stackentries, &dependencies); if (length <= 0) { return length; } } + assert(length < UOP_MAX_TRACE_LENGTH); assert(length >= 1); /* Fix up */ for (int pc = 0; pc < length; pc++) { @@ -1041,7 +1082,9 @@ uop_optimize( assert(_PyOpcode_uop_name[buffer[pc].opcode]); assert(strncmp(_PyOpcode_uop_name[buffer[pc].opcode], _PyOpcode_uop_name[opcode], strlen(_PyOpcode_uop_name[opcode])) == 0); } - _PyExecutorObject *executor = make_executor_from_uops(buffer, &dependencies); + length = insert_exits_and_deopts(buffer, length); + assert(length <= UOP_MAX_TRACE_LENGTH); + _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies); if (executor == NULL) { return -1; } @@ -1125,7 +1168,7 @@ counter_optimize( { .opcode = _INTERNAL_INCREMENT_OPT_COUNTER }, { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)), .format=UOP_FORMAT_TARGET } }; - _PyExecutorObject *executor = make_executor_from_uops(buffer, &EMPTY_FILTER); + _PyExecutorObject *executor = make_executor_from_uops(buffer, 3, &EMPTY_FILTER); if (executor == NULL) { return -1; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index fed5730d2e50c1..4fe7756e406ce6 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2008,3 +2008,7 @@ break; } + case _DEOPT: { + break; + } + From 801062d6477f9ada4e23013cbf41921edc06fd74 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 12 Mar 2024 18:21:56 +0000 Subject: [PATCH 04/24] Further progress on hot-cold- splitting --- Include/internal/pycore_uop_ids.h | 198 +++---- Include/internal/pycore_uop_metadata.h | 488 ++++++++++++++++++ Python/bytecodes.c | 24 + Python/ceval.c | 26 +- Python/executor_cases.c.h | 244 +++++---- Python/optimizer.c | 71 ++- Python/optimizer_cases.c.h | 28 + Tools/cases_generator/tier2_generator.py | 15 +- .../cases_generator/uop_metadata_generator.py | 17 +- 9 files changed, 856 insertions(+), 255 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index f31198be07d50a..d316344533fcf6 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -90,42 +90,47 @@ extern "C" { #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE #define _END_SEND END_SEND +#define _ERROR_0 331 +#define _ERROR_1 332 +#define _ERROR_2 333 +#define _ERROR_3 334 +#define _ERROR_4 335 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _FATAL_ERROR 331 +#define _FATAL_ERROR 336 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 332 +#define _FOR_ITER 337 #define _FOR_ITER_GEN FOR_ITER_GEN -#define _FOR_ITER_TIER_TWO 333 +#define _FOR_ITER_TIER_TWO 338 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 334 -#define _GUARD_BOTH_INT 335 -#define _GUARD_BOTH_UNICODE 336 -#define _GUARD_BUILTINS_VERSION 337 -#define _GUARD_DORV_VALUES 338 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 339 -#define _GUARD_GLOBALS_VERSION 340 -#define _GUARD_IS_FALSE_POP 341 -#define _GUARD_IS_NONE_POP 342 -#define _GUARD_IS_NOT_NONE_POP 343 -#define _GUARD_IS_TRUE_POP 344 -#define _GUARD_KEYS_VERSION 345 -#define _GUARD_NOT_EXHAUSTED_LIST 346 -#define _GUARD_NOT_EXHAUSTED_RANGE 347 -#define _GUARD_NOT_EXHAUSTED_TUPLE 348 -#define _GUARD_TYPE_VERSION 349 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 350 -#define _INIT_CALL_PY_EXACT_ARGS 351 -#define _INIT_CALL_PY_EXACT_ARGS_0 352 -#define _INIT_CALL_PY_EXACT_ARGS_1 353 -#define _INIT_CALL_PY_EXACT_ARGS_2 354 -#define _INIT_CALL_PY_EXACT_ARGS_3 355 -#define _INIT_CALL_PY_EXACT_ARGS_4 356 +#define _GUARD_BOTH_FLOAT 339 +#define _GUARD_BOTH_INT 340 +#define _GUARD_BOTH_UNICODE 341 +#define _GUARD_BUILTINS_VERSION 342 +#define _GUARD_DORV_VALUES 343 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 344 +#define _GUARD_GLOBALS_VERSION 345 +#define _GUARD_IS_FALSE_POP 346 +#define _GUARD_IS_NONE_POP 347 +#define _GUARD_IS_NOT_NONE_POP 348 +#define _GUARD_IS_TRUE_POP 349 +#define _GUARD_KEYS_VERSION 350 +#define _GUARD_NOT_EXHAUSTED_LIST 351 +#define _GUARD_NOT_EXHAUSTED_RANGE 352 +#define _GUARD_NOT_EXHAUSTED_TUPLE 353 +#define _GUARD_TYPE_VERSION 354 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 355 +#define _INIT_CALL_PY_EXACT_ARGS 356 +#define _INIT_CALL_PY_EXACT_ARGS_0 357 +#define _INIT_CALL_PY_EXACT_ARGS_1 358 +#define _INIT_CALL_PY_EXACT_ARGS_2 359 +#define _INIT_CALL_PY_EXACT_ARGS_3 360 +#define _INIT_CALL_PY_EXACT_ARGS_4 361 #define _INSTRUMENTED_CALL INSTRUMENTED_CALL #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW @@ -142,65 +147,65 @@ extern "C" { #define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST #define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE #define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 357 -#define _IS_NONE 358 +#define _INTERNAL_INCREMENT_OPT_COUNTER 362 +#define _IS_NONE 363 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 359 -#define _ITER_CHECK_RANGE 360 -#define _ITER_CHECK_TUPLE 361 -#define _ITER_JUMP_LIST 362 -#define _ITER_JUMP_RANGE 363 -#define _ITER_JUMP_TUPLE 364 -#define _ITER_NEXT_LIST 365 -#define _ITER_NEXT_RANGE 366 -#define _ITER_NEXT_TUPLE 367 -#define _JUMP_TO_TOP 368 +#define _ITER_CHECK_LIST 364 +#define _ITER_CHECK_RANGE 365 +#define _ITER_CHECK_TUPLE 366 +#define _ITER_JUMP_LIST 367 +#define _ITER_JUMP_RANGE 368 +#define _ITER_JUMP_TUPLE 369 +#define _ITER_NEXT_LIST 370 +#define _ITER_NEXT_RANGE 371 +#define _ITER_NEXT_TUPLE 372 +#define _JUMP_TO_TOP 373 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND #define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR -#define _LOAD_ATTR 369 -#define _LOAD_ATTR_CLASS 370 -#define _LOAD_ATTR_CLASS_0 371 -#define _LOAD_ATTR_CLASS_1 372 +#define _LOAD_ATTR 374 +#define _LOAD_ATTR_CLASS 375 +#define _LOAD_ATTR_CLASS_0 376 +#define _LOAD_ATTR_CLASS_1 377 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 373 -#define _LOAD_ATTR_INSTANCE_VALUE_0 374 -#define _LOAD_ATTR_INSTANCE_VALUE_1 375 -#define _LOAD_ATTR_METHOD_LAZY_DICT 376 -#define _LOAD_ATTR_METHOD_NO_DICT 377 -#define _LOAD_ATTR_METHOD_WITH_VALUES 378 -#define _LOAD_ATTR_MODULE 379 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 380 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 381 +#define _LOAD_ATTR_INSTANCE_VALUE 378 +#define _LOAD_ATTR_INSTANCE_VALUE_0 379 +#define _LOAD_ATTR_INSTANCE_VALUE_1 380 +#define _LOAD_ATTR_METHOD_LAZY_DICT 381 +#define _LOAD_ATTR_METHOD_NO_DICT 382 +#define _LOAD_ATTR_METHOD_WITH_VALUES 383 +#define _LOAD_ATTR_MODULE 384 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 385 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 386 #define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_SLOT 382 -#define _LOAD_ATTR_SLOT_0 383 -#define _LOAD_ATTR_SLOT_1 384 -#define _LOAD_ATTR_WITH_HINT 385 +#define _LOAD_ATTR_SLOT 387 +#define _LOAD_ATTR_SLOT_0 388 +#define _LOAD_ATTR_SLOT_1 389 +#define _LOAD_ATTR_WITH_HINT 390 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 386 -#define _LOAD_CONST_INLINE_BORROW 387 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 388 -#define _LOAD_CONST_INLINE_WITH_NULL 389 +#define _LOAD_CONST_INLINE 391 +#define _LOAD_CONST_INLINE_BORROW 392 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 393 +#define _LOAD_CONST_INLINE_WITH_NULL 394 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 390 -#define _LOAD_FAST_0 391 -#define _LOAD_FAST_1 392 -#define _LOAD_FAST_2 393 -#define _LOAD_FAST_3 394 -#define _LOAD_FAST_4 395 -#define _LOAD_FAST_5 396 -#define _LOAD_FAST_6 397 -#define _LOAD_FAST_7 398 +#define _LOAD_FAST 395 +#define _LOAD_FAST_0 396 +#define _LOAD_FAST_1 397 +#define _LOAD_FAST_2 398 +#define _LOAD_FAST_3 399 +#define _LOAD_FAST_4 400 +#define _LOAD_FAST_5 401 +#define _LOAD_FAST_6 402 +#define _LOAD_FAST_7 403 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 399 -#define _LOAD_GLOBAL_BUILTINS 400 -#define _LOAD_GLOBAL_MODULE 401 +#define _LOAD_GLOBAL 404 +#define _LOAD_GLOBAL_BUILTINS 405 +#define _LOAD_GLOBAL_MODULE 406 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR @@ -214,48 +219,49 @@ extern "C" { #define _MATCH_SEQUENCE MATCH_SEQUENCE #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_FRAME 402 -#define _POP_JUMP_IF_FALSE 403 -#define _POP_JUMP_IF_TRUE 404 +#define _POP_FRAME 407 +#define _POP_JUMP_IF_FALSE 408 +#define _POP_JUMP_IF_TRUE 409 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 405 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 410 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 406 +#define _PUSH_FRAME 411 #define _PUSH_NULL PUSH_NULL -#define _REPLACE_WITH_TRUE 407 +#define _REPLACE_WITH_TRUE 412 #define _RESUME_CHECK RESUME_CHECK -#define _SAVE_RETURN_OFFSET 408 -#define _SEND 409 +#define _SAVE_RETURN_OFFSET 413 +#define _SEND 414 #define _SEND_GEN SEND_GEN #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 410 -#define _STORE_ATTR 411 -#define _STORE_ATTR_INSTANCE_VALUE 412 -#define _STORE_ATTR_SLOT 413 +#define _SIDE_EXIT 415 +#define _START_EXECUTOR 416 +#define _STORE_ATTR 417 +#define _STORE_ATTR_INSTANCE_VALUE 418 +#define _STORE_ATTR_SLOT 419 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 414 -#define _STORE_FAST_0 415 -#define _STORE_FAST_1 416 -#define _STORE_FAST_2 417 -#define _STORE_FAST_3 418 -#define _STORE_FAST_4 419 -#define _STORE_FAST_5 420 -#define _STORE_FAST_6 421 -#define _STORE_FAST_7 422 +#define _STORE_FAST 420 +#define _STORE_FAST_0 421 +#define _STORE_FAST_1 422 +#define _STORE_FAST_2 423 +#define _STORE_FAST_3 424 +#define _STORE_FAST_4 425 +#define _STORE_FAST_5 426 +#define _STORE_FAST_6 427 +#define _STORE_FAST_7 428 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 423 +#define _STORE_SUBSCR 429 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TO_BOOL 424 +#define _TO_BOOL 430 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -265,12 +271,12 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 425 +#define _UNPACK_SEQUENCE 431 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define MAX_UOP_ID 425 +#define MAX_UOP_ID 431 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 0b5a96b378d542..b65311ae22bdf6 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -15,6 +15,8 @@ extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1]; extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1]; extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; +extern int _PyUop_Popped(int opcode, int oparg); + #ifdef NEED_OPCODE_METADATA const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_NOP] = HAS_PURE_FLAG, @@ -244,6 +246,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_FATAL_ERROR] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, [_DEOPT] = HAS_DEOPT_FLAG, + [_SIDE_EXIT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_ERROR_0] = HAS_ERROR_FLAG, + [_ERROR_1] = HAS_ERROR_FLAG, + [_ERROR_2] = HAS_ERROR_FLAG, + [_ERROR_3] = HAS_ERROR_FLAG, + [_ERROR_4] = HAS_ERROR_FLAG, }; const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { @@ -326,6 +334,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_DICT_MERGE] = "_DICT_MERGE", [_DICT_UPDATE] = "_DICT_UPDATE", [_END_SEND] = "_END_SEND", + [_ERROR_0] = "_ERROR_0", + [_ERROR_1] = "_ERROR_1", + [_ERROR_2] = "_ERROR_2", + [_ERROR_3] = "_ERROR_3", + [_ERROR_4] = "_ERROR_4", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", [_EXIT_TRACE] = "_EXIT_TRACE", [_FATAL_ERROR] = "_FATAL_ERROR", @@ -442,6 +455,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", [_SET_IP] = "_SET_IP", [_SET_UPDATE] = "_SET_UPDATE", + [_SIDE_EXIT] = "_SIDE_EXIT", [_START_EXECUTOR] = "_START_EXECUTOR", [_STORE_ATTR] = "_STORE_ATTR", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", @@ -481,6 +495,480 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_UNPACK_SEQUENCE_TWO_TUPLE] = "_UNPACK_SEQUENCE_TWO_TUPLE", [_WITH_EXCEPT_START] = "_WITH_EXCEPT_START", }; +int _PyUop_Popped(int opcode, int oparg) +{ + switch(opcode) { + case _NOP: + return 0; + case _RESUME_CHECK: + return 0; + case _LOAD_FAST_CHECK: + return 0; + case _LOAD_FAST_0: + return 0; + case _LOAD_FAST_1: + return 0; + case _LOAD_FAST_2: + return 0; + case _LOAD_FAST_3: + return 0; + case _LOAD_FAST_4: + return 0; + case _LOAD_FAST_5: + return 0; + case _LOAD_FAST_6: + return 0; + case _LOAD_FAST_7: + return 0; + case _LOAD_FAST: + return 0; + case _LOAD_FAST_AND_CLEAR: + return 0; + case _LOAD_FAST_LOAD_FAST: + return 0; + case _LOAD_CONST: + return 0; + case _STORE_FAST_0: + return 1; + case _STORE_FAST_1: + return 1; + case _STORE_FAST_2: + return 1; + case _STORE_FAST_3: + return 1; + case _STORE_FAST_4: + return 1; + case _STORE_FAST_5: + return 1; + case _STORE_FAST_6: + return 1; + case _STORE_FAST_7: + return 1; + case _STORE_FAST: + return 1; + case _STORE_FAST_LOAD_FAST: + return 1; + case _STORE_FAST_STORE_FAST: + return 2; + case _POP_TOP: + return 1; + case _PUSH_NULL: + return 0; + case _END_SEND: + return 2; + case _UNARY_NEGATIVE: + return 1; + case _UNARY_NOT: + return 1; + case _TO_BOOL: + return 1; + case _TO_BOOL_BOOL: + return 1; + case _TO_BOOL_INT: + return 1; + case _TO_BOOL_LIST: + return 1; + case _TO_BOOL_NONE: + return 1; + case _TO_BOOL_STR: + return 1; + case _REPLACE_WITH_TRUE: + return 1; + case _UNARY_INVERT: + return 1; + case _GUARD_BOTH_INT: + return 2; + case _BINARY_OP_MULTIPLY_INT: + return 2; + case _BINARY_OP_ADD_INT: + return 2; + case _BINARY_OP_SUBTRACT_INT: + return 2; + case _GUARD_BOTH_FLOAT: + return 2; + case _BINARY_OP_MULTIPLY_FLOAT: + return 2; + case _BINARY_OP_ADD_FLOAT: + return 2; + case _BINARY_OP_SUBTRACT_FLOAT: + return 2; + case _GUARD_BOTH_UNICODE: + return 2; + case _BINARY_OP_ADD_UNICODE: + return 2; + case _BINARY_SUBSCR: + return 2; + case _BINARY_SLICE: + return 3; + case _STORE_SLICE: + return 4; + case _BINARY_SUBSCR_LIST_INT: + return 2; + case _BINARY_SUBSCR_STR_INT: + return 2; + case _BINARY_SUBSCR_TUPLE_INT: + return 2; + case _BINARY_SUBSCR_DICT: + return 2; + case _LIST_APPEND: + return 2 + (oparg-1); + case _SET_ADD: + return 2 + (oparg-1); + case _STORE_SUBSCR: + return 3; + case _STORE_SUBSCR_LIST_INT: + return 3; + case _STORE_SUBSCR_DICT: + return 3; + case _DELETE_SUBSCR: + return 2; + case _CALL_INTRINSIC_1: + return 1; + case _CALL_INTRINSIC_2: + return 2; + case _POP_FRAME: + return 1; + case _GET_AITER: + return 1; + case _GET_ANEXT: + return 1; + case _GET_AWAITABLE: + return 1; + case _POP_EXCEPT: + return 1; + case _LOAD_ASSERTION_ERROR: + return 0; + case _LOAD_BUILD_CLASS: + return 0; + case _STORE_NAME: + return 1; + case _DELETE_NAME: + return 0; + case _UNPACK_SEQUENCE: + return 1; + case _UNPACK_SEQUENCE_TWO_TUPLE: + return 1; + case _UNPACK_SEQUENCE_TUPLE: + return 1; + case _UNPACK_SEQUENCE_LIST: + return 1; + case _UNPACK_EX: + return 1; + case _STORE_ATTR: + return 2; + case _DELETE_ATTR: + return 1; + case _STORE_GLOBAL: + return 1; + case _DELETE_GLOBAL: + return 0; + case _LOAD_LOCALS: + return 0; + case _LOAD_FROM_DICT_OR_GLOBALS: + return 1; + case _LOAD_NAME: + return 0; + case _LOAD_GLOBAL: + return 0; + case _GUARD_GLOBALS_VERSION: + return 0; + case _GUARD_BUILTINS_VERSION: + return 0; + case _LOAD_GLOBAL_MODULE: + return 0; + case _LOAD_GLOBAL_BUILTINS: + return 0; + case _DELETE_FAST: + return 0; + case _MAKE_CELL: + return 0; + case _DELETE_DEREF: + return 0; + case _LOAD_FROM_DICT_OR_DEREF: + return 1; + case _LOAD_DEREF: + return 0; + case _STORE_DEREF: + return 1; + case _COPY_FREE_VARS: + return 0; + case _BUILD_STRING: + return oparg; + case _BUILD_TUPLE: + return oparg; + case _BUILD_LIST: + return oparg; + case _LIST_EXTEND: + return 2 + (oparg-1); + case _SET_UPDATE: + return 2 + (oparg-1); + case _BUILD_SET: + return oparg; + case _BUILD_MAP: + return oparg*2; + case _SETUP_ANNOTATIONS: + return 0; + case _BUILD_CONST_KEY_MAP: + return 1 + oparg; + case _DICT_UPDATE: + return 2 + (oparg - 1); + case _DICT_MERGE: + return 5 + (oparg - 1); + case _MAP_ADD: + return 3 + (oparg - 1); + case _LOAD_SUPER_ATTR_ATTR: + return 3; + case _LOAD_SUPER_ATTR_METHOD: + return 3; + case _LOAD_ATTR: + return 1; + case _GUARD_TYPE_VERSION: + return 1; + case _CHECK_MANAGED_OBJECT_HAS_VALUES: + return 1; + case _LOAD_ATTR_INSTANCE_VALUE_0: + return 1; + case _LOAD_ATTR_INSTANCE_VALUE_1: + return 1; + case _LOAD_ATTR_INSTANCE_VALUE: + return 1; + case _CHECK_ATTR_MODULE: + return 1; + case _LOAD_ATTR_MODULE: + return 1; + case _CHECK_ATTR_WITH_HINT: + return 1; + case _LOAD_ATTR_WITH_HINT: + return 1; + case _LOAD_ATTR_SLOT_0: + return 1; + case _LOAD_ATTR_SLOT_1: + return 1; + case _LOAD_ATTR_SLOT: + return 1; + case _CHECK_ATTR_CLASS: + return 1; + case _LOAD_ATTR_CLASS_0: + return 1; + case _LOAD_ATTR_CLASS_1: + return 1; + case _LOAD_ATTR_CLASS: + return 1; + case _GUARD_DORV_VALUES: + return 1; + case _STORE_ATTR_INSTANCE_VALUE: + return 2; + case _STORE_ATTR_SLOT: + return 2; + case _COMPARE_OP: + return 2; + case _COMPARE_OP_FLOAT: + return 2; + case _COMPARE_OP_INT: + return 2; + case _COMPARE_OP_STR: + return 2; + case _IS_OP: + return 2; + case _CONTAINS_OP: + return 2; + case _CONTAINS_OP_SET: + return 2; + case _CONTAINS_OP_DICT: + return 2; + case _CHECK_EG_MATCH: + return 2; + case _CHECK_EXC_MATCH: + return 2; + case _IS_NONE: + return 1; + case _GET_LEN: + return 1; + case _MATCH_CLASS: + return 3; + case _MATCH_MAPPING: + return 1; + case _MATCH_SEQUENCE: + return 1; + case _MATCH_KEYS: + return 2; + case _GET_ITER: + return 1; + case _GET_YIELD_FROM_ITER: + return 1; + case _FOR_ITER_TIER_TWO: + return 1; + case _ITER_CHECK_LIST: + return 1; + case _GUARD_NOT_EXHAUSTED_LIST: + return 1; + case _ITER_NEXT_LIST: + return 1; + case _ITER_CHECK_TUPLE: + return 1; + case _GUARD_NOT_EXHAUSTED_TUPLE: + return 1; + case _ITER_NEXT_TUPLE: + return 1; + case _ITER_CHECK_RANGE: + return 1; + case _GUARD_NOT_EXHAUSTED_RANGE: + return 1; + case _ITER_NEXT_RANGE: + return 1; + case _BEFORE_ASYNC_WITH: + return 1; + case _BEFORE_WITH: + return 1; + case _WITH_EXCEPT_START: + return 4; + case _PUSH_EXC_INFO: + return 1; + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: + return 1; + case _GUARD_KEYS_VERSION: + return 1; + case _LOAD_ATTR_METHOD_WITH_VALUES: + return 1; + case _LOAD_ATTR_METHOD_NO_DICT: + return 1; + case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + return 1; + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + return 1; + case _CHECK_ATTR_METHOD_LAZY_DICT: + return 1; + case _LOAD_ATTR_METHOD_LAZY_DICT: + return 1; + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: + return 2 + oparg; + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: + return 2 + oparg; + case _CHECK_PEP_523: + return 0; + case _CHECK_FUNCTION_EXACT_ARGS: + return 2 + oparg; + case _CHECK_STACK_SPACE: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_0: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_1: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_2: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_3: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_4: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS: + return 2 + oparg; + case _PUSH_FRAME: + return 1; + case _CALL_TYPE_1: + return 3; + case _CALL_STR_1: + return 3; + case _CALL_TUPLE_1: + return 3; + case _EXIT_INIT_CHECK: + return 1; + case _CALL_BUILTIN_CLASS: + return 2 + oparg; + case _CALL_BUILTIN_O: + return 2 + oparg; + case _CALL_BUILTIN_FAST: + return 2 + oparg; + case _CALL_BUILTIN_FAST_WITH_KEYWORDS: + return 2 + oparg; + case _CALL_LEN: + return 2 + oparg; + case _CALL_ISINSTANCE: + return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_O: + return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: + return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_NOARGS: + return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_FAST: + return 2 + oparg; + case _MAKE_FUNCTION: + return 1; + case _SET_FUNCTION_ATTRIBUTE: + return 2; + case _BUILD_SLICE: + return 2 + ((oparg == 3) ? 1 : 0); + case _CONVERT_VALUE: + return 1; + case _FORMAT_SIMPLE: + return 1; + case _FORMAT_WITH_SPEC: + return 2; + case _COPY: + return 1 + (oparg-1); + case _BINARY_OP: + return 2; + case _SWAP: + return 2 + (oparg-2); + case _GUARD_IS_TRUE_POP: + return 1; + case _GUARD_IS_FALSE_POP: + return 1; + case _GUARD_IS_NONE_POP: + return 1; + case _GUARD_IS_NOT_NONE_POP: + return 1; + case _JUMP_TO_TOP: + return 0; + case _SET_IP: + return 0; + case _SAVE_RETURN_OFFSET: + return 0; + case _EXIT_TRACE: + return 0; + case _CHECK_VALIDITY: + return 0; + case _LOAD_CONST_INLINE: + return 0; + case _LOAD_CONST_INLINE_BORROW: + return 0; + case _POP_TOP_LOAD_CONST_INLINE_BORROW: + return 1; + case _LOAD_CONST_INLINE_WITH_NULL: + return 0; + case _LOAD_CONST_INLINE_BORROW_WITH_NULL: + return 0; + case _CHECK_FUNCTION: + return 0; + case _INTERNAL_INCREMENT_OPT_COUNTER: + return 1; + case _COLD_EXIT: + return 0; + case _START_EXECUTOR: + return 0; + case _FATAL_ERROR: + return 0; + case _CHECK_VALIDITY_AND_SET_IP: + return 0; + case _DEOPT: + return 0; + case _SIDE_EXIT: + return 0; + case _ERROR_0: + return 0; + case _ERROR_1: + return 1; + case _ERROR_2: + return 2; + case _ERROR_3: + return 3; + case _ERROR_4: + return 4; + default: + return -1; + } +} + #endif // NEED_OPCODE_METADATA diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 61e1995d84db57..6b38b2c1a57c69 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4161,6 +4161,30 @@ dummy_func( DEOPT_IF(1); } + tier2 op(_SIDE_EXIT, (--)) { + EXIT_IF(1); + } + + tier2 op(_ERROR_0, (--)) { + ERROR_IF(1, error); + } + + tier2 op(_ERROR_1, (value --)) { + ERROR_IF(1, error); + } + + tier2 op(_ERROR_2, (value, value1 --)) { + ERROR_IF(1, error); + } + + tier2 op(_ERROR_3, (value, value1, value2 --)) { + ERROR_IF(1, error); + } + + tier2 op(_ERROR_4, (value, value1, value2, val --)) { + ERROR_IF(1, error); + } + // END BYTECODES // } diff --git a/Python/ceval.c b/Python/ceval.c index c425cae56d463a..ab00c9e6c8b146 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -982,6 +982,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #undef DEOPT_IF #define DEOPTIMIZE goto deoptimize +#define JUMP_TO_ERROR goto error_tier_two #ifdef Py_STATS // Disable these macros that apply to Tier 1 stats when we are in Tier 2 @@ -1055,15 +1056,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int ); goto error_tier_two; -// JUMP to any of these from ERROR_IF(..., error) -pop_4_error_tier_two: - STACK_SHRINK(1); -pop_3_error_tier_two: - STACK_SHRINK(1); -pop_2_error_tier_two: - STACK_SHRINK(1); -pop_1_error_tier_two: - STACK_SHRINK(1); error_tier_two: #ifdef Py_DEBUG if (lltrace >= 2) { @@ -1075,6 +1067,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } #endif OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (next_uop[-1].format == UOP_FORMAT_DEOPT) { + uint16_t target = uop_get_error_target(&next_uop[-1]); + next_uop = current_executor->trace + target; + goto tier2_dispatch; + } + assert(next_uop[-1].format == UOP_FORMAT_TARGET); frame->return_offset = 0; // Don't leave this random _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(current_executor); @@ -1084,8 +1082,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int // Jump here from DEOPT_IF() deoptimize: if (next_uop[-1].format == UOP_FORMAT_DEOPT) { - uint16_t deopt_target = uop_get_deopt_target(&next_uop[-1]); - next_uop = current_executor->trace + deopt_target; + uint16_t target = uop_get_deopt_target(&next_uop[-1]); + next_uop = current_executor->trace + target; goto tier2_dispatch; } assert(next_uop[-1].format == UOP_FORMAT_TARGET); @@ -1106,6 +1104,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int // Jump here from EXIT_IF() side_exit: + if (next_uop[-1].format == UOP_FORMAT_DEOPT) { + uint16_t target = uop_get_deopt_target(&next_uop[-1]); + next_uop = current_executor->trace + target; + goto tier2_dispatch; + } + assert(next_uop[-1].format == UOP_FORMAT_EXIT); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); UOP_STAT_INC(uopcode, miss); uint32_t exit_index = next_uop[-1].exit_index; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0f1eb3cce4054a..e88d0573a22c15 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -30,7 +30,7 @@ PyObject *value; oparg = CURRENT_OPARG(); value = GETLOCAL(oparg); - if (value == NULL) goto unbound_local_error_tier_two; + if (value == NULL) JUMP_TO_ERROR; Py_INCREF(value); stack_pointer[0] = value; stack_pointer += 1; @@ -287,7 +287,7 @@ value = stack_pointer[-1]; res = PyNumber_Negative(value); Py_DECREF(value); - if (res == NULL) goto pop_1_error_tier_two; + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-1] = res; break; } @@ -308,7 +308,7 @@ value = stack_pointer[-1]; int err = PyObject_IsTrue(value); Py_DECREF(value); - if (err < 0) goto pop_1_error_tier_two; + if (err < 0) JUMP_TO_ERROR; res = err ? Py_True : Py_False; stack_pointer[-1] = res; break; @@ -399,7 +399,7 @@ value = stack_pointer[-1]; res = PyNumber_Invert(value); Py_DECREF(value); - if (res == NULL) goto pop_1_error_tier_two; + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-1] = res; break; } @@ -424,7 +424,7 @@ res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error_tier_two; + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-2] = res; stack_pointer += -1; break; @@ -440,7 +440,7 @@ res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error_tier_two; + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-2] = res; stack_pointer += -1; break; @@ -456,7 +456,7 @@ res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error_tier_two; + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-2] = res; stack_pointer += -1; break; @@ -540,7 +540,7 @@ res = PyUnicode_Concat(left, right); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res == NULL) goto pop_2_error_tier_two; + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-2] = res; stack_pointer += -1; break; @@ -555,7 +555,7 @@ res = PyObject_GetItem(container, sub); Py_DECREF(container); Py_DECREF(sub); - if (res == NULL) goto pop_2_error_tier_two; + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-2] = res; stack_pointer += -1; break; @@ -580,7 +580,7 @@ Py_DECREF(slice); } Py_DECREF(container); - if (res == NULL) goto pop_3_error_tier_two; + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-3] = res; stack_pointer += -2; break; @@ -606,7 +606,7 @@ } Py_DECREF(v); Py_DECREF(container); - if (err) goto pop_4_error_tier_two; + if (err) JUMP_TO_ERROR; stack_pointer += -4; break; } @@ -694,7 +694,7 @@ } Py_DECREF(dict); Py_DECREF(sub); - if (rc <= 0) goto pop_2_error_tier_two; + if (rc <= 0) JUMP_TO_ERROR; // not found or error stack_pointer[-2] = res; stack_pointer += -1; @@ -709,7 +709,7 @@ oparg = CURRENT_OPARG(); v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error_tier_two; + if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) JUMP_TO_ERROR; stack_pointer += -1; break; } @@ -722,7 +722,7 @@ set = stack_pointer[-2 - (oparg-1)]; int err = PySet_Add(set, v); Py_DECREF(v); - if (err) goto pop_1_error_tier_two; + if (err) JUMP_TO_ERROR; stack_pointer += -1; break; } @@ -739,7 +739,7 @@ Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); - if (err) goto pop_3_error_tier_two; + if (err) JUMP_TO_ERROR; stack_pointer += -3; break; } @@ -780,7 +780,7 @@ STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); - if (err) goto pop_3_error_tier_two; + if (err) JUMP_TO_ERROR; stack_pointer += -3; break; } @@ -794,7 +794,7 @@ int err = PyObject_DelItem(container, sub); Py_DECREF(container); Py_DECREF(sub); - if (err) goto pop_2_error_tier_two; + if (err) JUMP_TO_ERROR; stack_pointer += -2; break; } @@ -807,7 +807,7 @@ assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); Py_DECREF(value); - if (res == NULL) goto pop_1_error_tier_two; + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-1] = res; break; } @@ -823,7 +823,7 @@ res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); Py_DECREF(value2); Py_DECREF(value1); - if (res == NULL) goto pop_2_error_tier_two; + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-2] = res; stack_pointer += -1; break; @@ -874,11 +874,11 @@ "__aiter__ method, got %.100s", type->tp_name); Py_DECREF(obj); - if (true) goto pop_1_error_tier_two; + if (true) JUMP_TO_ERROR; } iter = (*getter)(obj); Py_DECREF(obj); - if (iter == NULL) goto pop_1_error_tier_two; + if (iter == NULL) JUMP_TO_ERROR; if (Py_TYPE(iter)->tp_as_async == NULL || Py_TYPE(iter)->tp_as_async->am_anext == NULL) { _PyErr_Format(tstate, PyExc_TypeError, @@ -886,7 +886,7 @@ "that does not implement __anext__: %.100s", Py_TYPE(iter)->tp_name); Py_DECREF(iter); - if (true) goto pop_1_error_tier_two; + if (true) JUMP_TO_ERROR; } stack_pointer[-1] = iter; break; @@ -962,7 +962,7 @@ /* The code below jumps to `error` if `iter` is NULL. */ } } - if (iter == NULL) goto pop_1_error_tier_two; + if (iter == NULL) JUMP_TO_ERROR; stack_pointer[-1] = iter; break; } @@ -992,11 +992,11 @@ case _LOAD_BUILD_CLASS: { PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error_tier_two; + if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) JUMP_TO_ERROR; if (bc == NULL) { _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); - if (true) goto error_tier_two; + if (true) JUMP_TO_ERROR; } stack_pointer[0] = bc; stack_pointer += 1; @@ -1014,14 +1014,14 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); Py_DECREF(v); - if (true) goto pop_1_error_tier_two; + if (true) JUMP_TO_ERROR; } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); Py_DECREF(v); - if (err) goto pop_1_error_tier_two; + if (err) JUMP_TO_ERROR; stack_pointer += -1; break; } @@ -1054,7 +1054,7 @@ PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); Py_DECREF(seq); - if (res == 0) goto pop_1_error_tier_two; + if (res == 0) JUMP_TO_ERROR; stack_pointer += -1 + oparg; break; } @@ -1122,7 +1122,7 @@ PyObject **top = stack_pointer + totalargs - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq); - if (res == 0) goto pop_1_error_tier_two; + if (res == 0) JUMP_TO_ERROR; stack_pointer += (oparg >> 8) + (oparg & 0xFF); break; } @@ -1137,7 +1137,7 @@ int err = PyObject_SetAttr(owner, name, v); Py_DECREF(v); Py_DECREF(owner); - if (err) goto pop_2_error_tier_two; + if (err) JUMP_TO_ERROR; stack_pointer += -2; break; } @@ -1149,7 +1149,7 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_DelAttr(owner, name); Py_DECREF(owner); - if (err) goto pop_1_error_tier_two; + if (err) JUMP_TO_ERROR; stack_pointer += -1; break; } @@ -1161,7 +1161,7 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); - if (err) goto pop_1_error_tier_two; + if (err) JUMP_TO_ERROR; stack_pointer += -1; break; } @@ -1188,7 +1188,7 @@ if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - if (true) goto error_tier_two; + if (true) JUMP_TO_ERROR; } Py_INCREF(locals); stack_pointer[0] = locals; @@ -1233,7 +1233,7 @@ if (mod_or_class_dict == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - if (true) goto error_tier_two; + if (true) JUMP_TO_ERROR; } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { @@ -1278,22 +1278,22 @@ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); } - if (true) goto error_tier_two; + if (true) JUMP_TO_ERROR; } Py_INCREF(res); } else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error_tier_two; + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) JUMP_TO_ERROR; if (res == NULL) { /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error_tier_two; + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) JUMP_TO_ERROR; if (res == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - if (true) goto error_tier_two; + if (true) JUMP_TO_ERROR; } } } @@ -1361,7 +1361,7 @@ case _DELETE_FAST: { oparg = CURRENT_OPARG(); PyObject *v = GETLOCAL(oparg); - if (v == NULL) goto unbound_local_error_tier_two; + if (v == NULL) JUMP_TO_ERROR; SETLOCAL(oparg, NULL); break; } @@ -1427,7 +1427,7 @@ value = PyCell_GET(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - if (true) goto error_tier_two; + if (true) JUMP_TO_ERROR; } Py_INCREF(value); stack_pointer[0] = value; @@ -1471,7 +1471,7 @@ for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - if (str == NULL) { stack_pointer += -oparg; goto error_tier_two; } + if (str == NULL) JUMP_TO_ERROR; stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; break; @@ -1483,7 +1483,7 @@ oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg]; tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { stack_pointer += -oparg; goto error_tier_two; } + if (tup == NULL) JUMP_TO_ERROR; stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; break; @@ -1495,7 +1495,7 @@ oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg]; list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { stack_pointer += -oparg; goto error_tier_two; } + if (list == NULL) JUMP_TO_ERROR; stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; break; @@ -1518,7 +1518,7 @@ Py_TYPE(iterable)->tp_name); } Py_DECREF(iterable); - if (true) goto pop_1_error_tier_two; + if (true) JUMP_TO_ERROR; } assert(Py_IsNone(none_val)); Py_DECREF(iterable); @@ -1534,7 +1534,7 @@ set = stack_pointer[-2 - (oparg-1)]; int err = _PySet_Update(set, iterable); Py_DECREF(iterable); - if (err < 0) goto pop_1_error_tier_two; + if (err < 0) JUMP_TO_ERROR; stack_pointer += -1; break; } @@ -1556,7 +1556,7 @@ } if (err != 0) { Py_DECREF(set); - if (true) { stack_pointer += -oparg; goto error_tier_two; } + if (true) JUMP_TO_ERROR; } stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; @@ -1575,7 +1575,7 @@ for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - if (map == NULL) { stack_pointer += -oparg*2; goto error_tier_two; } + if (map == NULL) JUMP_TO_ERROR; stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; break; @@ -1587,17 +1587,17 @@ if (LOCALS() == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); - if (true) goto error_tier_two; + if (true) JUMP_TO_ERROR; } /* check if __annotations__ in locals()... */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error_tier_two; + if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) JUMP_TO_ERROR; if (ann_dict == NULL) { ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error_tier_two; + if (ann_dict == NULL) JUMP_TO_ERROR; err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); Py_DECREF(ann_dict); - if (err) goto error_tier_two; + if (err) JUMP_TO_ERROR; } else { Py_DECREF(ann_dict); @@ -1625,7 +1625,7 @@ Py_DECREF(values[_i]); } Py_DECREF(keys); - if (map == NULL) { stack_pointer += -1 - oparg; goto error_tier_two; } + if (map == NULL) JUMP_TO_ERROR; stack_pointer[-1 - oparg] = map; stack_pointer += -oparg; break; @@ -1644,7 +1644,7 @@ Py_TYPE(update)->tp_name); } Py_DECREF(update); - if (true) goto pop_1_error_tier_two; + if (true) JUMP_TO_ERROR; } Py_DECREF(update); stack_pointer += -1; @@ -1662,7 +1662,7 @@ if (_PyDict_MergeEx(dict, update, 2) < 0) { _PyEval_FormatKwargsError(tstate, callable, update); Py_DECREF(update); - if (true) goto pop_1_error_tier_two; + if (true) JUMP_TO_ERROR; } Py_DECREF(update); stack_pointer += -1; @@ -1680,7 +1680,7 @@ assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error_tier_two; + if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) JUMP_TO_ERROR; stack_pointer += -2; break; } @@ -1705,7 +1705,7 @@ Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); - if (attr == NULL) goto pop_3_error_tier_two; + if (attr == NULL) JUMP_TO_ERROR; stack_pointer[-3] = attr; stack_pointer += -2; break; @@ -1734,7 +1734,7 @@ Py_DECREF(class); if (attr == NULL) { Py_DECREF(self); - if (true) goto pop_3_error_tier_two; + if (true) JUMP_TO_ERROR; } if (method_found) { self_or_null = self; // transfer ownership @@ -1774,7 +1774,7 @@ meth | NULL | arg1 | ... | argN */ Py_DECREF(owner); - if (attr == NULL) goto pop_1_error_tier_two; + if (attr == NULL) JUMP_TO_ERROR; self_or_null = NULL; } } @@ -1782,7 +1782,7 @@ /* Classic, pushes one value. */ attr = PyObject_GetAttr(owner, name); Py_DECREF(owner); - if (attr == NULL) goto pop_1_error_tier_two; + if (attr == NULL) JUMP_TO_ERROR; } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = self_or_null; @@ -2078,11 +2078,11 @@ res = PyObject_RichCompare(left, right, oparg >> 5); Py_DECREF(left); Py_DECREF(right); - if (res == NULL) goto pop_2_error_tier_two; + if (res == NULL) JUMP_TO_ERROR; if (oparg & 16) { int res_bool = PyObject_IsTrue(res); Py_DECREF(res); - if (res_bool < 0) goto pop_2_error_tier_two; + if (res_bool < 0) JUMP_TO_ERROR; res = res_bool ? Py_True : Py_False; } stack_pointer[-2] = res; @@ -2184,7 +2184,7 @@ int res = PySequence_Contains(right, left); Py_DECREF(left); Py_DECREF(right); - if (res < 0) goto pop_2_error_tier_two; + if (res < 0) JUMP_TO_ERROR; b = (res ^ oparg) ? Py_True : Py_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -2204,7 +2204,7 @@ int res = _PySet_Contains((PySetObject *)right, left); Py_DECREF(left); Py_DECREF(right); - if (res < 0) goto pop_2_error_tier_two; + if (res < 0) JUMP_TO_ERROR; b = (res ^ oparg) ? Py_True : Py_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -2223,7 +2223,7 @@ int res = PyDict_Contains(right, left); Py_DECREF(left); Py_DECREF(right); - if (res < 0) goto pop_2_error_tier_two; + if (res < 0) JUMP_TO_ERROR; b = (res ^ oparg) ? Py_True : Py_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -2240,7 +2240,7 @@ if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { Py_DECREF(exc_value); Py_DECREF(match_type); - if (true) goto pop_2_error_tier_two; + if (true) JUMP_TO_ERROR; } match = NULL; rest = NULL; @@ -2248,9 +2248,9 @@ &match, &rest); Py_DECREF(exc_value); Py_DECREF(match_type); - if (res < 0) goto pop_2_error_tier_two; + if (res < 0) JUMP_TO_ERROR; assert((match == NULL) == (rest == NULL)); - if (match == NULL) goto pop_2_error_tier_two; + if (match == NULL) JUMP_TO_ERROR; if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } @@ -2268,7 +2268,7 @@ assert(PyExceptionInstance_Check(left)); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { Py_DECREF(right); - if (true) goto pop_1_error_tier_two; + if (true) JUMP_TO_ERROR; } int res = PyErr_GivenExceptionMatches(left, right); Py_DECREF(right); @@ -2302,9 +2302,9 @@ obj = stack_pointer[-1]; // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); - if (len_i < 0) goto error_tier_two; + if (len_i < 0) JUMP_TO_ERROR; len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) goto error_tier_two; + if (len_o == NULL) JUMP_TO_ERROR; stack_pointer[0] = len_o; stack_pointer += 1; break; @@ -2330,7 +2330,7 @@ assert(PyTuple_CheckExact(attrs)); // Success! } else { - if (_PyErr_Occurred(tstate)) goto pop_3_error_tier_two; + if (_PyErr_Occurred(tstate)) JUMP_TO_ERROR; // Error! attrs = Py_None; // Failure! } @@ -2369,7 +2369,7 @@ subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = _PyEval_MatchKeys(tstate, subject, keys); - if (values_or_none == NULL) goto error_tier_two; + if (values_or_none == NULL) JUMP_TO_ERROR; stack_pointer[0] = values_or_none; stack_pointer += 1; break; @@ -2382,7 +2382,7 @@ /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); Py_DECREF(iterable); - if (iter == NULL) goto pop_1_error_tier_two; + if (iter == NULL) JUMP_TO_ERROR; stack_pointer[-1] = iter; break; } @@ -2548,7 +2548,7 @@ r->start = value + r->step; r->len--; next = PyLong_FromLong(value); - if (next == NULL) goto error_tier_two; + if (next == NULL) JUMP_TO_ERROR; stack_pointer[0] = next; stack_pointer += 1; break; @@ -2588,7 +2588,7 @@ Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); - if (true) goto pop_1_error_tier_two; + if (true) JUMP_TO_ERROR; } stack_pointer[-1] = exit; stack_pointer[0] = res; @@ -2631,7 +2631,7 @@ Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); - if (true) goto pop_1_error_tier_two; + if (true) JUMP_TO_ERROR; } stack_pointer[-1] = exit; stack_pointer[0] = res; @@ -2670,7 +2670,7 @@ PyObject *stack[4] = {NULL, exc, val, tb}; res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (res == NULL) goto error_tier_two; + if (res == NULL) JUMP_TO_ERROR; stack_pointer[0] = res; stack_pointer += 1; break; @@ -3093,7 +3093,7 @@ STAT_INC(CALL, hit); res = PyObject_Str(arg); Py_DECREF(arg); - if (res == NULL) goto pop_3_error_tier_two; + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-3] = res; stack_pointer += -2; CHECK_EVAL_BREAKER(); @@ -3115,7 +3115,7 @@ STAT_INC(CALL, hit); res = PySequence_Tuple(arg); Py_DECREF(arg); - if (res == NULL) goto pop_3_error_tier_two; + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-3] = res; stack_pointer += -2; CHECK_EVAL_BREAKER(); @@ -3162,7 +3162,7 @@ Py_DECREF(args[i]); } Py_DECREF(tp); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -3200,7 +3200,7 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -3237,7 +3237,7 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + if (res == NULL) JUMP_TO_ERROR; /* Not deopting because this doesn't mean our optimization was wrong. `res` can be NULL for valid reasons. Eg. getattr(x, 'invalid'). In those cases an exception is set, so we must @@ -3278,7 +3278,7 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -3313,7 +3313,7 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(callable); Py_DECREF(arg); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3349,7 +3349,7 @@ Py_DECREF(inst); Py_DECREF(cls); Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3390,7 +3390,7 @@ Py_DECREF(self); Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -3429,7 +3429,7 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -3470,7 +3470,7 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -3509,7 +3509,7 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); @@ -3589,7 +3589,7 @@ Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - if (slice == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); goto error_tier_two; } + if (slice == NULL) JUMP_TO_ERROR; stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); break; @@ -3605,7 +3605,7 @@ conv_fn = _PyEval_ConversionFuncs[oparg]; result = conv_fn(value); Py_DECREF(value); - if (result == NULL) goto pop_1_error_tier_two; + if (result == NULL) JUMP_TO_ERROR; stack_pointer[-1] = result; break; } @@ -3619,7 +3619,7 @@ if (!PyUnicode_CheckExact(value)) { res = PyObject_Format(value, NULL); Py_DECREF(value); - if (res == NULL) goto pop_1_error_tier_two; + if (res == NULL) JUMP_TO_ERROR; } else { res = value; @@ -3637,7 +3637,7 @@ res = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_DECREF(fmt_spec); - if (res == NULL) goto pop_2_error_tier_two; + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-2] = res; stack_pointer += -1; break; @@ -3666,7 +3666,7 @@ res = _PyEval_BinaryOps[oparg](lhs, rhs); Py_DECREF(lhs); Py_DECREF(rhs); - if (res == NULL) goto pop_2_error_tier_two; + if (res == NULL) JUMP_TO_ERROR; stack_pointer[-2] = res; stack_pointer += -1; break; @@ -3862,7 +3862,7 @@ if (optimized < 0) { Py_DECREF(previous); tstate->previous_executor = Py_None; - if (1) goto error_tier_two; + if (1) JUMP_TO_ERROR; } GOTO_TIER_ONE(target); } @@ -3903,4 +3903,58 @@ break; } + case _SIDE_EXIT: { + if (1) goto side_exit; + break; + } + + case _ERROR_0: { + if (1) JUMP_TO_ERROR; + break; + } + + case _ERROR_1: { + PyObject *value; + value = stack_pointer[-1]; + if (1) JUMP_TO_ERROR; + stack_pointer += -1; + break; + } + + case _ERROR_2: { + PyObject *value1; + PyObject *value; + value1 = stack_pointer[-1]; + value = stack_pointer[-2]; + if (1) JUMP_TO_ERROR; + stack_pointer += -2; + break; + } + + case _ERROR_3: { + PyObject *value2; + PyObject *value1; + PyObject *value; + value2 = stack_pointer[-1]; + value1 = stack_pointer[-2]; + value = stack_pointer[-3]; + if (1) JUMP_TO_ERROR; + stack_pointer += -3; + break; + } + + case _ERROR_4: { + PyObject *val; + PyObject *value2; + PyObject *value1; + PyObject *value; + val = stack_pointer[-1]; + value2 = stack_pointer[-2]; + value1 = stack_pointer[-3]; + value = stack_pointer[-4]; + if (1) JUMP_TO_ERROR; + stack_pointer += -4; + break; + } + #undef TIER_TWO diff --git a/Python/optimizer.c b/Python/optimizer.c index 620559ee7e89f1..3327eeff7f5e0f 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -859,34 +859,55 @@ count_exits_and_nops(_PyUOpInstruction *buffer, int length, int *exit_count_ptr) return nop_count; } +static uint16_t ERRORS[5] = { + [0] = _ERROR_0, + [1] = _ERROR_1, + [2] = _ERROR_2, + [3] = _ERROR_3, + [4] = _ERROR_4, +}; + /* Convert implicit exits, errors and deopts * into explicit ones. */ static int prepare_for_execution(_PyUOpInstruction *buffer, int length) { int next_exit = length; + int exit_index = 0; for (int i = 0; i < length; i++) { _PyUOpInstruction *inst = &buffer[i]; int opcode = inst->opcode; - int current_exit = -1; - int current_exit_target = -1; - int current_error_target = -1; - int current_error = -1; - uint32_t target = uop_get_target(inst); + int32_t current_exit = -1; + int32_t current_exit_target = -1; + int32_t current_error_target = -1; + int32_t current_error = -1; + int32_t target = (int32_t)uop_get_target(inst); if (_PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) { if (target != current_exit_target) { current_exit_target = target; - buffer[next_exit].opcode = _PyUop_Flags[opcode] & HAS_EXIT_FLAG ? _SIDE_EXIT : _DEOPT; - buffer[next_exit].target = target; + if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { + buffer[next_exit].opcode = _SIDE_EXIT; + buffer[next_exit].format = UOP_FORMAT_EXIT; + buffer[next_exit].exit_index = exit_index; + } + else { + buffer[next_exit].opcode = _DEOPT; + buffer[next_exit].format = UOP_FORMAT_TARGET; + buffer[next_exit].target = target; + exit_index++; + } current_exit = next_exit; next_exit++; } buffer[i].deopt_target = current_exit; + buffer[i].format = _PyUop_Flags[opcode] & HAS_EXIT_FLAG ? UOP_FORMAT_EXIT : UOP_FORMAT_DEOPT ; } if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) { if (target != current_error_target) { + int popped = _PyUop_Popped(opcode, inst->oparg); + assert(popped < 5); current_error_target = target; - buffer[next_exit].opcode = _PyUop_Flags[opcode] & HAS_EXIT_FLAG ? _ERROR; + buffer[next_exit].opcode = ERRORS[popped]; buffer[next_exit].target = target; current_error = next_exit; next_exit++; @@ -935,7 +956,6 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil } int next_exit = exit_count-1; _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[executor_length-1]; - /* Scan backwards, so that we see the destinations of jumps before the jumps themselves. */ for (int i = length-1; i >= 0; i--) { int opcode = buffer[i].opcode; if (opcode == NOP) { @@ -943,7 +963,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil } *dest = buffer[i]; assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE); - if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { + if (opcode == _SIDE_EXIT) { executor->exits[next_exit].target = buffer[i].target; dest->exit_index = next_exit; next_exit--; @@ -956,6 +976,8 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil assert(next_exit == -1); assert(dest == executor->trace); dest->opcode = _START_EXECUTOR; + dest->oparg = 0; + dest->target = 0; dest->operand = (uintptr_t)executor; _Py_ExecutorInit(executor, dependencies); #ifdef Py_DEBUG @@ -1010,33 +1032,6 @@ init_cold_exit_executor(_PyExecutorObject *executor, int oparg) return 0; } -static int -insert_exits_and_deopts( - _PyUOpInstruction *buffer, - int length) -{ - assert(length < UOP_MAX_TRACE_LENGTH); - int last_target = -1; - int deopt_target = length-1; - int next_exit = 0; - for (int pc = 0; pc < length; pc++) { - int opcode = buffer[pc].opcode; - if (_PyUop_Flags[opcode] & HAS_DEOPT_FLAG) { - uint16_t target = uop_get_target(&buffer[pc]); - if (last_target != target) { - last_target = target; - deopt_target++; - buffer[deopt_target].opcode = _DEOPT; - buffer[deopt_target].target = target; - buffer[deopt_target].format = UOP_FORMAT_TARGET; - } - buffer[pc].format = UOP_FORMAT_DEOPT; - buffer[pc].deopt_target = deopt_target; - } - } - return deopt_target + 1; -} - static int uop_optimize( _PyOptimizerObject *self, @@ -1082,7 +1077,7 @@ uop_optimize( assert(_PyOpcode_uop_name[buffer[pc].opcode]); assert(strncmp(_PyOpcode_uop_name[buffer[pc].opcode], _PyOpcode_uop_name[opcode], strlen(_PyOpcode_uop_name[opcode])) == 0); } - length = insert_exits_and_deopts(buffer, length); + length = prepare_for_execution(buffer, length); assert(length <= UOP_MAX_TRACE_LENGTH); _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies); if (executor == NULL) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 4fe7756e406ce6..78df1b9d8ea8f8 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2012,3 +2012,31 @@ break; } + case _SIDE_EXIT: { + break; + } + + case _ERROR_0: { + break; + } + + case _ERROR_1: { + stack_pointer += -1; + break; + } + + case _ERROR_2: { + stack_pointer += -2; + break; + } + + case _ERROR_3: { + stack_pointer += -3; + break; + } + + case _ERROR_4: { + stack_pointer += -4; + break; + } + diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index a2d62f4bf026c5..a82f9076f6b9fe 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -72,20 +72,7 @@ def tier2_replace_error( label = next(tkn_iter).text next(tkn_iter) # RPAREN next(tkn_iter) # Semi colon - out.emit(") ") - c_offset = stack.peek_offset.to_c() - try: - offset = -int(c_offset) - close = ";\n" - except ValueError: - offset = None - out.emit(f"{{ stack_pointer += {c_offset}; ") - close = "; }\n" - out.emit("goto ") - if offset: - out.emit(f"pop_{offset}_") - out.emit(label + "_tier_two") - out.emit(close) + out.emit(") JUMP_TO_ERROR;\n") def tier2_replace_deopt( diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py index 72eed3041c55c9..3feccbe8b962fa 100644 --- a/Tools/cases_generator/uop_metadata_generator.py +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -15,10 +15,10 @@ write_header, cflags, ) +from stack import Stack from cwriter import CWriter from typing import TextIO - DEFAULT_OUTPUT = ROOT / "Include/internal/pycore_uop_metadata.h" @@ -26,6 +26,7 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: out.emit("extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1];\n") out.emit("extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1];\n") out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];\n\n") + out.emit("extern int _PyUop_Popped(int opcode, int oparg);\n\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") out.emit("const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") for uop in analysis.uops.values(): @@ -44,6 +45,20 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: if uop.is_viable() and uop.properties.tier != 1: out.emit(f'[{uop.name}] = "{uop.name}",\n') out.emit("};\n") + out.emit("int _PyUop_Popped(int opcode, int oparg)\n{\n") + out.emit("switch(opcode) {\n") + for uop in analysis.uops.values(): + if uop.is_viable() and uop.properties.tier != 1: + stack = Stack() + for var in reversed(uop.stack.inputs): + stack.pop(var) + popped = (-stack.base_offset).to_c() + out.emit(f"case {uop.name}:\n") + out.emit(f" return {popped};\n") + out.emit("default:\n") + out.emit(" return -1;\n") + out.emit("}\n") + out.emit("}\n\n") out.emit("#endif // NEED_OPCODE_METADATA\n\n") From 53b90bca7ec3340ae564ca8584136c137e91940b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 14 Mar 2024 08:58:07 +0000 Subject: [PATCH 05/24] Change error exit code to fix length. All test passing for T2 interpreter --- Include/cpython/optimizer.h | 11 +- Include/internal/pycore_opcode_metadata.h | 87 ++-- Include/internal/pycore_uop_ids.h | 199 ++++---- Include/internal/pycore_uop_metadata.h | 78 +-- Python/bytecodes.c | 24 +- Python/ceval.c | 10 +- Python/executor_cases.c.h | 454 ++---------------- Python/generated_cases.c.h | 8 +- Python/optimizer.c | 275 +++++++---- Python/optimizer_analysis.c | 40 +- Python/optimizer_cases.c.h | 108 +---- Tools/cases_generator/analyzer.py | 58 ++- Tools/cases_generator/generators_common.py | 2 + .../opcode_metadata_generator.py | 1 + Tools/cases_generator/tier2_generator.py | 5 +- 15 files changed, 486 insertions(+), 874 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index e0b4c4842b7623..eff8866a4204e0 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -32,7 +32,8 @@ typedef struct { #define UOP_FORMAT_TARGET 0 #define UOP_FORMAT_EXIT 1 -#define UOP_FORMAT_DEOPT 2 +#define UOP_FORMAT_JUMP 2 +#define UOP_FORMAT_UNUSED 3 typedef struct { uint16_t opcode:14; @@ -43,7 +44,7 @@ typedef struct { struct { union { uint16_t exit_index; - uint16_t deopt_target; + uint16_t jump_target; }; uint16_t error_target; }; @@ -63,10 +64,10 @@ static inline uint16_t uop_get_exit_index(const _PyUOpInstruction *inst) return inst->exit_index; } -static inline uint16_t uop_get_deopt_target(const _PyUOpInstruction *inst) +static inline uint16_t uop_get_jump_target(const _PyUOpInstruction *inst) { - assert(inst->format == UOP_FORMAT_DEOPT); - return inst->deopt_target; + assert(inst->format == UOP_FORMAT_JUMP); + return inst->jump_target; } static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index de93d4ef14de2a..6d90136a2257ad 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -921,6 +921,7 @@ enum InstructionFormat { #define HAS_PURE_FLAG (2048) #define HAS_PASSTHROUGH_FLAG (4096) #define HAS_OPARG_AND_1_FLAG (8192) +#define HAS_NO_POP_ERROR_FLAG (16384) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -935,6 +936,7 @@ enum InstructionFormat { #define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG)) #define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) #define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) +#define OPCODE_HAS_NO_POP_ERROR(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NO_POP_ERROR_FLAG)) #define OPARG_FULL 0 #define OPARG_CACHE_1 1 @@ -954,8 +956,8 @@ struct opcode_metadata { extern const struct opcode_metadata _PyOpcode_opcode_metadata[268]; #ifdef NEED_OPCODE_METADATA const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { - [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, @@ -975,29 +977,29 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1005,7 +1007,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, @@ -1017,40 +1019,40 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, - [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1059,10 +1061,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1091,18 +1093,18 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1117,15 +1119,15 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [POP_TOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, - [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ESCAPES_FLAG }, - [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1188,8 +1190,6 @@ extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[256]; #ifdef NEED_OPCODE_METADATA const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { - [BEFORE_ASYNC_WITH] = { .nuops = 1, .uops = { { _BEFORE_ASYNC_WITH, 0, 0 } } }, - [BEFORE_WITH] = { .nuops = 1, .uops = { { _BEFORE_WITH, 0, 0 } } }, [BINARY_OP] = { .nuops = 1, .uops = { { _BINARY_OP, 0, 0 } } }, [BINARY_OP_ADD_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_ADD_FLOAT, 0, 0 } } }, [BINARY_OP_ADD_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_ADD_INT, 0, 0 } } }, @@ -1207,7 +1207,6 @@ _PyOpcode_macro_expansion[256] = { [BUILD_CONST_KEY_MAP] = { .nuops = 1, .uops = { { _BUILD_CONST_KEY_MAP, 0, 0 } } }, [BUILD_LIST] = { .nuops = 1, .uops = { { _BUILD_LIST, 0, 0 } } }, [BUILD_MAP] = { .nuops = 1, .uops = { { _BUILD_MAP, 0, 0 } } }, - [BUILD_SET] = { .nuops = 1, .uops = { { _BUILD_SET, 0, 0 } } }, [BUILD_SLICE] = { .nuops = 1, .uops = { { _BUILD_SLICE, 0, 0 } } }, [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, 0, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, 0, 0 } } }, @@ -1215,15 +1214,10 @@ _PyOpcode_macro_expansion[256] = { [CALL_BUILTIN_CLASS] = { .nuops = 1, .uops = { { _CALL_BUILTIN_CLASS, 0, 0 } } }, [CALL_BUILTIN_FAST] = { .nuops = 1, .uops = { { _CALL_BUILTIN_FAST, 0, 0 } } }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, 0, 0 } } }, - [CALL_BUILTIN_O] = { .nuops = 1, .uops = { { _CALL_BUILTIN_O, 0, 0 } } }, [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_1, 0, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, 0, 0 } } }, - [CALL_ISINSTANCE] = { .nuops = 1, .uops = { { _CALL_ISINSTANCE, 0, 0 } } }, - [CALL_LEN] = { .nuops = 1, .uops = { { _CALL_LEN, 0, 0 } } }, [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, 0, 0 } } }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_O, 0, 0 } } }, [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_STR_1] = { .nuops = 1, .uops = { { _CALL_STR_1, 0, 0 } } }, [CALL_TUPLE_1] = { .nuops = 1, .uops = { { _CALL_TUPLE_1, 0, 0 } } }, @@ -1291,7 +1285,6 @@ _PyOpcode_macro_expansion[256] = { [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, 0, 0 } } }, - [LOAD_NAME] = { .nuops = 1, .uops = { { _LOAD_NAME, 0, 0 } } }, [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_ATTR, 0, 0 } } }, [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, [MAKE_CELL] = { .nuops = 1, .uops = { { _MAKE_CELL, 0, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index d316344533fcf6..95e57f146af798 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -91,46 +91,43 @@ extern "C" { #define _DICT_UPDATE DICT_UPDATE #define _END_SEND END_SEND #define _ERROR_0 331 -#define _ERROR_1 332 -#define _ERROR_2 333 -#define _ERROR_3 334 -#define _ERROR_4 335 +#define _ERROR_N 332 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _FATAL_ERROR 336 +#define _FATAL_ERROR 333 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 337 +#define _FOR_ITER 334 #define _FOR_ITER_GEN FOR_ITER_GEN -#define _FOR_ITER_TIER_TWO 338 +#define _FOR_ITER_TIER_TWO 335 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 339 -#define _GUARD_BOTH_INT 340 -#define _GUARD_BOTH_UNICODE 341 -#define _GUARD_BUILTINS_VERSION 342 -#define _GUARD_DORV_VALUES 343 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 344 -#define _GUARD_GLOBALS_VERSION 345 -#define _GUARD_IS_FALSE_POP 346 -#define _GUARD_IS_NONE_POP 347 -#define _GUARD_IS_NOT_NONE_POP 348 -#define _GUARD_IS_TRUE_POP 349 -#define _GUARD_KEYS_VERSION 350 -#define _GUARD_NOT_EXHAUSTED_LIST 351 -#define _GUARD_NOT_EXHAUSTED_RANGE 352 -#define _GUARD_NOT_EXHAUSTED_TUPLE 353 -#define _GUARD_TYPE_VERSION 354 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 355 -#define _INIT_CALL_PY_EXACT_ARGS 356 -#define _INIT_CALL_PY_EXACT_ARGS_0 357 -#define _INIT_CALL_PY_EXACT_ARGS_1 358 -#define _INIT_CALL_PY_EXACT_ARGS_2 359 -#define _INIT_CALL_PY_EXACT_ARGS_3 360 -#define _INIT_CALL_PY_EXACT_ARGS_4 361 +#define _GUARD_BOTH_FLOAT 336 +#define _GUARD_BOTH_INT 337 +#define _GUARD_BOTH_UNICODE 338 +#define _GUARD_BUILTINS_VERSION 339 +#define _GUARD_DORV_VALUES 340 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 341 +#define _GUARD_GLOBALS_VERSION 342 +#define _GUARD_IS_FALSE_POP 343 +#define _GUARD_IS_NONE_POP 344 +#define _GUARD_IS_NOT_NONE_POP 345 +#define _GUARD_IS_TRUE_POP 346 +#define _GUARD_KEYS_VERSION 347 +#define _GUARD_NOT_EXHAUSTED_LIST 348 +#define _GUARD_NOT_EXHAUSTED_RANGE 349 +#define _GUARD_NOT_EXHAUSTED_TUPLE 350 +#define _GUARD_TYPE_VERSION 351 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 352 +#define _INIT_CALL_PY_EXACT_ARGS 353 +#define _INIT_CALL_PY_EXACT_ARGS_0 354 +#define _INIT_CALL_PY_EXACT_ARGS_1 355 +#define _INIT_CALL_PY_EXACT_ARGS_2 356 +#define _INIT_CALL_PY_EXACT_ARGS_3 357 +#define _INIT_CALL_PY_EXACT_ARGS_4 358 #define _INSTRUMENTED_CALL INSTRUMENTED_CALL #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW @@ -147,65 +144,65 @@ extern "C" { #define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST #define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE #define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 362 -#define _IS_NONE 363 +#define _INTERNAL_INCREMENT_OPT_COUNTER 359 +#define _IS_NONE 360 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 364 -#define _ITER_CHECK_RANGE 365 -#define _ITER_CHECK_TUPLE 366 -#define _ITER_JUMP_LIST 367 -#define _ITER_JUMP_RANGE 368 -#define _ITER_JUMP_TUPLE 369 -#define _ITER_NEXT_LIST 370 -#define _ITER_NEXT_RANGE 371 -#define _ITER_NEXT_TUPLE 372 -#define _JUMP_TO_TOP 373 +#define _ITER_CHECK_LIST 361 +#define _ITER_CHECK_RANGE 362 +#define _ITER_CHECK_TUPLE 363 +#define _ITER_JUMP_LIST 364 +#define _ITER_JUMP_RANGE 365 +#define _ITER_JUMP_TUPLE 366 +#define _ITER_NEXT_LIST 367 +#define _ITER_NEXT_RANGE 368 +#define _ITER_NEXT_TUPLE 369 +#define _JUMP_TO_TOP 370 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND #define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR -#define _LOAD_ATTR 374 -#define _LOAD_ATTR_CLASS 375 -#define _LOAD_ATTR_CLASS_0 376 -#define _LOAD_ATTR_CLASS_1 377 +#define _LOAD_ATTR 371 +#define _LOAD_ATTR_CLASS 372 +#define _LOAD_ATTR_CLASS_0 373 +#define _LOAD_ATTR_CLASS_1 374 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 378 -#define _LOAD_ATTR_INSTANCE_VALUE_0 379 -#define _LOAD_ATTR_INSTANCE_VALUE_1 380 -#define _LOAD_ATTR_METHOD_LAZY_DICT 381 -#define _LOAD_ATTR_METHOD_NO_DICT 382 -#define _LOAD_ATTR_METHOD_WITH_VALUES 383 -#define _LOAD_ATTR_MODULE 384 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 385 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 386 +#define _LOAD_ATTR_INSTANCE_VALUE 375 +#define _LOAD_ATTR_INSTANCE_VALUE_0 376 +#define _LOAD_ATTR_INSTANCE_VALUE_1 377 +#define _LOAD_ATTR_METHOD_LAZY_DICT 378 +#define _LOAD_ATTR_METHOD_NO_DICT 379 +#define _LOAD_ATTR_METHOD_WITH_VALUES 380 +#define _LOAD_ATTR_MODULE 381 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 382 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 383 #define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_SLOT 387 -#define _LOAD_ATTR_SLOT_0 388 -#define _LOAD_ATTR_SLOT_1 389 -#define _LOAD_ATTR_WITH_HINT 390 +#define _LOAD_ATTR_SLOT 384 +#define _LOAD_ATTR_SLOT_0 385 +#define _LOAD_ATTR_SLOT_1 386 +#define _LOAD_ATTR_WITH_HINT 387 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 391 -#define _LOAD_CONST_INLINE_BORROW 392 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 393 -#define _LOAD_CONST_INLINE_WITH_NULL 394 +#define _LOAD_CONST_INLINE 388 +#define _LOAD_CONST_INLINE_BORROW 389 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 390 +#define _LOAD_CONST_INLINE_WITH_NULL 391 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 395 -#define _LOAD_FAST_0 396 -#define _LOAD_FAST_1 397 -#define _LOAD_FAST_2 398 -#define _LOAD_FAST_3 399 -#define _LOAD_FAST_4 400 -#define _LOAD_FAST_5 401 -#define _LOAD_FAST_6 402 -#define _LOAD_FAST_7 403 +#define _LOAD_FAST 392 +#define _LOAD_FAST_0 393 +#define _LOAD_FAST_1 394 +#define _LOAD_FAST_2 395 +#define _LOAD_FAST_3 396 +#define _LOAD_FAST_4 397 +#define _LOAD_FAST_5 398 +#define _LOAD_FAST_6 399 +#define _LOAD_FAST_7 400 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 404 -#define _LOAD_GLOBAL_BUILTINS 405 -#define _LOAD_GLOBAL_MODULE 406 +#define _LOAD_GLOBAL 401 +#define _LOAD_GLOBAL_BUILTINS 402 +#define _LOAD_GLOBAL_MODULE 403 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR @@ -219,49 +216,49 @@ extern "C" { #define _MATCH_SEQUENCE MATCH_SEQUENCE #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_FRAME 407 -#define _POP_JUMP_IF_FALSE 408 -#define _POP_JUMP_IF_TRUE 409 +#define _POP_FRAME 404 +#define _POP_JUMP_IF_FALSE 405 +#define _POP_JUMP_IF_TRUE 406 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 410 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 407 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 411 +#define _PUSH_FRAME 408 #define _PUSH_NULL PUSH_NULL -#define _REPLACE_WITH_TRUE 412 +#define _REPLACE_WITH_TRUE 409 #define _RESUME_CHECK RESUME_CHECK -#define _SAVE_RETURN_OFFSET 413 -#define _SEND 414 +#define _SAVE_RETURN_OFFSET 410 +#define _SEND 411 #define _SEND_GEN SEND_GEN #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _SIDE_EXIT 415 -#define _START_EXECUTOR 416 -#define _STORE_ATTR 417 -#define _STORE_ATTR_INSTANCE_VALUE 418 -#define _STORE_ATTR_SLOT 419 +#define _SIDE_EXIT 412 +#define _START_EXECUTOR 413 +#define _STORE_ATTR 414 +#define _STORE_ATTR_INSTANCE_VALUE 415 +#define _STORE_ATTR_SLOT 416 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 420 -#define _STORE_FAST_0 421 -#define _STORE_FAST_1 422 -#define _STORE_FAST_2 423 -#define _STORE_FAST_3 424 -#define _STORE_FAST_4 425 -#define _STORE_FAST_5 426 -#define _STORE_FAST_6 427 -#define _STORE_FAST_7 428 +#define _STORE_FAST 417 +#define _STORE_FAST_0 418 +#define _STORE_FAST_1 419 +#define _STORE_FAST_2 420 +#define _STORE_FAST_3 421 +#define _STORE_FAST_4 422 +#define _STORE_FAST_5 423 +#define _STORE_FAST_6 424 +#define _STORE_FAST_7 425 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 429 +#define _STORE_SUBSCR 426 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TO_BOOL 430 +#define _TO_BOOL 427 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -271,12 +268,12 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 431 +#define _UNPACK_SEQUENCE 428 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define MAX_UOP_ID 431 +#define MAX_UOP_ID 428 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index b65311ae22bdf6..73b8ffc15b3eeb 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -85,13 +85,13 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_POP_FRAME] = HAS_ESCAPES_FLAG, [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_ASSERTION_ERROR] = 0, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, @@ -100,19 +100,18 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_LOCALS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_FROM_DICT_OR_GLOBALS] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_FROM_DICT_OR_GLOBALS] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_GLOBALS_VERSION] = HAS_DEOPT_FLAG, [_GUARD_BUILTINS_VERSION] = HAS_DEOPT_FLAG, [_LOAD_GLOBAL_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_LOAD_GLOBAL_BUILTINS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, - [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG, [_COPY_FREE_VARS] = HAS_ARG_FLAG, @@ -121,7 +120,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SET_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BUILD_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SETUP_ANNOTATIONS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_CONST_KEY_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -167,8 +165,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MATCH_SEQUENCE] = 0, [_MATCH_KEYS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_FOR_ITER_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_FOR_ITER_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_ITER_NEXT_LIST] = 0, @@ -178,8 +176,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_ITER_CHECK_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG, - [_BEFORE_ASYNC_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BEFORE_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_EXC_INFO] = 0, [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, @@ -205,18 +201,13 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CALL_STR_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, - [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, @@ -248,10 +239,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_DEOPT] = HAS_DEOPT_FLAG, [_SIDE_EXIT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, [_ERROR_0] = HAS_ERROR_FLAG, - [_ERROR_1] = HAS_ERROR_FLAG, - [_ERROR_2] = HAS_ERROR_FLAG, - [_ERROR_3] = HAS_ERROR_FLAG, - [_ERROR_4] = HAS_ERROR_FLAG, + [_ERROR_N] = HAS_ARG_FLAG | HAS_ERROR_FLAG, }; const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { @@ -261,8 +249,6 @@ const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { }; const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { - [_BEFORE_ASYNC_WITH] = "_BEFORE_ASYNC_WITH", - [_BEFORE_WITH] = "_BEFORE_WITH", [_BINARY_OP] = "_BINARY_OP", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", @@ -280,22 +266,16 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BUILD_CONST_KEY_MAP] = "_BUILD_CONST_KEY_MAP", [_BUILD_LIST] = "_BUILD_LIST", [_BUILD_MAP] = "_BUILD_MAP", - [_BUILD_SET] = "_BUILD_SET", [_BUILD_SLICE] = "_BUILD_SLICE", [_BUILD_STRING] = "_BUILD_STRING", [_BUILD_TUPLE] = "_BUILD_TUPLE", [_CALL_BUILTIN_CLASS] = "_CALL_BUILTIN_CLASS", [_CALL_BUILTIN_FAST] = "_CALL_BUILTIN_FAST", [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = "_CALL_BUILTIN_FAST_WITH_KEYWORDS", - [_CALL_BUILTIN_O] = "_CALL_BUILTIN_O", [_CALL_INTRINSIC_1] = "_CALL_INTRINSIC_1", [_CALL_INTRINSIC_2] = "_CALL_INTRINSIC_2", - [_CALL_ISINSTANCE] = "_CALL_ISINSTANCE", - [_CALL_LEN] = "_CALL_LEN", [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - [_CALL_METHOD_DESCRIPTOR_NOARGS] = "_CALL_METHOD_DESCRIPTOR_NOARGS", - [_CALL_METHOD_DESCRIPTOR_O] = "_CALL_METHOD_DESCRIPTOR_O", [_CALL_STR_1] = "_CALL_STR_1", [_CALL_TUPLE_1] = "_CALL_TUPLE_1", [_CALL_TYPE_1] = "_CALL_TYPE_1", @@ -335,10 +315,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_DICT_UPDATE] = "_DICT_UPDATE", [_END_SEND] = "_END_SEND", [_ERROR_0] = "_ERROR_0", - [_ERROR_1] = "_ERROR_1", - [_ERROR_2] = "_ERROR_2", - [_ERROR_3] = "_ERROR_3", - [_ERROR_4] = "_ERROR_4", + [_ERROR_N] = "_ERROR_N", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", [_EXIT_TRACE] = "_EXIT_TRACE", [_FATAL_ERROR] = "_FATAL_ERROR", @@ -429,7 +406,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", [_LOAD_LOCALS] = "_LOAD_LOCALS", - [_LOAD_NAME] = "_LOAD_NAME", [_LOAD_SUPER_ATTR_ATTR] = "_LOAD_SUPER_ATTR_ATTR", [_LOAD_SUPER_ATTR_METHOD] = "_LOAD_SUPER_ATTR_METHOD", [_MAKE_CELL] = "_MAKE_CELL", @@ -666,8 +642,6 @@ int _PyUop_Popped(int opcode, int oparg) return 0; case _LOAD_FROM_DICT_OR_GLOBALS: return 1; - case _LOAD_NAME: - return 0; case _LOAD_GLOBAL: return 0; case _GUARD_GLOBALS_VERSION: @@ -702,8 +676,6 @@ int _PyUop_Popped(int opcode, int oparg) return 2 + (oparg-1); case _SET_UPDATE: return 2 + (oparg-1); - case _BUILD_SET: - return oparg; case _BUILD_MAP: return oparg*2; case _SETUP_ANNOTATIONS: @@ -816,10 +788,6 @@ int _PyUop_Popped(int opcode, int oparg) return 1; case _ITER_NEXT_RANGE: return 1; - case _BEFORE_ASYNC_WITH: - return 1; - case _BEFORE_WITH: - return 1; case _WITH_EXCEPT_START: return 4; case _PUSH_EXC_INFO: @@ -874,22 +842,12 @@ int _PyUop_Popped(int opcode, int oparg) return 1; case _CALL_BUILTIN_CLASS: return 2 + oparg; - case _CALL_BUILTIN_O: - return 2 + oparg; case _CALL_BUILTIN_FAST: return 2 + oparg; case _CALL_BUILTIN_FAST_WITH_KEYWORDS: return 2 + oparg; - case _CALL_LEN: - return 2 + oparg; - case _CALL_ISINSTANCE: - return 2 + oparg; - case _CALL_METHOD_DESCRIPTOR_O: - return 2 + oparg; case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: return 2 + oparg; - case _CALL_METHOD_DESCRIPTOR_NOARGS: - return 2 + oparg; case _CALL_METHOD_DESCRIPTOR_FAST: return 2 + oparg; case _MAKE_FUNCTION: @@ -956,14 +914,8 @@ int _PyUop_Popped(int opcode, int oparg) return 0; case _ERROR_0: return 0; - case _ERROR_1: - return 1; - case _ERROR_2: - return 2; - case _ERROR_3: - return 3; - case _ERROR_4: - return 4; + case _ERROR_N: + return oparg; default: return -1; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6b38b2c1a57c69..35664fc3708187 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1662,12 +1662,8 @@ dummy_func( } inst(BUILD_CONST_KEY_MAP, (values[oparg], keys -- map)) { - if (!PyTuple_CheckExact(keys) || - PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { - _PyErr_SetString(tstate, PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); - GOTO_ERROR(error); // Pop the keys and values. - } + assert(PyTuple_CheckExact(keys)); + assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); @@ -4169,19 +4165,9 @@ dummy_func( ERROR_IF(1, error); } - tier2 op(_ERROR_1, (value --)) { - ERROR_IF(1, error); - } - - tier2 op(_ERROR_2, (value, value1 --)) { - ERROR_IF(1, error); - } - - tier2 op(_ERROR_3, (value, value1, value2 --)) { - ERROR_IF(1, error); - } - - tier2 op(_ERROR_4, (value, value1, value2, val --)) { + tier2 op(_ERROR_N, (values[oparg] --)) { + (void)values; + SYNC_SP(); ERROR_IF(1, error); } diff --git a/Python/ceval.c b/Python/ceval.c index ab00c9e6c8b146..6e9d05e6d8e1af 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1067,7 +1067,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } #endif OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (next_uop[-1].format == UOP_FORMAT_DEOPT) { + if (next_uop[-1].format == UOP_FORMAT_JUMP) { uint16_t target = uop_get_error_target(&next_uop[-1]); next_uop = current_executor->trace + target; goto tier2_dispatch; @@ -1081,8 +1081,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int // Jump here from DEOPT_IF() deoptimize: - if (next_uop[-1].format == UOP_FORMAT_DEOPT) { - uint16_t target = uop_get_deopt_target(&next_uop[-1]); + if (next_uop[-1].format == UOP_FORMAT_JUMP) { + uint16_t target = uop_get_jump_target(&next_uop[-1]); next_uop = current_executor->trace + target; goto tier2_dispatch; } @@ -1104,8 +1104,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int // Jump here from EXIT_IF() side_exit: - if (next_uop[-1].format == UOP_FORMAT_DEOPT) { - uint16_t target = uop_get_deopt_target(&next_uop[-1]); + if (next_uop[-1].format == UOP_FORMAT_JUMP) { + uint16_t target = uop_get_jump_target(&next_uop[-1]); next_uop = current_executor->trace + target; goto tier2_dispatch; } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e88d0573a22c15..d3eeabc537ff66 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -24,7 +24,7 @@ break; } - /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 because it is instrumented */ case _LOAD_FAST_CHECK: { PyObject *value; @@ -701,7 +701,7 @@ break; } - /* _BINARY_SUBSCR_GETITEM is not a viable micro-op for tier 2 */ + /* _BINARY_SUBSCR_GETITEM is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _LIST_APPEND: { PyObject *v; @@ -855,9 +855,9 @@ break; } - /* _INSTRUMENTED_RETURN_VALUE is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_RETURN_VALUE is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 because it is instrumented */ case _GET_AITER: { PyObject *obj; @@ -967,11 +967,11 @@ break; } - /* _SEND is not a viable micro-op for tier 2 */ + /* _SEND is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - /* _SEND_GEN is not a viable micro-op for tier 2 */ + /* _SEND_GEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 because it is instrumented */ case _POP_EXCEPT: { PyObject *exc_value; @@ -1226,39 +1226,7 @@ break; } - case _LOAD_NAME: { - PyObject *v; - oparg = CURRENT_OPARG(); - PyObject *mod_or_class_dict = LOCALS(); - if (mod_or_class_dict == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) JUMP_TO_ERROR; - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - GOTO_ERROR(error); - } - } - } - stack_pointer[0] = v; - stack_pointer += 1; - break; - } + /* _LOAD_NAME is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ case _LOAD_GLOBAL: { PyObject *res; @@ -1539,29 +1507,7 @@ break; } - case _BUILD_SET: { - PyObject **values; - PyObject *set; - oparg = CURRENT_OPARG(); - values = &stack_pointer[-oparg]; - set = PySet_New(NULL); - if (set == NULL) - GOTO_ERROR(error); - int err = 0; - for (int i = 0; i < oparg; i++) { - PyObject *item = values[i]; - if (err == 0) - err = PySet_Add(set, item); - Py_DECREF(item); - } - if (err != 0) { - Py_DECREF(set); - if (true) JUMP_TO_ERROR; - } - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; - break; - } + /* _BUILD_SET is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ case _BUILD_MAP: { PyObject **values; @@ -1612,12 +1558,8 @@ oparg = CURRENT_OPARG(); keys = stack_pointer[-1]; values = &stack_pointer[-1 - oparg]; - if (!PyTuple_CheckExact(keys) || - PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { - _PyErr_SetString(tstate, PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); - GOTO_ERROR(error); // Pop the keys and values. - } + assert(PyTuple_CheckExact(keys)); + assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); @@ -1685,7 +1627,7 @@ break; } - /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 because it is instrumented */ case _LOAD_SUPER_ATTR_ATTR: { PyObject *self; @@ -2014,9 +1956,9 @@ /* _LOAD_ATTR_CLASS is split on (oparg & 1) */ - /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 */ + /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ + /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _GUARD_DORV_VALUES: { PyObject *owner; @@ -2049,7 +1991,7 @@ break; } - /* _STORE_ATTR_WITH_HINT is not a viable micro-op for tier 2 */ + /* _STORE_ATTR_WITH_HINT is not a viable micro-op for tier 2 because it has unused cache entries */ case _STORE_ATTR_SLOT: { PyObject *owner; @@ -2277,9 +2219,9 @@ break; } - /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is replaced */ - /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is replaced */ case _IS_NONE: { PyObject *value; @@ -2419,7 +2361,7 @@ break; } - /* _FOR_ITER is not a viable micro-op for tier 2 */ + /* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */ case _FOR_ITER_TIER_TWO: { PyObject *iter; @@ -2446,7 +2388,7 @@ break; } - /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */ case _ITER_CHECK_LIST: { PyObject *iter; @@ -2455,7 +2397,7 @@ break; } - /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 */ + /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */ case _GUARD_NOT_EXHAUSTED_LIST: { PyObject *iter; @@ -2490,7 +2432,7 @@ break; } - /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 */ + /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */ case _GUARD_NOT_EXHAUSTED_TUPLE: { PyObject *iter; @@ -2526,7 +2468,7 @@ break; } - /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 */ + /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 because it is replaced */ case _GUARD_NOT_EXHAUSTED_RANGE: { PyObject *iter; @@ -2554,90 +2496,11 @@ break; } - /* _FOR_ITER_GEN is not a viable micro-op for tier 2 */ + /* _FOR_ITER_GEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _BEFORE_ASYNC_WITH: { - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol", - Py_TYPE(mgr)->tp_name); - } - GOTO_ERROR(error); - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol " - "(missed __aexit__ method)", - Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); - GOTO_ERROR(error); - } - Py_DECREF(mgr); - res = PyObject_CallNoArgs(enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) JUMP_TO_ERROR; - } - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; - break; - } + /* _BEFORE_ASYNC_WITH is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ - case _BEFORE_WITH: { - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - /* pop the context manager, push its __exit__ and the - * value returned from calling its __enter__ - */ - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol", - Py_TYPE(mgr)->tp_name); - } - GOTO_ERROR(error); - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol " - "(missed __exit__ method)", - Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); - GOTO_ERROR(error); - } - Py_DECREF(mgr); - res = PyObject_CallNoArgs(enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) JUMP_TO_ERROR; - } - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; - break; - } + /* _BEFORE_WITH is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ case _WITH_EXCEPT_START: { PyObject *val; @@ -2815,9 +2678,9 @@ break; } - /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 because it is instrumented */ - /* _CALL is not a viable micro-op for tier 2 */ + /* _CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { PyObject *null; @@ -3056,7 +2919,7 @@ break; } - /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 */ + /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _CALL_TYPE_1: { PyObject *arg; @@ -3122,7 +2985,7 @@ break; } - /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 */ + /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _EXIT_INIT_CHECK: { PyObject *should_be_none; @@ -3169,43 +3032,7 @@ break; } - case _CALL_BUILTIN_O: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_O functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - if (total_args != 1) DEOPTIMIZE; - if (!PyCFunction_CheckExact(callable)) DEOPTIMIZE; - if (PyCFunction_GET_FLAGS(callable) != METH_O) DEOPTIMIZE; - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } - PyObject *arg = args[0]; - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - break; - } + /* _CALL_BUILTIN_O is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ case _CALL_BUILTIN_FAST: { PyObject **args; @@ -3285,117 +3112,11 @@ break; } - case _CALL_LEN: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* len(o) */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - if (total_args != 1) DEOPTIMIZE; - PyInterpreterState *interp = tstate->interp; - if (callable != interp->callable_cache.len) DEOPTIMIZE; - STAT_INC(CALL, hit); - PyObject *arg = args[0]; - Py_ssize_t len_i = PyObject_Length(arg); - if (len_i < 0) { - GOTO_ERROR(error); - } - res = PyLong_FromSsize_t(len_i); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - Py_DECREF(arg); - if (res == NULL) JUMP_TO_ERROR; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - break; - } + /* _CALL_LEN is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ - case _CALL_ISINSTANCE: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* isinstance(o, o2) */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - if (total_args != 2) DEOPTIMIZE; - PyInterpreterState *interp = tstate->interp; - if (callable != interp->callable_cache.isinstance) DEOPTIMIZE; - STAT_INC(CALL, hit); - PyObject *cls = args[1]; - PyObject *inst = args[0]; - int retval = PyObject_IsInstance(inst, cls); - if (retval < 0) { - GOTO_ERROR(error); - } - res = PyBool_FromLong(retval); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - break; - } + /* _CALL_ISINSTANCE is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ - case _CALL_METHOD_DESCRIPTOR_O: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (total_args != 2) DEOPTIMIZE; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) DEOPTIMIZE; - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_O) DEOPTIMIZE; - PyObject *arg = args[1]; - PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) DEOPTIMIZE; - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } - res = _PyCFunction_TrampolineCall(cfunc, self, arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - break; - } + /* _CALL_METHOD_DESCRIPTOR_O is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { PyObject **args; @@ -3436,46 +3157,7 @@ break; } - case _CALL_METHOD_DESCRIPTOR_NOARGS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - if (total_args != 1) DEOPTIMIZE; - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) DEOPTIMIZE; - PyMethodDef *meth = method->d_method; - PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) DEOPTIMIZE; - if (meth->ml_flags != METH_NOARGS) DEOPTIMIZE; - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); - } - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - break; - } + /* _CALL_METHOD_DESCRIPTOR_NOARGS is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ case _CALL_METHOD_DESCRIPTOR_FAST: { PyObject **args; @@ -3516,13 +3198,13 @@ break; } - /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 because it is instrumented */ - /* _CALL_KW is not a viable micro-op for tier 2 */ + /* _CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it is instrumented */ - /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _MAKE_FUNCTION: { PyObject *codeobj; @@ -3684,19 +3366,19 @@ break; } - /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_JUMP_BACKWARD is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_JUMP_BACKWARD is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_POP_JUMP_IF_NONE is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_POP_JUMP_IF_NONE is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 because it is instrumented */ case _GUARD_IS_TRUE_POP: { PyObject *flag; @@ -3913,47 +3595,13 @@ break; } - case _ERROR_1: { - PyObject *value; - value = stack_pointer[-1]; - if (1) JUMP_TO_ERROR; - stack_pointer += -1; - break; - } - - case _ERROR_2: { - PyObject *value1; - PyObject *value; - value1 = stack_pointer[-1]; - value = stack_pointer[-2]; - if (1) JUMP_TO_ERROR; - stack_pointer += -2; - break; - } - - case _ERROR_3: { - PyObject *value2; - PyObject *value1; - PyObject *value; - value2 = stack_pointer[-1]; - value1 = stack_pointer[-2]; - value = stack_pointer[-3]; - if (1) JUMP_TO_ERROR; - stack_pointer += -3; - break; - } - - case _ERROR_4: { - PyObject *val; - PyObject *value2; - PyObject *value1; - PyObject *value; - val = stack_pointer[-1]; - value2 = stack_pointer[-2]; - value1 = stack_pointer[-3]; - value = stack_pointer[-4]; + case _ERROR_N: { + PyObject **values; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg]; + (void)values; + stack_pointer += -oparg; if (1) JUMP_TO_ERROR; - stack_pointer += -4; break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 53c0211be2fe6c..cb396887b8bbf7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -605,12 +605,8 @@ PyObject *map; keys = stack_pointer[-1]; values = &stack_pointer[-1 - oparg]; - if (!PyTuple_CheckExact(keys) || - PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { - _PyErr_SetString(tstate, PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); - GOTO_ERROR(error); // Pop the keys and values. - } + assert(PyTuple_CheckExact(keys)); + assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); diff --git a/Python/optimizer.c b/Python/optimizer.c index 3327eeff7f5e0f..ebb71673215de6 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -313,10 +313,33 @@ _PyUOpPrint(const _PyUOpInstruction *uop) else { printf("%s", name); } - printf(" (%d, target=%d, operand=%" PRIx64 ")", - uop->oparg, - uop->target, - (uint64_t)uop->operand); + switch(uop->format) { + case UOP_FORMAT_TARGET: + printf(" (%d, target=%d, operand=%" PRIx64, + uop->oparg, + uop->target, + (uint64_t)uop->operand); + break; + case UOP_FORMAT_JUMP: + printf(" (%d, jump_target=%d, operand=%" PRIx64, + uop->oparg, + uop->jump_target, + (uint64_t)uop->operand); + break; + case UOP_FORMAT_EXIT: + printf(" (%d, exit_index=%d, operand=%" PRIx64, + uop->oparg, + uop->exit_index, + (uint64_t)uop->operand); + break; + default: + printf(" (%d, Unknown format)", uop->oparg); + } + if (_PyUop_Flags[uop->opcode] & HAS_ERROR_FLAG) { + printf(", error_target=%d", uop->error_target); + } + + printf(")"); } #endif @@ -432,30 +455,36 @@ BRANCH_TO_GUARD[4][2] = { #endif -// Beware: Macro arg order differs from struct member order +static inline int +add_to_trace( + _PyUOpInstruction *trace, + int trace_length, + uint16_t opcode, + uint16_t oparg, + uint64_t operand, + uint32_t target) +{ + trace[trace_length].opcode = opcode; + trace[trace_length].format = UOP_FORMAT_TARGET; + trace[trace_length].target = target; + trace[trace_length].oparg = oparg; + trace[trace_length].operand = operand; + return trace_length + 1; +} + #ifdef Py_DEBUG #define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ assert(trace_length < max_length); \ - trace[trace_length].opcode = (OPCODE); \ - trace[trace_length].format = UOP_FORMAT_TARGET; \ - trace[trace_length].oparg = (OPARG); \ - trace[trace_length].target = (TARGET); \ - trace[trace_length].operand = (OPERAND); \ + trace_length = add_to_trace(trace, trace_length, (OPCODE), (OPARG), (OPERAND), (TARGET)); \ if (lltrace >= 2) { \ printf("%4d ADD_TO_TRACE: ", trace_length); \ - _PyUOpPrint(&trace[trace_length]); \ + _PyUOpPrint(&trace[trace_length-1]); \ printf("\n"); \ - } \ - trace_length++; + } #else #define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ assert(trace_length < max_length); \ - trace[trace_length].opcode = (OPCODE); \ - trace[trace_length].format = UOP_FORMAT_TARGET; \ - trace[trace_length].oparg = (OPARG); \ - trace[trace_length].target = (TARGET); \ - trace[trace_length].operand = (OPERAND); \ - trace_length++; + trace_length = add_to_trace(trace, trace_length, (OPCODE), (OPARG), (OPERAND), (TARGET)); #endif #define INSTR_IP(INSTR, CODE) \ @@ -478,7 +507,6 @@ BRANCH_TO_GUARD[4][2] = { if (trace_stack_depth >= TRACE_STACK_SIZE) { \ DPRINTF(2, "Trace stack overflow\n"); \ OPT_STAT_INC(trace_stack_overflow); \ - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); \ goto done; \ } \ assert(func->func_code == (PyObject *)code); \ @@ -514,7 +542,8 @@ translate_bytecode_to_trace( _Py_BloomFilter_Add(dependencies, initial_code); _Py_CODEUNIT *initial_instr = instr; int trace_length = 0; - int max_length = buffer_size-1; + // Leave space for possible trailing _EXIT_TRACE + int max_length = buffer_size-2; struct { PyFunctionObject *func; _Py_CODEUNIT *instr; @@ -536,6 +565,7 @@ translate_bytecode_to_trace( PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, 2 * INSTR_IP(initial_instr, code)); + ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)instr, INSTR_IP(instr, code)); uint32_t target = 0; top: // Jump here after _PUSH_FRAME or likely branches @@ -587,10 +617,14 @@ translate_bytecode_to_trace( } } - if (OPCODE_HAS_DEOPT(opcode) || OPCODE_HAS_ERROR(opcode)) { + if (OPCODE_HAS_DEOPT(opcode) || OPCODE_HAS_EXIT(opcode)) { // Make space for exit code max_length--; } + if (OPCODE_HAS_ERROR(opcode)) { + // Make space for error code + max_length--; + } switch (opcode) { case POP_JUMP_IF_NONE: case POP_JUMP_IF_NOT_NONE: @@ -817,7 +851,9 @@ translate_bytecode_to_trace( 2 * INSTR_IP(initial_instr, code)); return 0; } - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + if (trace[trace_length-1].opcode != _JUMP_TO_TOP) { + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + } DPRINTF(1, "Created a trace for %s (%s:%d) at byte offset %d -- length %d\n", PyUnicode_AsUTF8(code->co_qualname), @@ -842,80 +878,72 @@ translate_bytecode_to_trace( /* Count the number of unused uops and exits */ static int -count_exits_and_nops(_PyUOpInstruction *buffer, int length, int *exit_count_ptr) +count_exits(_PyUOpInstruction *buffer, int length) { int exit_count = 0; - int nop_count = 0; for (int i = 0; i < length; i++) { int opcode = buffer[i].opcode; - if (opcode == _NOP) { - nop_count++; - } if (opcode == _SIDE_EXIT) { exit_count++; } } - *exit_count_ptr = exit_count; - return nop_count; + return exit_count; } -static uint16_t ERRORS[5] = { - [0] = _ERROR_0, - [1] = _ERROR_1, - [2] = _ERROR_2, - [3] = _ERROR_3, - [4] = _ERROR_4, -}; +static void make_exit(_PyUOpInstruction *inst, int opcode, int target) +{ + inst->opcode = opcode; + inst->oparg = 0; + inst->format = UOP_FORMAT_TARGET; + inst->target = target; +} /* Convert implicit exits, errors and deopts * into explicit ones. */ static int prepare_for_execution(_PyUOpInstruction *buffer, int length) { - int next_exit = length; - int exit_index = 0; + int next_spare = length; + int32_t current_jump = -1; + int32_t current_jump_target = -1; + int32_t current_error = -1; + int32_t current_error_target = -1; + int32_t current_popped = -1; for (int i = 0; i < length; i++) { _PyUOpInstruction *inst = &buffer[i]; int opcode = inst->opcode; - int32_t current_exit = -1; - int32_t current_exit_target = -1; - int32_t current_error_target = -1; - int32_t current_error = -1; int32_t target = (int32_t)uop_get_target(inst); if (_PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) { - if (target != current_exit_target) { - current_exit_target = target; - if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { - buffer[next_exit].opcode = _SIDE_EXIT; - buffer[next_exit].format = UOP_FORMAT_EXIT; - buffer[next_exit].exit_index = exit_index; - } - else { - buffer[next_exit].opcode = _DEOPT; - buffer[next_exit].format = UOP_FORMAT_TARGET; - buffer[next_exit].target = target; - exit_index++; - } - current_exit = next_exit; - next_exit++; + if (target != current_jump_target) { + uint16_t exit_op = (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) ? _SIDE_EXIT : _DEOPT; + make_exit(&buffer[next_spare], exit_op, target); + current_jump_target = target; + current_jump = next_spare; + next_spare++; } - buffer[i].deopt_target = current_exit; - buffer[i].format = _PyUop_Flags[opcode] & HAS_EXIT_FLAG ? UOP_FORMAT_EXIT : UOP_FORMAT_DEOPT ; + buffer[i].jump_target = current_jump; + buffer[i].format = UOP_FORMAT_JUMP; } if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) { - if (target != current_error_target) { - int popped = _PyUop_Popped(opcode, inst->oparg); - assert(popped < 5); + int popped = (_PyUop_Flags[opcode] & HAS_NO_POP_ERROR_FLAG) ? + 0 : _PyUop_Popped(opcode, inst->oparg); + if (target != current_error_target || popped != current_popped) { + current_popped = popped; + current_error = next_spare; current_error_target = target; - buffer[next_exit].opcode = ERRORS[popped]; - buffer[next_exit].target = target; - current_error = next_exit; - next_exit++; + uint16_t error_op = popped ? _ERROR_N : _ERROR_0; + make_exit(&buffer[next_spare], error_op, 0); + buffer[next_spare].oparg = popped; + next_spare++; } buffer[i].error_target = current_error; + if (buffer[i].format == UOP_FORMAT_TARGET) { + buffer[i].format = UOP_FORMAT_JUMP; + buffer[i].jump_target = 0; + } } } - return next_exit; + return next_spare; } /* Executor side exits */ @@ -934,6 +962,81 @@ allocate_executor(int exit_count, int length) return res; } +#ifdef Py_DEBUG + +#define CHECK(PRED) \ +if (!(PRED)) { \ + printf(#PRED " at %d\n", i); \ + assert(0); \ +} + +static int +target_unused(int opcode) +{ + return (_PyUop_Flags[opcode] & (HAS_ERROR_FLAG | HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) == 0; +} + +static void +sanity_check(_PyExecutorObject *executor) +{ + for (uint32_t i = 0; i < executor->exit_count; i++) { + _PyExitData *exit = &executor->exits[i]; + CHECK(exit->target < (1 << 25)); + } + bool ended = false; + uint32_t i = 0; + CHECK(executor->trace[0].opcode == _START_EXECUTOR); + if (executor->trace[1].opcode == _COLD_EXIT) { + return; + } + for (; i < executor->code_size; i++) { + const _PyUOpInstruction *inst = &executor->trace[i]; + uint16_t opcode = inst->opcode; + CHECK(opcode <= MAX_UOP_ID); + CHECK(_PyOpcode_uop_name[opcode] != NULL); + switch(inst->format) { + case UOP_FORMAT_TARGET: + CHECK(target_unused(opcode)); + break; + case UOP_FORMAT_EXIT: + CHECK(opcode == _SIDE_EXIT); + CHECK(inst->exit_index < executor->exit_count); + break; + case UOP_FORMAT_JUMP: + CHECK(inst->jump_target < executor->code_size); + break; + case UOP_FORMAT_UNUSED: + CHECK(0); + break; + } + if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) { + CHECK(inst->format == UOP_FORMAT_JUMP); + CHECK(inst->error_target < executor->code_size); + } + if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE) { + ended = true; + i++; + break; + } + } + CHECK(ended); + for (; i < executor->code_size; i++) { + const _PyUOpInstruction *inst = &executor->trace[i]; + uint16_t opcode = inst->opcode; + CHECK( + opcode == _DEOPT || + opcode == _SIDE_EXIT || + opcode == _ERROR_0 || + opcode == _ERROR_N); + if (opcode == _SIDE_EXIT) { + CHECK(inst->format == UOP_FORMAT_EXIT); + } + } +} + +#undef CHECK +#endif + /* Makes an executor from a buffer of uops. * Account for the buffer having gaps and NOPs by computing a "used" * bit vector and only copying the used uops. Here "used" means reachable @@ -942,10 +1045,8 @@ allocate_executor(int exit_count, int length) static _PyExecutorObject * make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies) { - int exit_count; - int nop_count = count_exits_and_nops(buffer, length, &exit_count); - int executor_length = length-nop_count+1; // 1 for _START_EXECUTOR - _PyExecutorObject *executor = allocate_executor(exit_count, executor_length); + int exit_count = count_exits(buffer, length); + _PyExecutorObject *executor = allocate_executor(exit_count, length); if (executor == NULL) { return NULL; } @@ -955,30 +1056,24 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil executor->exits[i].temperature = 0; } int next_exit = exit_count-1; - _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[executor_length-1]; + _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; + assert(buffer[0].opcode == _START_EXECUTOR); + buffer[0].operand = (uint64_t)executor; for (int i = length-1; i >= 0; i--) { int opcode = buffer[i].opcode; - if (opcode == NOP) { - continue; - } + dest--; *dest = buffer[i]; assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE); if (opcode == _SIDE_EXIT) { executor->exits[next_exit].target = buffer[i].target; dest->exit_index = next_exit; + dest->format = UOP_FORMAT_EXIT; next_exit--; } - if (buffer[i].format == UOP_FORMAT_DEOPT) { - dest->deopt_target -= nop_count; - } - dest--; } assert(next_exit == -1); assert(dest == executor->trace); - dest->opcode = _START_EXECUTOR; - dest->oparg = 0; - dest->target = 0; - dest->operand = (uintptr_t)executor; + assert(dest->opcode == _START_EXECUTOR); _Py_ExecutorInit(executor, dependencies); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -994,6 +1089,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil printf("\n"); } } + sanity_check(executor); #endif #ifdef _Py_JIT executor->jit_code = NULL; @@ -1078,11 +1174,12 @@ uop_optimize( assert(strncmp(_PyOpcode_uop_name[buffer[pc].opcode], _PyOpcode_uop_name[opcode], strlen(_PyOpcode_uop_name[opcode])) == 0); } length = prepare_for_execution(buffer, length); - assert(length <= UOP_MAX_TRACE_LENGTH); + // assert(length <= UOP_MAX_TRACE_LENGTH); _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies); if (executor == NULL) { return -1; } + assert(length <= UOP_MAX_TRACE_LENGTH); OPT_HIST(Py_SIZE(executor), optimized_trace_length_hist); *exec_ptr = executor; return 1; @@ -1158,12 +1255,14 @@ counter_optimize( return 0; } _Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[JUMP_BACKWARD] - oparg; - _PyUOpInstruction buffer[3] = { + _PyUOpInstruction buffer[5] = { + { .opcode = _START_EXECUTOR }, { .opcode = _LOAD_CONST_INLINE_BORROW, .operand = (uintptr_t)self }, { .opcode = _INTERNAL_INCREMENT_OPT_COUNTER }, - { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)), .format=UOP_FORMAT_TARGET } + { .opcode = _EXIT_TRACE, .jump_target = 4, .format=UOP_FORMAT_JUMP }, + { .opcode = _SIDE_EXIT, .target = (uint32_t)(target - _PyCode_CODE(code)), .format=UOP_FORMAT_TARGET } }; - _PyExecutorObject *executor = make_executor_from_uops(buffer, 3, &EMPTY_FILTER); + _PyExecutorObject *executor = make_executor_from_uops(buffer, 5, &EMPTY_FILTER); if (executor == NULL) { return -1; } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index cbd782b445cbce..235c26b1af1bae 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -334,7 +334,7 @@ optimize_to_bool( return 0; } -static void +static bool eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit) { REPLACE_OP(this_instr, _POP_TOP, 0, 0); @@ -342,6 +342,7 @@ eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit) REPLACE_OP((this_instr+1), _EXIT_TRACE, 0, 0); this_instr[1].target = this_instr->target; } + return exit; } /* 1 for success, 0 for not ready, cannot error at the moment. */ @@ -368,9 +369,9 @@ optimize_uops( ctx->curr_frame_depth++; ctx->frame = frame; - for (_PyUOpInstruction *this_instr = trace; - this_instr < trace + trace_len && !op_is_end(this_instr->opcode); - this_instr++) { + _PyUOpInstruction *this_instr = NULL; + for (int i = 0; i < trace_len; i++) { + this_instr = &trace[i]; int oparg = this_instr->oparg; uint32_t opcode = this_instr->opcode; @@ -392,7 +393,10 @@ optimize_uops( ctx->frame->stack_pointer = stack_pointer; assert(STACK_LEVEL() >= 0); } - + if (this_instr != trace + trace_len) { + assert (this_instr < trace + trace_len && this_instr > trace); + trace_len = this_instr - trace + 1; + } _Py_uop_abstractcontext_fini(ctx); return trace_len; @@ -417,7 +421,7 @@ optimize_uops( } -static void +static int remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) { /* Remove _SET_IP and _CHECK_VALIDITY where possible. @@ -426,7 +430,8 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) * instruction could have escaped. */ int last_set_ip = -1; bool may_have_escaped = true; - for (int pc = 0; pc < buffer_size; pc++) { + for (int pc = 0; ; pc++) { + assert(pc < buffer_size && "No terminating uop"); int opcode = buffer[pc].opcode; switch (opcode) { case _SET_IP: @@ -472,7 +477,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } case _JUMP_TO_TOP: case _EXIT_TRACE: - return; + return pc + 1; default: { bool needs_ip = false; @@ -556,11 +561,8 @@ _Py_uop_analyze_and_optimize( OPT_STAT_INC(optimizer_attempts); int err = remove_globals(frame, buffer, length, dependencies); - if (err == 0) { - goto not_ready; - } - if (err < 0) { - goto error; + if (err <= 0) { + return err; } peephole_opt(frame, buffer, length); @@ -569,17 +571,13 @@ _Py_uop_analyze_and_optimize( (PyCodeObject *)frame->f_executable, buffer, length, curr_stacklen, dependencies); - if (err == 0) { - goto not_ready; + if (length <= 0) { + return length; } - assert(err == 1); - remove_unneeded_uops(buffer, length); + length = remove_unneeded_uops(buffer, length); + assert(length > 0); OPT_STAT_INC(optimizer_successes); return length; -not_ready: - return 0; -error: - return -1; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 78df1b9d8ea8f8..4c2a644ebe5183 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -769,14 +769,7 @@ break; } - case _LOAD_NAME: { - _Py_UopsSymbol *v; - v = sym_new_unknown(ctx); - if (v == NULL) goto out_of_space; - stack_pointer[0] = v; - stack_pointer += 1; - break; - } + /* _LOAD_NAME is not a viable micro-op for tier 2 */ case _LOAD_GLOBAL: { _Py_UopsSymbol *res; @@ -900,14 +893,7 @@ break; } - case _BUILD_SET: { - _Py_UopsSymbol *set; - set = sym_new_unknown(ctx); - if (set == NULL) goto out_of_space; - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; - break; - } + /* _BUILD_SET is not a viable micro-op for tier 2 */ case _BUILD_MAP: { _Py_UopsSymbol *map; @@ -1405,31 +1391,9 @@ /* _FOR_ITER_GEN is not a viable micro-op for tier 2 */ - case _BEFORE_ASYNC_WITH: { - _Py_UopsSymbol *exit; - _Py_UopsSymbol *res; - exit = sym_new_unknown(ctx); - if (exit == NULL) goto out_of_space; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; - break; - } + /* _BEFORE_ASYNC_WITH is not a viable micro-op for tier 2 */ - case _BEFORE_WITH: { - _Py_UopsSymbol *exit; - _Py_UopsSymbol *res; - exit = sym_new_unknown(ctx); - if (exit == NULL) goto out_of_space; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; - break; - } + /* _BEFORE_WITH is not a viable micro-op for tier 2 */ case _WITH_EXCEPT_START: { _Py_UopsSymbol *res; @@ -1673,14 +1637,7 @@ break; } - case _CALL_BUILTIN_O: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - break; - } + /* _CALL_BUILTIN_O is not a viable micro-op for tier 2 */ case _CALL_BUILTIN_FAST: { _Py_UopsSymbol *res; @@ -1700,32 +1657,11 @@ break; } - case _CALL_LEN: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - break; - } + /* _CALL_LEN is not a viable micro-op for tier 2 */ - case _CALL_ISINSTANCE: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - break; - } + /* _CALL_ISINSTANCE is not a viable micro-op for tier 2 */ - case _CALL_METHOD_DESCRIPTOR_O: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - break; - } + /* _CALL_METHOD_DESCRIPTOR_O is not a viable micro-op for tier 2 */ case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { _Py_UopsSymbol *res; @@ -1736,14 +1672,7 @@ break; } - case _CALL_METHOD_DESCRIPTOR_NOARGS: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - break; - } + /* _CALL_METHOD_DESCRIPTOR_NOARGS is not a viable micro-op for tier 2 */ case _CALL_METHOD_DESCRIPTOR_FAST: { _Py_UopsSymbol *res; @@ -2020,23 +1949,8 @@ break; } - case _ERROR_1: { - stack_pointer += -1; - break; - } - - case _ERROR_2: { - stack_pointer += -2; - break; - } - - case _ERROR_3: { - stack_pointer += -3; - break; - } - - case _ERROR_4: { - stack_pointer += -4; + case _ERROR_N: { + stack_pointer += -oparg; break; } diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 27e6ba2b3fdedf..3f9a016c731688 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -8,7 +8,8 @@ @dataclass class Properties: escapes: bool - infallible: bool + pop_error: bool + no_pop_error: bool deopts: bool oparg: bool jumps: bool @@ -37,7 +38,8 @@ def dump(self, indent: str) -> None: def from_list(properties: list["Properties"]) -> "Properties": return Properties( escapes=any(p.escapes for p in properties), - infallible=all(p.infallible for p in properties), + pop_error=any(p.pop_error for p in properties), + no_pop_error=any(p.no_pop_error for p in properties), deopts=any(p.deopts for p in properties), oparg=any(p.oparg for p in properties), jumps=any(p.jumps for p in properties), @@ -55,10 +57,16 @@ def from_list(properties: list["Properties"]) -> "Properties": passthrough=all(p.passthrough for p in properties), ) + @property + def infallible(self): + return not self.pop_error and not self.no_pop_error + + SKIP_PROPERTIES = Properties( escapes=False, - infallible=True, + pop_error=False, + no_pop_error=False, deopts=False, oparg=False, jumps=False, @@ -157,20 +165,25 @@ def size(self) -> int: self._size = sum(c.size for c in self.caches) return self._size - def is_viable(self) -> bool: + def why_not_viable(self) -> str | None: if self.name == "_SAVE_RETURN_OFFSET": - return True # Adjusts next_instr, but only in tier 1 code - if self.properties.needs_this: - return False + return None # Adjusts next_instr, but only in tier 1 code if "INSTRUMENTED" in self.name: - return False + return "is instrumented" if "replaced" in self.annotations: - return False + return "is replaced" if self.name in ("INTERPRETER_EXIT", "JUMP_BACKWARD"): - return False + return "has tier 1 control flow" + if self.properties.needs_this: + return "uses the 'this_instr' variable" if len([c for c in self.caches if c.name != "unused"]) > 1: - return False - return True + return "has unused cache entries" + if self.properties.pop_error and self.properties.no_pop_error: + return "has both popping and not-popping errors" + return None + + def is_viable(self) -> bool: + return self.why_not_viable() is None def is_super(self) -> bool: for tkn in self.body: @@ -320,10 +333,17 @@ def tier_variable(node: parser.InstDef) -> int | None: return int(token.text[-1]) return None -def is_infallible(op: parser.InstDef) -> bool: - return not ( +def has_pop_error(op: parser.InstDef) -> bool: + return ( variable_used(op, "ERROR_IF") - or variable_used(op, "error") + or variable_used(op, "pop_1_error") + or variable_used(op, "exception_unwind") + or variable_used(op, "resume_with_error") + ) + +def has_no_pop_error(op: parser.InstDef) -> bool: + return ( + variable_used(op, "GOTO_ERROR") or variable_used(op, "pop_1_error") or variable_used(op, "exception_unwind") or variable_used(op, "resume_with_error") @@ -507,11 +527,15 @@ def compute_properties(op: parser.InstDef) -> Properties: tkn.column, op.name, ) - infallible = is_infallible(op) + pop_error = has_pop_error(op) + no_pop_error = has_no_pop_error(op) + infallible = not pop_error and not no_pop_error passthrough = stack_effect_only_peeks(op) and infallible return Properties( escapes=makes_escaping_api_call(op), - infallible=infallible, + pop_error=pop_error, + no_pop_error=no_pop_error, + # FIX ME!!! deopts=deopts_if or exits_if, side_exit=exits_if, oparg=variable_used(op, "oparg"), diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 0b4b99c60768b5..e0c0489b566085 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -213,6 +213,8 @@ def cflags(p: Properties) -> str: flags.append("HAS_EXIT_FLAG") if not p.infallible: flags.append("HAS_ERROR_FLAG") + if p.no_pop_error: + flags.append("HAS_NO_POP_ERROR_FLAG") if p.escapes: flags.append("HAS_ESCAPES_FLAG") if p.pure: diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index ab597834a8892f..6df1852d8a46de 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -54,6 +54,7 @@ "PURE", "PASSTHROUGH", "OPARG_AND_1", + "NO_POP_ERROR", ] diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index a82f9076f6b9fe..f5a64d04bd4193 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -188,8 +188,9 @@ def generate_tier2( continue if uop.is_super(): continue - if not uop.is_viable(): - out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 */\n\n") + why_not_viable = uop.why_not_viable() + if why_not_viable is not None: + out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 because it {why_not_viable} */\n\n") continue out.emit(f"case {uop.name}: {{\n") declare_variables(uop, out) From aecdfc2172a81828482f315ee5b3de2a819d7141 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 14 Mar 2024 11:09:23 +0000 Subject: [PATCH 06/24] Do not allow eval breaker and jumps in same T2 micro-op --- Include/internal/pycore_opcode_metadata.h | 11 +- Include/internal/pycore_uop_metadata.h | 40 +- Python/bytecodes.c | 130 +++--- Python/ceval.c | 61 +-- Python/executor_cases.c.h | 497 ++++++--------------- Python/generated_cases.c.h | 120 ++--- Python/optimizer_cases.c.h | 63 +-- Tools/cases_generator/analyzer.py | 9 +- Tools/cases_generator/generators_common.py | 15 + Tools/cases_generator/tier2_generator.py | 18 +- 10 files changed, 358 insertions(+), 606 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 6d90136a2257ad..fac3071b4d5894 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1020,7 +1020,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, + [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1091,7 +1091,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, + [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1211,16 +1211,9 @@ _PyOpcode_macro_expansion[256] = { [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, 0, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, 0, 0 } } }, [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, - [CALL_BUILTIN_CLASS] = { .nuops = 1, .uops = { { _CALL_BUILTIN_CLASS, 0, 0 } } }, - [CALL_BUILTIN_FAST] = { .nuops = 1, .uops = { { _CALL_BUILTIN_FAST, 0, 0 } } }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, 0, 0 } } }, [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_1, 0, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, 0, 0 } } }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 1, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 } } }, [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, - [CALL_STR_1] = { .nuops = 1, .uops = { { _CALL_STR_1, 0, 0 } } }, - [CALL_TUPLE_1] = { .nuops = 1, .uops = { { _CALL_TUPLE_1, 0, 0 } } }, [CALL_TYPE_1] = { .nuops = 1, .uops = { { _CALL_TYPE_1, 0, 0 } } }, [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { _CHECK_EG_MATCH, 0, 0 } } }, [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { _CHECK_EXC_MATCH, 0, 0 } } }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 73b8ffc15b3eeb..e6f054cde58575 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -21,7 +21,7 @@ extern int _PyUop_Popped(int opcode, int oparg); const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_NOP] = HAS_PURE_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, - [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, + [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, @@ -108,7 +108,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_BUILTINS_VERSION] = HAS_DEOPT_FLAG, [_LOAD_GLOBAL_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_LOAD_GLOBAL_BUILTINS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, + [_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -199,14 +199,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_PUSH_FRAME] = 0, [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_CALL_STR_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, - [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, @@ -236,10 +229,10 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_START_EXECUTOR] = 0, [_FATAL_ERROR] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, - [_DEOPT] = HAS_DEOPT_FLAG, - [_SIDE_EXIT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_ERROR_0] = HAS_ERROR_FLAG, - [_ERROR_N] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_DEOPT] = 0, + [_SIDE_EXIT] = 0, + [_ERROR_0] = 0, + [_ERROR_N] = HAS_ARG_FLAG, }; const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { @@ -269,15 +262,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BUILD_SLICE] = "_BUILD_SLICE", [_BUILD_STRING] = "_BUILD_STRING", [_BUILD_TUPLE] = "_BUILD_TUPLE", - [_CALL_BUILTIN_CLASS] = "_CALL_BUILTIN_CLASS", - [_CALL_BUILTIN_FAST] = "_CALL_BUILTIN_FAST", - [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = "_CALL_BUILTIN_FAST_WITH_KEYWORDS", [_CALL_INTRINSIC_1] = "_CALL_INTRINSIC_1", [_CALL_INTRINSIC_2] = "_CALL_INTRINSIC_2", - [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - [_CALL_STR_1] = "_CALL_STR_1", - [_CALL_TUPLE_1] = "_CALL_TUPLE_1", [_CALL_TYPE_1] = "_CALL_TYPE_1", [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", @@ -834,22 +820,8 @@ int _PyUop_Popped(int opcode, int oparg) return 1; case _CALL_TYPE_1: return 3; - case _CALL_STR_1: - return 3; - case _CALL_TUPLE_1: - return 3; case _EXIT_INIT_CHECK: return 1; - case _CALL_BUILTIN_CLASS: - return 2 + oparg; - case _CALL_BUILTIN_FAST: - return 2 + oparg; - case _CALL_BUILTIN_FAST_WITH_KEYWORDS: - return 2 + oparg; - case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: - return 2 + oparg; - case _CALL_METHOD_DESCRIPTOR_FAST: - return 2 + oparg; case _MAKE_FUNCTION: return 1; case _SET_FUNCTION_ATTRIBUTE: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 35664fc3708187..1ee09cf4b2000f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -179,7 +179,7 @@ dummy_func( uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; if (code_version != global_version) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { - GOTO_ERROR(error); + NO_POP_ERROR(); } next_instr = this_instr; } @@ -206,7 +206,13 @@ dummy_func( inst(LOAD_FAST_CHECK, (-- value)) { value = GETLOCAL(oparg); - ERROR_IF(value == NULL, unbound_local_error); + if (value == NULL) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + ERROR_IF(1, error); + } Py_INCREF(value); } @@ -275,7 +281,7 @@ dummy_func( if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); + NO_POP_ERROR(); } PyErr_SetRaisedException(NULL); } @@ -290,7 +296,7 @@ dummy_func( if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); + NO_POP_ERROR(); } PyErr_SetRaisedException(NULL); } @@ -826,7 +832,7 @@ dummy_func( int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); - if (err) GOTO_ERROR(error); + if (err) NO_POP_ERROR(); STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -850,7 +856,7 @@ dummy_func( int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); - if (err) GOTO_ERROR(error); + if (err) NO_POP_ERROR(); Py_INCREF(retval); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -906,7 +912,7 @@ dummy_func( if (PyAsyncGen_CheckExact(aiter)) { awaitable = type->tp_as_async->am_anext(aiter); if (awaitable == NULL) { - GOTO_ERROR(error); + NO_POP_ERROR(); } } else { if (type->tp_as_async != NULL){ @@ -916,7 +922,7 @@ dummy_func( if (getter != NULL) { next_iter = (*getter)(aiter); if (next_iter == NULL) { - GOTO_ERROR(error); + NO_POP_ERROR(); } } else { @@ -924,7 +930,7 @@ dummy_func( "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); - GOTO_ERROR(error); + NO_POP_ERROR(); } awaitable = _PyCoro_GetAwaitableIter(next_iter); @@ -936,7 +942,7 @@ dummy_func( Py_TYPE(next_iter)->tp_name); Py_DECREF(next_iter); - GOTO_ERROR(error); + NO_POP_ERROR(); } else { Py_DECREF(next_iter); } @@ -1018,7 +1024,7 @@ dummy_func( JUMPBY(oparg); } else { - GOTO_ERROR(error); + NO_POP_ERROR(); } } Py_DECREF(v); @@ -1054,7 +1060,7 @@ dummy_func( int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, this_instr, retval); - if (err) GOTO_ERROR(error); + if (err) NO_POP_ERROR(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -1108,7 +1114,7 @@ dummy_func( else { assert(PyLong_Check(lasti)); _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - GOTO_ERROR(error); + NO_POP_ERROR(); } } assert(exc && PyExceptionInstance_Check(exc)); @@ -1184,7 +1190,7 @@ dummy_func( if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - GOTO_ERROR(error); + NO_POP_ERROR(); } err = PyObject_DelItem(ns, name); // Can't use ERROR_IF here. @@ -1192,7 +1198,7 @@ dummy_func( _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + NO_POP_ERROR(); } } @@ -1312,12 +1318,12 @@ dummy_func( int err = PyDict_Pop(GLOBALS(), name, NULL); // Can't use ERROR_IF here. if (err < 0) { - GOTO_ERROR(error); + NO_POP_ERROR(); } if (err == 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + NO_POP_ERROR(); } } @@ -1334,21 +1340,21 @@ dummy_func( inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); + NO_POP_ERROR(); } if (v == NULL) { if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); + NO_POP_ERROR(); } if (v == NULL) { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); + NO_POP_ERROR(); } if (v == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + NO_POP_ERROR(); } } } @@ -1364,21 +1370,21 @@ dummy_func( } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); + NO_POP_ERROR(); } if (v == NULL) { if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); + NO_POP_ERROR(); } if (v == NULL) { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); + NO_POP_ERROR(); } if (v == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + NO_POP_ERROR(); } } } @@ -1494,7 +1500,13 @@ dummy_func( inst(DELETE_FAST, (--)) { PyObject *v = GETLOCAL(oparg); - ERROR_IF(v == NULL, unbound_local_error); + if (v == NULL) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + ERROR_IF(1, error); + } SETLOCAL(oparg, NULL); } @@ -1504,7 +1516,7 @@ dummy_func( PyObject *initial = GETLOCAL(oparg); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - GOTO_ERROR(error); + NO_POP_ERROR(); } SETLOCAL(oparg, cell); } @@ -1516,7 +1528,7 @@ dummy_func( // Fortunately we don't need its superpower. if (oldobj == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); + NO_POP_ERROR(); } PyCell_SET(cell, NULL); Py_DECREF(oldobj); @@ -1528,14 +1540,14 @@ dummy_func( assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { - GOTO_ERROR(error); + NO_POP_ERROR(); } if (!value) { PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); + NO_POP_ERROR(); } Py_INCREF(value); } @@ -1615,7 +1627,7 @@ dummy_func( inst(BUILD_SET, (values[oparg] -- set)) { set = PySet_New(NULL); if (set == NULL) - GOTO_ERROR(error); + NO_POP_ERROR(); int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; @@ -2499,7 +2511,7 @@ dummy_func( _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); - GOTO_ERROR(error); + NO_POP_ERROR(); } iter = iterable; } @@ -2510,7 +2522,7 @@ dummy_func( /* `iterable` is not a generator. */ iter = PyObject_GetIter(iterable); if (iter == NULL) { - GOTO_ERROR(error); + NO_POP_ERROR(); } DECREF_INPUTS(); } @@ -2547,7 +2559,7 @@ dummy_func( if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); + NO_POP_ERROR(); } monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); @@ -2570,7 +2582,7 @@ dummy_func( if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); + NO_POP_ERROR(); } _PyErr_Clear(tstate); } @@ -2596,7 +2608,7 @@ dummy_func( else { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); + NO_POP_ERROR(); } monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); @@ -2776,7 +2788,7 @@ dummy_func( "asynchronous context manager protocol", Py_TYPE(mgr)->tp_name); } - GOTO_ERROR(error); + NO_POP_ERROR(); } exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); if (exit == NULL) { @@ -2788,7 +2800,7 @@ dummy_func( Py_TYPE(mgr)->tp_name); } Py_DECREF(enter); - GOTO_ERROR(error); + NO_POP_ERROR(); } DECREF_INPUTS(); res = PyObject_CallNoArgs(enter); @@ -2811,7 +2823,7 @@ dummy_func( "context manager protocol", Py_TYPE(mgr)->tp_name); } - GOTO_ERROR(error); + NO_POP_ERROR(); } exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); if (exit == NULL) { @@ -2823,7 +2835,7 @@ dummy_func( Py_TYPE(mgr)->tp_name); } Py_DECREF(enter); - GOTO_ERROR(error); + NO_POP_ERROR(); } DECREF_INPUTS(); res = PyObject_CallNoArgs(enter); @@ -3072,7 +3084,7 @@ dummy_func( // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - GOTO_ERROR(error); + NO_POP_ERROR(); } frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); @@ -3284,7 +3296,7 @@ dummy_func( STAT_INC(CALL, hit); PyObject *self = _PyType_NewManagedObject(tp); if (self == NULL) { - GOTO_ERROR(error); + NO_POP_ERROR(); } Py_DECREF(tp); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( @@ -3321,7 +3333,7 @@ dummy_func( PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(should_be_none)->tp_name); - GOTO_ERROR(error); + NO_POP_ERROR(); } } @@ -3360,7 +3372,7 @@ dummy_func( // This is slower but CPython promises to check all non-vectorcall // function calls. if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + NO_POP_ERROR(); } PyObject *arg = args[0]; res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); @@ -3445,7 +3457,7 @@ dummy_func( PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { - GOTO_ERROR(error); + NO_POP_ERROR(); } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3470,7 +3482,7 @@ dummy_func( PyObject *inst = args[0]; int retval = PyObject_IsInstance(inst, cls); if (retval < 0) { - GOTO_ERROR(error); + NO_POP_ERROR(); } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3520,7 +3532,7 @@ dummy_func( // This is slower but CPython promises to check all non-vectorcall // function calls. if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + NO_POP_ERROR(); } res = _PyCFunction_TrampolineCall(cfunc, self, arg); _Py_LeaveRecursiveCallTstate(tstate); @@ -3580,7 +3592,7 @@ dummy_func( // This is slower but CPython promises to check all non-vectorcall // function calls. if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + NO_POP_ERROR(); } res = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); @@ -3667,7 +3679,7 @@ dummy_func( // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - GOTO_ERROR(error); + NO_POP_ERROR(); } assert(next_instr - this_instr == 1); frame->return_offset = 1; @@ -3715,11 +3727,11 @@ dummy_func( assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (!PyTuple_CheckExact(callargs)) { if (check_args_iterable(tstate, func, callargs) < 0) { - GOTO_ERROR(error); + NO_POP_ERROR(); } PyObject *tuple = PySequence_Tuple(callargs); if (tuple == NULL) { - GOTO_ERROR(error); + NO_POP_ERROR(); } Py_SETREF(callargs, tuple); } @@ -3733,7 +3745,7 @@ dummy_func( int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); - if (err) GOTO_ERROR(error); + if (err) NO_POP_ERROR(); result = PyObject_Call(func, callargs, kwargs); if (result == NULL) { _Py_call_instrumentation_exc2( @@ -3764,7 +3776,7 @@ dummy_func( // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { - GOTO_ERROR(error); + NO_POP_ERROR(); } assert(next_instr - this_instr == 1); frame->return_offset = 1; @@ -3785,7 +3797,7 @@ dummy_func( Py_DECREF(codeobj); if (func_obj == NULL) { - GOTO_ERROR(error); + NO_POP_ERROR(); } _PyFunction_SetVersion( @@ -3825,7 +3837,7 @@ dummy_func( PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); if (gen == NULL) { - GOTO_ERROR(error); + NO_POP_ERROR(); } assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4154,21 +4166,21 @@ dummy_func( } tier2 op(_DEOPT, (--)) { - DEOPT_IF(1); + EXIT_TO_TIER1(); } tier2 op(_SIDE_EXIT, (--)) { - EXIT_IF(1); + EXIT_TO_TRACE(); } tier2 op(_ERROR_0, (--)) { - ERROR_IF(1, error); + GOTO_UNWIND(); } tier2 op(_ERROR_N, (values[oparg] --)) { (void)values; SYNC_SP(); - ERROR_IF(1, error); + GOTO_UNWIND(); } // END BYTECODES // diff --git a/Python/ceval.c b/Python/ceval.c index 6e9d05e6d8e1af..fd35e0bb82e23e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -642,7 +642,6 @@ int _Py_CheckRecursiveCallPy( return 0; } - static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { /* Put a NOP at the start, so that the IP points into * the code, rather than before it */ @@ -850,15 +849,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int or goto error. */ Py_UNREACHABLE(); -unbound_local_error: - { - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - goto error; - } - pop_4_error: STACK_SHRINK(1); pop_3_error: @@ -981,8 +971,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #define GOTO_ERROR(LABEL) goto LABEL ## _tier_two #undef DEOPT_IF -#define DEOPTIMIZE goto deoptimize -#define JUMP_TO_ERROR goto error_tier_two +#define JUMP_TO_JUMP_TARGET goto jump_to_jump_target +#define JUMP_TO_ERROR goto jump_to_error_target +#define NO_POP_ERROR() goto jump_to_error_target +#define GOTO_UNWIND() goto error_tier_two +#define EXIT_TO_TRACE() goto exit_to_trace +#define EXIT_TO_TIER1() goto exit_to_tier1 #ifdef Py_STATS // Disable these macros that apply to Tier 1 stats when we are in Tier 2 @@ -1048,15 +1042,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } } -// Jump here from ERROR_IF(..., unbound_local_error) -unbound_local_error_tier_two: - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - goto error_tier_two; - -error_tier_two: +jump_to_error_target: #ifdef Py_DEBUG if (lltrace >= 2) { printf("Error: [UOp "); @@ -1066,12 +1052,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _PyOpcode_OpName[frame->instr_ptr->op.code]); } #endif + assert (next_uop[-1].format == UOP_FORMAT_JUMP); + uint16_t target = uop_get_error_target(&next_uop[-1]); + next_uop = current_executor->trace + target; + goto tier2_dispatch; + +error_tier_two: OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (next_uop[-1].format == UOP_FORMAT_JUMP) { - uint16_t target = uop_get_error_target(&next_uop[-1]); - next_uop = current_executor->trace + target; - goto tier2_dispatch; - } assert(next_uop[-1].format == UOP_FORMAT_TARGET); frame->return_offset = 0; // Don't leave this random _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1079,13 +1066,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int tstate->previous_executor = NULL; goto resume_with_error; -// Jump here from DEOPT_IF() -deoptimize: - if (next_uop[-1].format == UOP_FORMAT_JUMP) { - uint16_t target = uop_get_jump_target(&next_uop[-1]); - next_uop = current_executor->trace + target; - goto tier2_dispatch; - } +jump_to_jump_target: + assert(next_uop[-1].format == UOP_FORMAT_JUMP); + target = uop_get_jump_target(&next_uop[-1]); + next_uop = current_executor->trace + target; + goto tier2_dispatch; + +exit_to_tier1: assert(next_uop[-1].format == UOP_FORMAT_TARGET); next_instr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); #ifdef Py_DEBUG @@ -1102,13 +1089,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int tstate->previous_executor = NULL; DISPATCH(); -// Jump here from EXIT_IF() -side_exit: - if (next_uop[-1].format == UOP_FORMAT_JUMP) { - uint16_t target = uop_get_jump_target(&next_uop[-1]); - next_uop = current_executor->trace + target; - goto tier2_dispatch; - } +exit_to_trace: assert(next_uop[-1].format == UOP_FORMAT_EXIT); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); UOP_STAT_INC(uopcode, miss); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d3eeabc537ff66..e11c6d118cbe0e 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -14,13 +14,13 @@ case _RESUME_CHECK: { #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) DEOPTIMIZE; + if (_Py_emscripten_signal_clock == 0) JUMP_TO_JUMP_TARGET; _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) DEOPTIMIZE; + if (eval_breaker != version) JUMP_TO_JUMP_TARGET; break; } @@ -30,7 +30,13 @@ PyObject *value; oparg = CURRENT_OPARG(); value = GETLOCAL(oparg); - if (value == NULL) JUMP_TO_ERROR; + if (value == NULL) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + if (1) JUMP_TO_ERROR; + } Py_INCREF(value); stack_pointer[0] = value; stack_pointer += 1; @@ -317,7 +323,7 @@ case _TO_BOOL_BOOL: { PyObject *value; value = stack_pointer[-1]; - if (!PyBool_Check(value)) goto side_exit; + if (!PyBool_Check(value)) JUMP_TO_JUMP_TARGET; STAT_INC(TO_BOOL, hit); break; } @@ -326,7 +332,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyLong_CheckExact(value)) goto side_exit; + if (!PyLong_CheckExact(value)) JUMP_TO_JUMP_TARGET; STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { assert(_Py_IsImmortal(value)); @@ -344,7 +350,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyList_CheckExact(value)) goto side_exit; + if (!PyList_CheckExact(value)) JUMP_TO_JUMP_TARGET; STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; Py_DECREF(value); @@ -357,7 +363,7 @@ PyObject *res; value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - if (!Py_IsNone(value)) goto side_exit; + if (!Py_IsNone(value)) JUMP_TO_JUMP_TARGET; STAT_INC(TO_BOOL, hit); res = Py_False; stack_pointer[-1] = res; @@ -368,7 +374,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyUnicode_CheckExact(value)) goto side_exit; + if (!PyUnicode_CheckExact(value)) JUMP_TO_JUMP_TARGET; STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { assert(_Py_IsImmortal(value)); @@ -409,8 +415,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyLong_CheckExact(left)) goto side_exit; - if (!PyLong_CheckExact(right)) goto side_exit; + if (!PyLong_CheckExact(left)) JUMP_TO_JUMP_TARGET; + if (!PyLong_CheckExact(right)) JUMP_TO_JUMP_TARGET; break; } @@ -467,8 +473,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyFloat_CheckExact(left)) goto side_exit; - if (!PyFloat_CheckExact(right)) goto side_exit; + if (!PyFloat_CheckExact(left)) JUMP_TO_JUMP_TARGET; + if (!PyFloat_CheckExact(right)) JUMP_TO_JUMP_TARGET; break; } @@ -525,8 +531,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyUnicode_CheckExact(left)) goto side_exit; - if (!PyUnicode_CheckExact(right)) goto side_exit; + if (!PyUnicode_CheckExact(left)) JUMP_TO_JUMP_TARGET; + if (!PyUnicode_CheckExact(right)) JUMP_TO_JUMP_TARGET; break; } @@ -617,12 +623,12 @@ PyObject *res; sub = stack_pointer[-1]; list = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) DEOPTIMIZE; - if (!PyList_CheckExact(list)) DEOPTIMIZE; + if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET; + if (!PyList_CheckExact(list)) JUMP_TO_JUMP_TARGET; // Deopt unless 0 <= sub < PyList_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) DEOPTIMIZE; + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET; Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyList_GET_SIZE(list)) DEOPTIMIZE; + if (index >= PyList_GET_SIZE(list)) JUMP_TO_JUMP_TARGET; STAT_INC(BINARY_SUBSCR, hit); res = PyList_GET_ITEM(list, index); assert(res != NULL); @@ -640,14 +646,14 @@ PyObject *res; sub = stack_pointer[-1]; str = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) DEOPTIMIZE; - if (!PyUnicode_CheckExact(str)) DEOPTIMIZE; - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) DEOPTIMIZE; + if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET; + if (!PyUnicode_CheckExact(str)) JUMP_TO_JUMP_TARGET; + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET; Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) DEOPTIMIZE; + if (PyUnicode_GET_LENGTH(str) <= index) JUMP_TO_JUMP_TARGET; // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) DEOPTIMIZE; + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) JUMP_TO_JUMP_TARGET; STAT_INC(BINARY_SUBSCR, hit); res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); @@ -663,12 +669,12 @@ PyObject *res; sub = stack_pointer[-1]; tuple = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) DEOPTIMIZE; - if (!PyTuple_CheckExact(tuple)) DEOPTIMIZE; + if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET; + if (!PyTuple_CheckExact(tuple)) JUMP_TO_JUMP_TARGET; // Deopt unless 0 <= sub < PyTuple_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) DEOPTIMIZE; + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET; Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) DEOPTIMIZE; + if (index >= PyTuple_GET_SIZE(tuple)) JUMP_TO_JUMP_TARGET; STAT_INC(BINARY_SUBSCR, hit); res = PyTuple_GET_ITEM(tuple, index); assert(res != NULL); @@ -686,7 +692,7 @@ PyObject *res; sub = stack_pointer[-1]; dict = stack_pointer[-2]; - if (!PyDict_CheckExact(dict)) DEOPTIMIZE; + if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET; STAT_INC(BINARY_SUBSCR, hit); int rc = PyDict_GetItemRef(dict, sub, &res); if (rc == 0) { @@ -751,13 +757,13 @@ sub = stack_pointer[-1]; list = stack_pointer[-2]; value = stack_pointer[-3]; - if (!PyLong_CheckExact(sub)) DEOPTIMIZE; - if (!PyList_CheckExact(list)) DEOPTIMIZE; + if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET; + if (!PyList_CheckExact(list)) JUMP_TO_JUMP_TARGET; // Ensure nonnegative, zero-or-one-digit ints. - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) DEOPTIMIZE; + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET; Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; // Ensure index < len(list) - if (index >= PyList_GET_SIZE(list)) DEOPTIMIZE; + if (index >= PyList_GET_SIZE(list)) JUMP_TO_JUMP_TARGET; STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); @@ -776,7 +782,7 @@ sub = stack_pointer[-1]; dict = stack_pointer[-2]; value = stack_pointer[-3]; - if (!PyDict_CheckExact(dict)) DEOPTIMIZE; + if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET; STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); @@ -902,7 +908,7 @@ if (PyAsyncGen_CheckExact(aiter)) { awaitable = type->tp_as_async->am_anext(aiter); if (awaitable == NULL) { - GOTO_ERROR(error); + JUMP_TO_ERROR; } } else { if (type->tp_as_async != NULL){ @@ -911,7 +917,7 @@ if (getter != NULL) { next_iter = (*getter)(aiter); if (next_iter == NULL) { - GOTO_ERROR(error); + JUMP_TO_ERROR; } } else { @@ -919,7 +925,7 @@ "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); - GOTO_ERROR(error); + JUMP_TO_ERROR; } awaitable = _PyCoro_GetAwaitableIter(next_iter); if (awaitable == NULL) { @@ -929,7 +935,7 @@ "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); Py_DECREF(next_iter); - GOTO_ERROR(error); + JUMP_TO_ERROR; } else { Py_DECREF(next_iter); } @@ -1034,7 +1040,7 @@ if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - GOTO_ERROR(error); + JUMP_TO_ERROR; } err = PyObject_DelItem(ns, name); // Can't use ERROR_IF here. @@ -1042,7 +1048,7 @@ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + JUMP_TO_ERROR; } break; } @@ -1066,8 +1072,8 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; assert(oparg == 2); - if (!PyTuple_CheckExact(seq)) DEOPTIMIZE; - if (PyTuple_GET_SIZE(seq) != 2) DEOPTIMIZE; + if (!PyTuple_CheckExact(seq)) JUMP_TO_JUMP_TARGET; + if (PyTuple_GET_SIZE(seq) != 2) JUMP_TO_JUMP_TARGET; STAT_INC(UNPACK_SEQUENCE, hit); val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); @@ -1084,8 +1090,8 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; values = &stack_pointer[-1]; - if (!PyTuple_CheckExact(seq)) DEOPTIMIZE; - if (PyTuple_GET_SIZE(seq) != oparg) DEOPTIMIZE; + if (!PyTuple_CheckExact(seq)) JUMP_TO_JUMP_TARGET; + if (PyTuple_GET_SIZE(seq) != oparg) JUMP_TO_JUMP_TARGET; STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -1102,8 +1108,8 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; values = &stack_pointer[-1]; - if (!PyList_CheckExact(seq)) DEOPTIMIZE; - if (PyList_GET_SIZE(seq) != oparg) DEOPTIMIZE; + if (!PyList_CheckExact(seq)) JUMP_TO_JUMP_TARGET; + if (PyList_GET_SIZE(seq) != oparg) JUMP_TO_JUMP_TARGET; STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -1172,12 +1178,12 @@ int err = PyDict_Pop(GLOBALS(), name, NULL); // Can't use ERROR_IF here. if (err < 0) { - GOTO_ERROR(error); + JUMP_TO_ERROR; } if (err == 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + JUMP_TO_ERROR; } break; } @@ -1203,21 +1209,21 @@ mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); + JUMP_TO_ERROR; } if (v == NULL) { if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); + JUMP_TO_ERROR; } if (v == NULL) { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); + JUMP_TO_ERROR; } if (v == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + JUMP_TO_ERROR; } } } @@ -1275,8 +1281,8 @@ case _GUARD_GLOBALS_VERSION: { uint16_t version = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) DEOPTIMIZE; - if (dict->ma_keys->dk_version != version) DEOPTIMIZE; + if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET; + if (dict->ma_keys->dk_version != version) JUMP_TO_JUMP_TARGET; assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1284,8 +1290,8 @@ case _GUARD_BUILTINS_VERSION: { uint16_t version = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)BUILTINS(); - if (!PyDict_CheckExact(dict)) DEOPTIMIZE; - if (dict->ma_keys->dk_version != version) DEOPTIMIZE; + if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET; + if (dict->ma_keys->dk_version != version) JUMP_TO_JUMP_TARGET; assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1298,7 +1304,7 @@ PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; - if (res == NULL) DEOPTIMIZE; + if (res == NULL) JUMP_TO_JUMP_TARGET; Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1316,7 +1322,7 @@ PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; - if (res == NULL) DEOPTIMIZE; + if (res == NULL) JUMP_TO_JUMP_TARGET; Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1329,7 +1335,13 @@ case _DELETE_FAST: { oparg = CURRENT_OPARG(); PyObject *v = GETLOCAL(oparg); - if (v == NULL) JUMP_TO_ERROR; + if (v == NULL) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + if (1) JUMP_TO_ERROR; + } SETLOCAL(oparg, NULL); break; } @@ -1341,7 +1353,7 @@ PyObject *initial = GETLOCAL(oparg); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - GOTO_ERROR(error); + JUMP_TO_ERROR; } SETLOCAL(oparg, cell); break; @@ -1355,7 +1367,7 @@ // Fortunately we don't need its superpower. if (oldobj == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); + JUMP_TO_ERROR; } PyCell_SET(cell, NULL); Py_DECREF(oldobj); @@ -1372,14 +1384,14 @@ assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { - GOTO_ERROR(error); + JUMP_TO_ERROR; } if (!value) { PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); + JUMP_TO_ERROR; } Py_INCREF(value); } @@ -1639,8 +1651,8 @@ class = stack_pointer[-2]; global_super = stack_pointer[-3]; assert(!(oparg & 1)); - if (global_super != (PyObject *)&PySuper_Type) DEOPTIMIZE; - if (!PyType_Check(class)) DEOPTIMIZE; + if (global_super != (PyObject *)&PySuper_Type) JUMP_TO_JUMP_TARGET; + if (!PyType_Check(class)) JUMP_TO_JUMP_TARGET; STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); @@ -1664,8 +1676,8 @@ class = stack_pointer[-2]; global_super = stack_pointer[-3]; assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) DEOPTIMIZE; - if (!PyType_Check(class)) DEOPTIMIZE; + if (global_super != (PyObject *)&PySuper_Type) JUMP_TO_JUMP_TARGET; + if (!PyType_Check(class)) JUMP_TO_JUMP_TARGET; STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -1738,7 +1750,7 @@ uint32_t type_version = (uint32_t)CURRENT_OPERAND(); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - if (tp->tp_version_tag != type_version) goto side_exit; + if (tp->tp_version_tag != type_version) JUMP_TO_JUMP_TARGET; break; } @@ -1748,7 +1760,7 @@ assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) DEOPTIMIZE; + if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET; break; } @@ -1761,7 +1773,7 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - if (attr == NULL) DEOPTIMIZE; + if (attr == NULL) JUMP_TO_JUMP_TARGET; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1779,7 +1791,7 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - if (attr == NULL) DEOPTIMIZE; + if (attr == NULL) JUMP_TO_JUMP_TARGET; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1796,10 +1808,10 @@ PyObject *owner; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)CURRENT_OPERAND(); - if (!PyModule_CheckExact(owner)) DEOPTIMIZE; + if (!PyModule_CheckExact(owner)) JUMP_TO_JUMP_TARGET; PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - if (dict->ma_keys->dk_version != dict_version) DEOPTIMIZE; + if (dict->ma_keys->dk_version != dict_version) JUMP_TO_JUMP_TARGET; break; } @@ -1815,7 +1827,7 @@ assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; attr = ep->me_value; - if (attr == NULL) DEOPTIMIZE; + if (attr == NULL) JUMP_TO_JUMP_TARGET; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1831,9 +1843,9 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - if (_PyDictOrValues_IsValues(dorv)) DEOPTIMIZE; + if (_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET; PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - if (dict == NULL) DEOPTIMIZE; + if (dict == NULL) JUMP_TO_JUMP_TARGET; assert(PyDict_CheckExact((PyObject *)dict)); break; } @@ -1847,19 +1859,19 @@ uint16_t hint = (uint16_t)CURRENT_OPERAND(); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - if (hint >= (size_t)dict->ma_keys->dk_nentries) DEOPTIMIZE; + if (hint >= (size_t)dict->ma_keys->dk_nentries) JUMP_TO_JUMP_TARGET; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) DEOPTIMIZE; + if (ep->me_key != name) JUMP_TO_JUMP_TARGET; attr = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) DEOPTIMIZE; + if (ep->me_key != name) JUMP_TO_JUMP_TARGET; attr = ep->me_value; } - if (attr == NULL) DEOPTIMIZE; + if (attr == NULL) JUMP_TO_JUMP_TARGET; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1879,7 +1891,7 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); char *addr = (char *)owner + index; attr = *(PyObject **)addr; - if (attr == NULL) DEOPTIMIZE; + if (attr == NULL) JUMP_TO_JUMP_TARGET; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1897,7 +1909,7 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); char *addr = (char *)owner + index; attr = *(PyObject **)addr; - if (attr == NULL) DEOPTIMIZE; + if (attr == NULL) JUMP_TO_JUMP_TARGET; STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1914,9 +1926,9 @@ PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)CURRENT_OPERAND(); - if (!PyType_Check(owner)) DEOPTIMIZE; + if (!PyType_Check(owner)) JUMP_TO_JUMP_TARGET; assert(type_version != 0); - if (((PyTypeObject *)owner)->tp_version_tag != type_version) DEOPTIMIZE; + if (((PyTypeObject *)owner)->tp_version_tag != type_version) JUMP_TO_JUMP_TARGET; break; } @@ -1965,7 +1977,7 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(dorv)) DEOPTIMIZE; + if (!_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET; break; } @@ -2060,8 +2072,8 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!_PyLong_IsCompact((PyLongObject *)left)) DEOPTIMIZE; - if (!_PyLong_IsCompact((PyLongObject *)right)) DEOPTIMIZE; + if (!_PyLong_IsCompact((PyLongObject *)left)) JUMP_TO_JUMP_TARGET; + if (!_PyLong_IsCompact((PyLongObject *)right)) JUMP_TO_JUMP_TARGET; STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && _PyLong_DigitCount((PyLongObject *)right) <= 1); @@ -2140,7 +2152,7 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))) DEOPTIMIZE; + if (!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))) JUMP_TO_JUMP_TARGET; STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! int res = _PySet_Contains((PySetObject *)right, left); @@ -2160,7 +2172,7 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyDict_CheckExact(right)) DEOPTIMIZE; + if (!PyDict_CheckExact(right)) JUMP_TO_JUMP_TARGET; STAT_INC(CONTAINS_OP, hit); int res = PyDict_Contains(right, left); Py_DECREF(left); @@ -2342,7 +2354,7 @@ _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); - GOTO_ERROR(error); + JUMP_TO_ERROR; } iter = iterable; } @@ -2353,7 +2365,7 @@ /* `iterable` is not a generator. */ iter = PyObject_GetIter(iterable); if (iter == NULL) { - GOTO_ERROR(error); + JUMP_TO_ERROR; } Py_DECREF(iterable); } @@ -2372,7 +2384,7 @@ if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); + JUMP_TO_ERROR; } _PyErr_Clear(tstate); } @@ -2380,7 +2392,7 @@ Py_DECREF(iter); STACK_SHRINK(1); /* The translator sets the deopt target just past END_FOR */ - if (true) DEOPTIMIZE; + if (true) JUMP_TO_JUMP_TARGET; } // Common case: no jump, leave it to the code generator stack_pointer[0] = next; @@ -2393,7 +2405,7 @@ case _ITER_CHECK_LIST: { PyObject *iter; iter = stack_pointer[-1]; - if (Py_TYPE(iter) != &PyListIter_Type) DEOPTIMIZE; + if (Py_TYPE(iter) != &PyListIter_Type) JUMP_TO_JUMP_TARGET; break; } @@ -2405,8 +2417,8 @@ _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; - if (seq == NULL) DEOPTIMIZE; - if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) DEOPTIMIZE; + if (seq == NULL) JUMP_TO_JUMP_TARGET; + if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) JUMP_TO_JUMP_TARGET; break; } @@ -2428,7 +2440,7 @@ case _ITER_CHECK_TUPLE: { PyObject *iter; iter = stack_pointer[-1]; - if (Py_TYPE(iter) != &PyTupleIter_Type) DEOPTIMIZE; + if (Py_TYPE(iter) != &PyTupleIter_Type) JUMP_TO_JUMP_TARGET; break; } @@ -2440,8 +2452,8 @@ _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; - if (seq == NULL) DEOPTIMIZE; - if (it->it_index >= PyTuple_GET_SIZE(seq)) DEOPTIMIZE; + if (seq == NULL) JUMP_TO_JUMP_TARGET; + if (it->it_index >= PyTuple_GET_SIZE(seq)) JUMP_TO_JUMP_TARGET; break; } @@ -2464,7 +2476,7 @@ PyObject *iter; iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - if (Py_TYPE(r) != &PyRangeIter_Type) DEOPTIMIZE; + if (Py_TYPE(r) != &PyRangeIter_Type) JUMP_TO_JUMP_TARGET; break; } @@ -2475,7 +2487,7 @@ iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); - if (r->len <= 0) DEOPTIMIZE; + if (r->len <= 0) JUMP_TO_JUMP_TARGET; break; } @@ -2563,7 +2575,7 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) DEOPTIMIZE; + if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET; break; } @@ -2573,7 +2585,7 @@ uint32_t keys_version = (uint32_t)CURRENT_OPERAND(); PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - if (owner_heap_type->ht_cached_keys->dk_version != keys_version) DEOPTIMIZE; + if (owner_heap_type->ht_cached_keys->dk_version != keys_version) JUMP_TO_JUMP_TARGET; break; } @@ -2655,7 +2667,7 @@ assert(dictoffset > 0); PyObject *dict = *(PyObject **)((char *)owner + dictoffset); /* This object has a __dict__, just not yet created */ - if (dict != NULL) DEOPTIMIZE; + if (dict != NULL) JUMP_TO_JUMP_TARGET; break; } @@ -2688,8 +2700,8 @@ oparg = CURRENT_OPARG(); null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - if (null != NULL) DEOPTIMIZE; - if (Py_TYPE(callable) != &PyMethod_Type) DEOPTIMIZE; + if (null != NULL) JUMP_TO_JUMP_TARGET; + if (Py_TYPE(callable) != &PyMethod_Type) JUMP_TO_JUMP_TARGET; break; } @@ -2711,7 +2723,7 @@ } case _CHECK_PEP_523: { - if (tstate->interp->eval_frame) DEOPTIMIZE; + if (tstate->interp->eval_frame) JUMP_TO_JUMP_TARGET; break; } @@ -2722,11 +2734,11 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND(); - if (!PyFunction_Check(callable)) DEOPTIMIZE; + if (!PyFunction_Check(callable)) JUMP_TO_JUMP_TARGET; PyFunctionObject *func = (PyFunctionObject *)callable; - if (func->func_version != func_version) DEOPTIMIZE; + if (func->func_version != func_version) JUMP_TO_JUMP_TARGET; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (self_or_null != NULL)) DEOPTIMIZE; + if (code->co_argcount != oparg + (self_or_null != NULL)) JUMP_TO_JUMP_TARGET; break; } @@ -2736,8 +2748,8 @@ callable = stack_pointer[-2 - oparg]; PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) DEOPTIMIZE; - if (tstate->py_recursion_remaining <= 1) DEOPTIMIZE; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) JUMP_TO_JUMP_TARGET; + if (tstate->py_recursion_remaining <= 1) JUMP_TO_JUMP_TARGET; break; } @@ -2931,8 +2943,8 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; assert(oparg == 1); - if (null != NULL) DEOPTIMIZE; - if (callable != (PyObject *)&PyType_Type) DEOPTIMIZE; + if (null != NULL) JUMP_TO_JUMP_TARGET; + if (callable != (PyObject *)&PyType_Type) JUMP_TO_JUMP_TARGET; STAT_INC(CALL, hit); res = Py_NewRef(Py_TYPE(arg)); Py_DECREF(arg); @@ -2941,49 +2953,9 @@ break; } - case _CALL_STR_1: { - PyObject *arg; - PyObject *null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; - assert(oparg == 1); - if (null != NULL) DEOPTIMIZE; - if (callable != (PyObject *)&PyUnicode_Type) DEOPTIMIZE; - STAT_INC(CALL, hit); - res = PyObject_Str(arg); - Py_DECREF(arg); - if (res == NULL) JUMP_TO_ERROR; - stack_pointer[-3] = res; - stack_pointer += -2; - CHECK_EVAL_BREAKER(); - break; - } + /* _CALL_STR_1 is not a viable micro-op for tier 2 because it has error handling and eval-breaker check */ - case _CALL_TUPLE_1: { - PyObject *arg; - PyObject *null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; - assert(oparg == 1); - if (null != NULL) DEOPTIMIZE; - if (callable != (PyObject *)&PyTuple_Type) DEOPTIMIZE; - STAT_INC(CALL, hit); - res = PySequence_Tuple(arg); - Py_DECREF(arg); - if (res == NULL) JUMP_TO_ERROR; - stack_pointer[-3] = res; - stack_pointer += -2; - CHECK_EVAL_BREAKER(); - break; - } + /* _CALL_TUPLE_1 is not a viable micro-op for tier 2 because it has error handling and eval-breaker check */ /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ @@ -2995,122 +2967,19 @@ PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(should_be_none)->tp_name); - GOTO_ERROR(error); + JUMP_TO_ERROR; } stack_pointer += -1; break; } - case _CALL_BUILTIN_CLASS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - if (!PyType_Check(callable)) DEOPTIMIZE; - PyTypeObject *tp = (PyTypeObject *)callable; - if (tp->tp_vectorcall == NULL) DEOPTIMIZE; - STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(tp); - if (res == NULL) JUMP_TO_ERROR; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - break; - } + /* _CALL_BUILTIN_CLASS is not a viable micro-op for tier 2 because it has error handling and eval-breaker check */ /* _CALL_BUILTIN_O is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ - case _CALL_BUILTIN_FAST: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL functions, without keywords */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - if (!PyCFunction_CheckExact(callable)) DEOPTIMIZE; - if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) DEOPTIMIZE; - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - /* res = func(self, args, nargs) */ - res = ((PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, - total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR; - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - break; - } + /* _CALL_BUILTIN_FAST is not a viable micro-op for tier 2 because it has error handling and eval-breaker check */ - case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - if (!PyCFunction_CheckExact(callable)) DEOPTIMIZE; - if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) DEOPTIMIZE; - STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - break; - } + /* _CALL_BUILTIN_FAST_WITH_KEYWORDS is not a viable micro-op for tier 2 because it has error handling and eval-breaker check */ /* _CALL_LEN is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ @@ -3118,85 +2987,11 @@ /* _CALL_METHOD_DESCRIPTOR_O is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ - case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) DEOPTIMIZE; - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) DEOPTIMIZE; - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = args[0]; - if (!Py_IS_TYPE(self, d_type)) DEOPTIMIZE; - STAT_INC(CALL, hit); - int nargs = total_args - 1; - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - break; - } + /* _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS is not a viable micro-op for tier 2 because it has error handling and eval-breaker check */ /* _CALL_METHOD_DESCRIPTOR_NOARGS is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ - case _CALL_METHOD_DESCRIPTOR_FAST: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - /* Builtin METH_FASTCALL methods, without keywords */ - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) DEOPTIMIZE; - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_FASTCALL) DEOPTIMIZE; - PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) DEOPTIMIZE; - STAT_INC(CALL, hit); - PyCFunctionFast cfunc = - (PyCFunctionFast)(void(*)(void))meth->ml_meth; - int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - CHECK_EVAL_BREAKER(); - break; - } + /* _CALL_METHOD_DESCRIPTOR_FAST is not a viable micro-op for tier 2 because it has error handling and eval-breaker check */ /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 because it is instrumented */ @@ -3214,7 +3009,7 @@ PyFunction_New(codeobj, GLOBALS()); Py_DECREF(codeobj); if (func_obj == NULL) { - GOTO_ERROR(error); + JUMP_TO_ERROR; } _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); @@ -3384,7 +3179,7 @@ PyObject *flag; flag = stack_pointer[-1]; stack_pointer += -1; - if (!Py_IsTrue(flag)) goto side_exit; + if (!Py_IsTrue(flag)) JUMP_TO_JUMP_TARGET; assert(Py_IsTrue(flag)); break; } @@ -3393,7 +3188,7 @@ PyObject *flag; flag = stack_pointer[-1]; stack_pointer += -1; - if (!Py_IsFalse(flag)) goto side_exit; + if (!Py_IsFalse(flag)) JUMP_TO_JUMP_TARGET; assert(Py_IsFalse(flag)); break; } @@ -3404,7 +3199,7 @@ stack_pointer += -1; if (!Py_IsNone(val)) { Py_DECREF(val); - if (1) goto side_exit; + if (1) JUMP_TO_JUMP_TARGET; } break; } @@ -3413,7 +3208,7 @@ PyObject *val; val = stack_pointer[-1]; stack_pointer += -1; - if (Py_IsNone(val)) goto side_exit; + if (Py_IsNone(val)) JUMP_TO_JUMP_TARGET; Py_DECREF(val); break; } @@ -3444,12 +3239,12 @@ } case _EXIT_TRACE: { - if (1) goto side_exit; + if (1) JUMP_TO_JUMP_TARGET; break; } case _CHECK_VALIDITY: { - if (!current_executor->vm_data.valid) DEOPTIMIZE; + if (!current_executor->vm_data.valid) JUMP_TO_JUMP_TARGET; break; } @@ -3509,7 +3304,7 @@ case _CHECK_FUNCTION: { uint32_t func_version = (uint32_t)CURRENT_OPERAND(); assert(PyFunction_Check(frame->f_funcobj)); - if (((PyFunctionObject *)frame->f_funcobj)->func_version != func_version) DEOPTIMIZE; + if (((PyFunctionObject *)frame->f_funcobj)->func_version != func_version) JUMP_TO_JUMP_TARGET; break; } @@ -3575,23 +3370,23 @@ case _CHECK_VALIDITY_AND_SET_IP: { PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND(); - if (!current_executor->vm_data.valid) DEOPTIMIZE; + if (!current_executor->vm_data.valid) JUMP_TO_JUMP_TARGET; frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; break; } case _DEOPT: { - if (1) DEOPTIMIZE; + EXIT_TO_TIER1(); break; } case _SIDE_EXIT: { - if (1) goto side_exit; + EXIT_TO_TRACE(); break; } case _ERROR_0: { - if (1) JUMP_TO_ERROR; + GOTO_UNWIND(); break; } @@ -3601,7 +3396,7 @@ values = &stack_pointer[-oparg]; (void)values; stack_pointer += -oparg; - if (1) JUMP_TO_ERROR; + GOTO_UNWIND(); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index cb396887b8bbf7..375a45b75761da 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -25,7 +25,7 @@ "asynchronous context manager protocol", Py_TYPE(mgr)->tp_name); } - GOTO_ERROR(error); + goto error; } exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); if (exit == NULL) { @@ -37,7 +37,7 @@ Py_TYPE(mgr)->tp_name); } Py_DECREF(enter); - GOTO_ERROR(error); + goto error; } Py_DECREF(mgr); res = PyObject_CallNoArgs(enter); @@ -71,7 +71,7 @@ "context manager protocol", Py_TYPE(mgr)->tp_name); } - GOTO_ERROR(error); + goto error; } exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); if (exit == NULL) { @@ -83,7 +83,7 @@ Py_TYPE(mgr)->tp_name); } Py_DECREF(enter); - GOTO_ERROR(error); + goto error; } Py_DECREF(mgr); res = PyObject_CallNoArgs(enter); @@ -663,7 +663,7 @@ values = &stack_pointer[-oparg]; set = PySet_New(NULL); if (set == NULL) - GOTO_ERROR(error); + goto error; int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; @@ -804,7 +804,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - GOTO_ERROR(error); + goto error; } frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); @@ -875,7 +875,7 @@ STAT_INC(CALL, hit); PyObject *self = _PyType_NewManagedObject(tp); if (self == NULL) { - GOTO_ERROR(error); + goto error; } Py_DECREF(tp); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( @@ -1157,7 +1157,7 @@ // This is slower but CPython promises to check all non-vectorcall // function calls. if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + goto error; } PyObject *arg = args[0]; res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); @@ -1191,11 +1191,11 @@ assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (!PyTuple_CheckExact(callargs)) { if (check_args_iterable(tstate, func, callargs) < 0) { - GOTO_ERROR(error); + goto error; } PyObject *tuple = PySequence_Tuple(callargs); if (tuple == NULL) { - GOTO_ERROR(error); + goto error; } Py_SETREF(callargs, tuple); } @@ -1209,7 +1209,7 @@ int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); - if (err) GOTO_ERROR(error); + if (err) goto error; result = PyObject_Call(func, callargs, kwargs); if (result == NULL) { _Py_call_instrumentation_exc2( @@ -1239,7 +1239,7 @@ // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { - GOTO_ERROR(error); + goto error; } assert(next_instr - this_instr == 1); frame->return_offset = 1; @@ -1320,7 +1320,7 @@ PyObject *inst = args[0]; int retval = PyObject_IsInstance(inst, cls); if (retval < 0) { - GOTO_ERROR(error); + goto error; } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -1383,7 +1383,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - GOTO_ERROR(error); + goto error; } assert(next_instr - this_instr == 1); frame->return_offset = 1; @@ -1451,7 +1451,7 @@ PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { - GOTO_ERROR(error); + goto error; } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -1614,7 +1614,7 @@ // This is slower but CPython promises to check all non-vectorcall // function calls. if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + goto error; } res = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); @@ -1660,7 +1660,7 @@ // This is slower but CPython promises to check all non-vectorcall // function calls. if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - GOTO_ERROR(error); + goto error; } res = _PyCFunction_TrampolineCall(cfunc, self, arg); _Py_LeaveRecursiveCallTstate(tstate); @@ -2273,7 +2273,7 @@ // Fortunately we don't need its superpower. if (oldobj == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); + goto error; } PyCell_SET(cell, NULL); Py_DECREF(oldobj); @@ -2285,7 +2285,13 @@ next_instr += 1; INSTRUCTION_STATS(DELETE_FAST); PyObject *v = GETLOCAL(oparg); - if (v == NULL) goto unbound_local_error; + if (v == NULL) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + if (1) goto error; + } SETLOCAL(oparg, NULL); DISPATCH(); } @@ -2298,12 +2304,12 @@ int err = PyDict_Pop(GLOBALS(), name, NULL); // Can't use ERROR_IF here. if (err < 0) { - GOTO_ERROR(error); + goto error; } if (err == 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + goto error; } DISPATCH(); } @@ -2318,7 +2324,7 @@ if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - GOTO_ERROR(error); + goto error; } err = PyObject_DelItem(ns, name); // Can't use ERROR_IF here. @@ -2326,7 +2332,7 @@ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + goto error; } DISPATCH(); } @@ -2467,7 +2473,7 @@ PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(should_be_none)->tp_name); - GOTO_ERROR(error); + goto error; } stack_pointer += -1; DISPATCH(); @@ -2554,7 +2560,7 @@ if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); + goto error; } monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); @@ -2785,7 +2791,7 @@ if (PyAsyncGen_CheckExact(aiter)) { awaitable = type->tp_as_async->am_anext(aiter); if (awaitable == NULL) { - GOTO_ERROR(error); + goto error; } } else { if (type->tp_as_async != NULL){ @@ -2794,7 +2800,7 @@ if (getter != NULL) { next_iter = (*getter)(aiter); if (next_iter == NULL) { - GOTO_ERROR(error); + goto error; } } else { @@ -2802,7 +2808,7 @@ "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); - GOTO_ERROR(error); + goto error; } awaitable = _PyCoro_GetAwaitableIter(next_iter); if (awaitable == NULL) { @@ -2812,7 +2818,7 @@ "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); Py_DECREF(next_iter); - GOTO_ERROR(error); + goto error; } else { Py_DECREF(next_iter); } @@ -2900,7 +2906,7 @@ _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); - GOTO_ERROR(error); + goto error; } iter = iterable; } @@ -2911,7 +2917,7 @@ /* `iterable` is not a generator. */ iter = PyObject_GetIter(iterable); if (iter == NULL) { - GOTO_ERROR(error); + goto error; } Py_DECREF(iterable); } @@ -3010,7 +3016,7 @@ if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); + goto error; } PyErr_SetRaisedException(NULL); } @@ -3031,7 +3037,7 @@ if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); + goto error; } PyErr_SetRaisedException(NULL); } @@ -3057,7 +3063,7 @@ else { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); + goto error; } monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); @@ -3212,7 +3218,7 @@ uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; if (code_version != global_version) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { - GOTO_ERROR(error); + goto error; } next_instr = this_instr; } @@ -3243,7 +3249,7 @@ int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); - if (err) GOTO_ERROR(error); + if (err) goto error; Py_INCREF(retval); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3268,7 +3274,7 @@ int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); - if (err) GOTO_ERROR(error); + if (err) goto error; STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3300,7 +3306,7 @@ int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, this_instr, retval); - if (err) GOTO_ERROR(error); + if (err) goto error; tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -4083,7 +4089,13 @@ INSTRUCTION_STATS(LOAD_FAST_CHECK); PyObject *value; value = GETLOCAL(oparg); - if (value == NULL) goto unbound_local_error; + if (value == NULL) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + if (1) goto error; + } Py_INCREF(value); stack_pointer[0] = value; stack_pointer += 1; @@ -4120,14 +4132,14 @@ assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { - GOTO_ERROR(error); + goto error; } if (!value) { PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); + goto error; } Py_INCREF(value); } @@ -4145,21 +4157,21 @@ mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); + goto error; } if (v == NULL) { if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); + goto error; } if (v == NULL) { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); + goto error; } if (v == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + goto error; } } } @@ -4343,21 +4355,21 @@ } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); + goto error; } if (v == NULL) { if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); + goto error; } if (v == NULL) { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); + goto error; } if (v == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + goto error; } } } @@ -4519,7 +4531,7 @@ PyObject *initial = GETLOCAL(oparg); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - GOTO_ERROR(error); + goto error; } SETLOCAL(oparg, cell); DISPATCH(); @@ -4536,7 +4548,7 @@ PyFunction_New(codeobj, GLOBALS()); Py_DECREF(codeobj); if (func_obj == NULL) { - GOTO_ERROR(error); + goto error; } _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); @@ -4855,7 +4867,7 @@ else { assert(PyLong_Check(lasti)); _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - GOTO_ERROR(error); + goto error; } } assert(exc && PyExceptionInstance_Check(exc)); @@ -4962,7 +4974,7 @@ PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); if (gen == NULL) { - GOTO_ERROR(error); + goto error; } assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5071,7 +5083,7 @@ JUMPBY(oparg); } else { - GOTO_ERROR(error); + goto error; } } Py_DECREF(v); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 4c2a644ebe5183..52332e6f219613 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1603,23 +1603,9 @@ break; } - case _CALL_STR_1: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-3] = res; - stack_pointer += -2; - break; - } + /* _CALL_STR_1 is not a viable micro-op for tier 2 */ - case _CALL_TUPLE_1: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-3] = res; - stack_pointer += -2; - break; - } + /* _CALL_TUPLE_1 is not a viable micro-op for tier 2 */ /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 */ @@ -1628,34 +1614,13 @@ break; } - case _CALL_BUILTIN_CLASS: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - break; - } + /* _CALL_BUILTIN_CLASS is not a viable micro-op for tier 2 */ /* _CALL_BUILTIN_O is not a viable micro-op for tier 2 */ - case _CALL_BUILTIN_FAST: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - break; - } + /* _CALL_BUILTIN_FAST is not a viable micro-op for tier 2 */ - case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - break; - } + /* _CALL_BUILTIN_FAST_WITH_KEYWORDS is not a viable micro-op for tier 2 */ /* _CALL_LEN is not a viable micro-op for tier 2 */ @@ -1663,25 +1628,11 @@ /* _CALL_METHOD_DESCRIPTOR_O is not a viable micro-op for tier 2 */ - case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - break; - } + /* _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS is not a viable micro-op for tier 2 */ /* _CALL_METHOD_DESCRIPTOR_NOARGS is not a viable micro-op for tier 2 */ - case _CALL_METHOD_DESCRIPTOR_FAST: { - _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - break; - } + /* _CALL_METHOD_DESCRIPTOR_FAST is not a viable micro-op for tier 2 */ /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 */ diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 3f9a016c731688..d6841e8e2b0835 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -180,6 +180,13 @@ def why_not_viable(self) -> str | None: return "has unused cache entries" if self.properties.pop_error and self.properties.no_pop_error: return "has both popping and not-popping errors" + if self.properties.eval_breaker: + if self.properties.pop_error or self.properties.no_pop_error: + return "has error handling and eval-breaker check" + if self.properties.side_exit: + return "exits and eval-breaker check" + if self.properties.deopts: + return "deopts and eval-breaker check" return None def is_viable(self) -> bool: @@ -343,7 +350,7 @@ def has_pop_error(op: parser.InstDef) -> bool: def has_no_pop_error(op: parser.InstDef) -> bool: return ( - variable_used(op, "GOTO_ERROR") + variable_used(op, "NO_POP_ERROR") or variable_used(op, "pop_1_error") or variable_used(op, "exception_unwind") or variable_used(op, "resume_with_error") diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index e0c0489b566085..679ecd1197b3d5 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -99,6 +99,20 @@ def replace_error( out.emit(close) +def replace_no_pop_error( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) # LPAREN + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + out.emit_at("goto error;", tkn) + + def replace_decrefs( out: CWriter, tkn: Token, @@ -160,6 +174,7 @@ def replace_check_eval_breaker( "EXIT_IF": replace_deopt, "DEOPT_IF": replace_deopt, "ERROR_IF": replace_error, + "NO_POP_ERROR": replace_no_pop_error, "DECREF_INPUTS": replace_decrefs, "CHECK_EVAL_BREAKER": replace_check_eval_breaker, "SYNC_SP": replace_sync_sp, diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index f5a64d04bd4193..57ac730a1abfa6 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -75,6 +75,19 @@ def tier2_replace_error( out.emit(") JUMP_TO_ERROR;\n") +def tier2_replace_no_pop_error( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) # LPAREN + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + out.emit_at("JUMP_TO_ERROR;", tkn) + def tier2_replace_deopt( out: CWriter, tkn: Token, @@ -87,7 +100,7 @@ def tier2_replace_deopt( out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "RPAREN") next(tkn_iter) # Semi colon - out.emit(") DEOPTIMIZE;\n") + out.emit(") JUMP_TO_JUMP_TARGET;\n") def tier2_replace_exit_if( @@ -102,7 +115,7 @@ def tier2_replace_exit_if( out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "RPAREN") next(tkn_iter) # Semi colon - out.emit(") goto side_exit;\n") + out.emit(") JUMP_TO_JUMP_TARGET;\n") def tier2_replace_oparg( @@ -128,6 +141,7 @@ def tier2_replace_oparg( TIER2_REPLACEMENT_FUNCTIONS = REPLACEMENT_FUNCTIONS.copy() TIER2_REPLACEMENT_FUNCTIONS["ERROR_IF"] = tier2_replace_error +TIER2_REPLACEMENT_FUNCTIONS["NO_POP_ERROR"] = tier2_replace_no_pop_error TIER2_REPLACEMENT_FUNCTIONS["DEOPT_IF"] = tier2_replace_deopt TIER2_REPLACEMENT_FUNCTIONS["oparg"] = tier2_replace_oparg TIER2_REPLACEMENT_FUNCTIONS["EXIT_IF"] = tier2_replace_exit_if From d365f5813d5415cd03301333b66114f5e0e5887e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 14 Mar 2024 11:38:02 +0000 Subject: [PATCH 07/24] Add JIT support --- Python/jit.c | 32 +++++++++++++++++++++++--------- Tools/jit/_stencils.py | 6 ++++++ Tools/jit/template.c | 35 ++++++++++++++++++----------------- 3 files changed, 47 insertions(+), 26 deletions(-) diff --git a/Python/jit.c b/Python/jit.c index dae25166b1f106..408c788e6b2e41 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -401,11 +401,13 @@ int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length) { // Loop once to find the total compiled size: - size_t code_size = 0; - size_t data_size = 0; + uint32_t instruction_starts[UOP_MAX_TRACE_LENGTH]; + uint32_t code_size = 0; + uint32_t data_size = 0; for (size_t i = 0; i < length; i++) { _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; const StencilGroup *group = &stencil_groups[instruction->opcode]; + instruction_starts[i] = code_size; code_size += group->code.body_size; data_size += group->data.body_size; } @@ -421,11 +423,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size // Loop again to emit the code: unsigned char *code = memory; unsigned char *data = memory + code_size; - unsigned char *top = code; - if (trace[0].opcode == _START_EXECUTOR) { - // Don't want to execute this more than once: - top += stencil_groups[_START_EXECUTOR].code.body_size; - } + assert(trace[0].opcode == _START_EXECUTOR); for (size_t i = 0; i < length; i++) { _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; const StencilGroup *group = &stencil_groups[instruction->opcode]; @@ -437,8 +435,24 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size patches[HoleValue_EXECUTOR] = (uint64_t)executor; patches[HoleValue_OPARG] = instruction->oparg; patches[HoleValue_OPERAND] = instruction->operand; - patches[HoleValue_TARGET] = instruction->target; - patches[HoleValue_TOP] = (uint64_t)top; + switch (instruction->format) { + case UOP_FORMAT_TARGET: + patches[HoleValue_TARGET] = instruction->target; + break; + case UOP_FORMAT_EXIT: + patches[HoleValue_EXIT_INDEX] = instruction->exit_index; + break; + case UOP_FORMAT_JUMP: + assert(instruction->jump_target < length); + patches[HoleValue_JUMP_TARGET] = (uint64_t)memory + instruction_starts[instruction->jump_target]; + if (instruction->error_target < length) { + patches[HoleValue_ERROR_TARGET] = (uint64_t)memory + instruction_starts[instruction->error_target]; + } + break; + default: + assert(0); + } + patches[HoleValue_TOP] = (uint64_t)memory + instruction_starts[1]; patches[HoleValue_ZERO] = 0; emit(group, patches); code += group->code.body_size; diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 78c566d9c8a7ef..3c4fb953162fe6 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -31,6 +31,12 @@ class HoleValue(enum.Enum): OPERAND = enum.auto() # The current uop's target (exposed as _JIT_TARGET): TARGET = enum.auto() + # The base address of the machine code for the jump target (exposed as _JIT_JUMP_TARGET): + JUMP_TARGET = enum.auto() + # The base address of the machine code for the error jump target (exposed as _JIT_ERROR_TARGET): + ERROR_TARGET = enum.auto() + # The index of the exit to be jumped through (exposed as _JIT_EXIT_INDEX): + EXIT_INDEX = enum.auto() # The base address of the machine code for the first uop (exposed as _JIT_TOP): TOP = enum.auto() # A hardcoded value of zero (used for symbol lookups): diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 504e6c875525ae..42137d9ce244a8 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -31,6 +31,10 @@ } \ } while (0) +#define GOTO_UNWIND() goto error_tier_two +#define EXIT_TO_TRACE() goto exit_to_trace +#define EXIT_TO_TIER1() goto exit_to_tier1 + #undef ENABLE_SPECIALIZATION #define ENABLE_SPECIALIZATION (0) @@ -64,9 +68,17 @@ do { \ TYPE NAME = (TYPE)(uint64_t)&ALIAS; #define PATCH_JUMP(ALIAS) \ +do { \ PyAPI_DATA(void) ALIAS; \ __attribute__((musttail)) \ - return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); + return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); \ +} while (0) + +#undef JUMP_TO_JUMP_TARGET +#define JUMP_TO_JUMP_TARGET PATCH_JUMP(_JIT_JUMP_TARGET) + +#undef JUMP_TO_ERROR +#define JUMP_TO_ERROR PATCH_JUMP(_JIT_ERROR_TARGET) _Py_CODEUNIT * _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate) @@ -79,6 +91,7 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND) PATCH_VALUE(uint32_t, _target, _JIT_TARGET) + PATCH_VALUE(uint16_t, _exit_index, _JIT_EXIT_INDEX) // The actual instruction definitions (only one will be used): if (opcode == _JUMP_TO_TOP) { CHECK_EVAL_BREAKER(); @@ -91,28 +104,16 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * } PATCH_JUMP(_JIT_CONTINUE); // Labels that the instruction implementations expect to exist: -unbound_local_error_tier_two: - _PyEval_FormatExcCheckArg( - tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)); - goto error_tier_two; -pop_4_error_tier_two: - STACK_SHRINK(1); -pop_3_error_tier_two: - STACK_SHRINK(1); -pop_2_error_tier_two: - STACK_SHRINK(1); -pop_1_error_tier_two: - STACK_SHRINK(1); + error_tier_two: tstate->previous_executor = (PyObject *)current_executor; GOTO_TIER_ONE(NULL); -deoptimize: +exit_to_tier1: tstate->previous_executor = (PyObject *)current_executor; GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target); -side_exit: +exit_to_trace: { - _PyExitData *exit = ¤t_executor->exits[_target]; + _PyExitData *exit = ¤t_executor->exits[_exit_index]; Py_INCREF(exit->executor); tstate->previous_executor = (PyObject *)current_executor; GOTO_TIER_TWO(exit->executor); From 97428fae30754b414324171e331d1b7a13136f50 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 14 Mar 2024 14:03:58 +0000 Subject: [PATCH 08/24] Revert unneeded change --- Python/optimizer_analysis.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 235c26b1af1bae..d6f06f29e41f36 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -334,7 +334,7 @@ optimize_to_bool( return 0; } -static bool +static void eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit) { REPLACE_OP(this_instr, _POP_TOP, 0, 0); @@ -342,7 +342,6 @@ eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit) REPLACE_OP((this_instr+1), _EXIT_TRACE, 0, 0); this_instr[1].target = this_instr->target; } - return exit; } /* 1 for success, 0 for not ready, cannot error at the moment. */ From 3c2154f162b48512e999cdd6e601a94774100cf0 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 14 Mar 2024 14:58:21 +0000 Subject: [PATCH 09/24] Add missing return annotation --- Tools/cases_generator/analyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index d6841e8e2b0835..1b6bd81f383eea 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -58,7 +58,7 @@ def from_list(properties: list["Properties"]) -> "Properties": ) @property - def infallible(self): + def infallible(self)->bool: return not self.pop_error and not self.no_pop_error From a74756dfee806f5976b0eb9cab724369898e3fd4 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 14 Mar 2024 14:59:30 +0000 Subject: [PATCH 10/24] Better formatting --- Tools/cases_generator/analyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 1b6bd81f383eea..4c6fc7eca87743 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -58,7 +58,7 @@ def from_list(properties: list["Properties"]) -> "Properties": ) @property - def infallible(self)->bool: + def infallible(self) -> bool: return not self.pop_error and not self.no_pop_error From 817a59095636364509e4795e5077d93c6a5f8c23 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 14 Mar 2024 15:53:16 +0000 Subject: [PATCH 11/24] Fix assert --- Python/jit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/jit.c b/Python/jit.c index 408c788e6b2e41..78990805e435d9 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -423,7 +423,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size // Loop again to emit the code: unsigned char *code = memory; unsigned char *data = memory + code_size; - assert(trace[0].opcode == _START_EXECUTOR); + assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT); for (size_t i = 0; i < length; i++) { _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; const StencilGroup *group = &stencil_groups[instruction->opcode]; From 9d0d03cfcd3f34c1fd9ce642588861be1e947801 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 15 Mar 2024 08:47:55 +0000 Subject: [PATCH 12/24] EXIT_IF does not implies DEOPT_IF --- Include/internal/pycore_opcode_metadata.h | 36 +++++++++++------------ Include/internal/pycore_uop_metadata.h | 28 +++++++++--------- Python/optimizer.c | 3 +- Python/optimizer_analysis.c | 2 +- Tools/cases_generator/analyzer.py | 3 +- 5 files changed, 36 insertions(+), 36 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 39d5abc0a04930..495862360c25f2 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -959,14 +959,14 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1009,9 +1009,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1078,10 +1078,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, @@ -1135,7 +1135,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -1149,12 +1149,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 34feaeb4988327..90eaea40def86b 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -51,22 +51,22 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNARY_NOT] = HAS_PURE_FLAG, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_TO_BOOL_BOOL] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, - [_TO_BOOL_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_TO_BOOL_LIST] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_TO_BOOL_NONE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_TO_BOOL_STR] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_TO_BOOL_BOOL] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, + [_TO_BOOL_INT] = HAS_EXIT_FLAG, + [_TO_BOOL_LIST] = HAS_EXIT_FLAG, + [_TO_BOOL_NONE] = HAS_EXIT_FLAG, + [_TO_BOOL_STR] = HAS_EXIT_FLAG, [_REPLACE_WITH_TRUE] = 0, [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_BOTH_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_INT] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_GUARD_BOTH_FLOAT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_FLOAT] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_BINARY_OP_MULTIPLY_FLOAT] = HAS_PURE_FLAG, [_BINARY_OP_ADD_FLOAT] = HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG, - [_GUARD_BOTH_UNICODE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_UNICODE] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -129,7 +129,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_TYPE_VERSION] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_TYPE_VERSION] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_LOAD_ATTR_INSTANCE_VALUE_0] = HAS_DEOPT_FLAG, [_LOAD_ATTR_INSTANCE_VALUE_1] = HAS_DEOPT_FLAG, @@ -217,14 +217,14 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_COPY] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_GUARD_IS_TRUE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_GUARD_IS_FALSE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_GUARD_IS_NONE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_GUARD_IS_NOT_NONE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG, + [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG, + [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG, + [_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG, [_JUMP_TO_TOP] = HAS_EVAL_BREAK_FLAG, [_SET_IP] = 0, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, - [_EXIT_TRACE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_EXIT_TRACE] = HAS_EXIT_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, diff --git a/Python/optimizer.c b/Python/optimizer.c index 64b30fe433dd92..82ed8e9cb05a69 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -610,9 +610,10 @@ translate_bytecode_to_trace( continue; } else { - if (OPCODE_HAS_DEOPT(opcode)) { + if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { opcode = _PyOpcode_Deopt[opcode]; } + assert(!OPCODE_HAS_EXIT(opcode)); assert(!OPCODE_HAS_DEOPT(opcode)); } } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index d6f06f29e41f36..f3651befdf7560 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -394,7 +394,7 @@ optimize_uops( } if (this_instr != trace + trace_len) { assert (this_instr < trace + trace_len && this_instr > trace); - trace_len = this_instr - trace + 1; + trace_len = (int)(this_instr - trace + 1); } _Py_uop_abstractcontext_fini(ctx); return trace_len; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 4c6fc7eca87743..219d023e01cd76 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -542,8 +542,7 @@ def compute_properties(op: parser.InstDef) -> Properties: escapes=makes_escaping_api_call(op), pop_error=pop_error, no_pop_error=no_pop_error, - # FIX ME!!! - deopts=deopts_if or exits_if, + deopts=deopts_if, side_exit=exits_if, oparg=variable_used(op, "oparg"), jumps=variable_used(op, "JUMPBY"), From 79ec8ef139fb55870a95985eb5e79f768fca9d04 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 15 Mar 2024 09:54:06 +0000 Subject: [PATCH 13/24] Don't overflow the trace buffer --- Include/internal/pycore_optimizer.h | 2 +- Python/optimizer.c | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index fcead4d8714870..44cafe61b75596 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -12,7 +12,7 @@ extern "C" { #include // This is the length of the trace we project initially. -#define UOP_MAX_TRACE_LENGTH 512 +#define UOP_MAX_TRACE_LENGTH 800 #define TRACE_STACK_SIZE 5 diff --git a/Python/optimizer.c b/Python/optimizer.c index 82ed8e9cb05a69..5207d00a9e626b 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -160,7 +160,7 @@ static int init_cold_exit_executor(_PyExecutorObject *executor, int oparg); static int cold_exits_initialized = 0; -static _PyExecutorObject COLD_EXITS[UOP_MAX_TRACE_LENGTH] = { 0 }; +static _PyExecutorObject COLD_EXITS[UOP_MAX_TRACE_LENGTH/4] = { 0 }; static const _PyBloomFilter EMPTY_FILTER = { 0 }; @@ -172,7 +172,7 @@ _Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer) } else if (cold_exits_initialized == 0) { cold_exits_initialized = 1; - for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { + for (int i = 0; i < UOP_MAX_TRACE_LENGTH/4; i++) { if (init_cold_exit_executor(&COLD_EXITS[i], i)) { return NULL; } @@ -571,8 +571,10 @@ translate_bytecode_to_trace( top: // Jump here after _PUSH_FRAME or likely branches for (;;) { target = INSTR_IP(instr, code); - RESERVE_RAW(2, "epilogue"); // Always need space for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE + RESERVE_RAW(2, "_CHECK_VALIDITY_AND_SET_IP"); ADD_TO_TRACE(_CHECK_VALIDITY_AND_SET_IP, 0, (uintptr_t)instr, target); + // Need space for _DEOPT + max_length--; uint32_t opcode = instr->op.code; uint32_t oparg = instr->op.arg; @@ -618,7 +620,7 @@ translate_bytecode_to_trace( } } - if (OPCODE_HAS_DEOPT(opcode) || OPCODE_HAS_EXIT(opcode)) { + if (OPCODE_HAS_EXIT(opcode)) { // Make space for exit code max_length--; } @@ -1052,6 +1054,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil return NULL; } /* Initialize exits */ + assert(exit_count < UOP_MAX_TRACE_LENGTH/4); for (int i = 0; i < exit_count; i++) { executor->exits[i].executor = &COLD_EXITS[i]; executor->exits[i].temperature = 0; @@ -1176,7 +1179,7 @@ uop_optimize( assert(strncmp(_PyOpcode_uop_name[buffer[pc].opcode], _PyOpcode_uop_name[opcode], strlen(_PyOpcode_uop_name[opcode])) == 0); } length = prepare_for_execution(buffer, length); - // assert(length <= UOP_MAX_TRACE_LENGTH); + assert(length <= UOP_MAX_TRACE_LENGTH); _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies); if (executor == NULL) { return -1; From 70b0ff5fde4810928a3d5051e5c76af0a738e0af Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 19 Mar 2024 10:49:18 +0000 Subject: [PATCH 14/24] Make sure COLD_EXIT has correct uop instruction format. --- Include/internal/pycore_uop_metadata.h | 2 +- Python/bytecodes.c | 2 +- Python/executor_cases.c.h | 2 +- Python/optimizer.c | 10 +++++----- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index e3c6d2c9375e07..b78d519271be4d 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -238,7 +238,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG, [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, - [_COLD_EXIT] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_COLD_EXIT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_START_EXECUTOR] = 0, [_FATAL_ERROR] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3e3c648b81ba50..3c466f02b7b88a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4179,7 +4179,7 @@ dummy_func( if (optimized < 0) { Py_DECREF(previous); tstate->previous_executor = Py_None; - ERROR_IF(1, error); + GOTO_UNWIND(); } GOTO_TIER_ONE(target); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 46c55f7432162d..d77b34d93b5dc4 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3725,7 +3725,7 @@ if (optimized < 0) { Py_DECREF(previous); tstate->previous_executor = Py_None; - if (1) JUMP_TO_ERROR; + GOTO_UNWIND(); } GOTO_TIER_ONE(target); } diff --git a/Python/optimizer.c b/Python/optimizer.c index 5207d00a9e626b..d3fda845b4d847 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -988,10 +988,7 @@ sanity_check(_PyExecutorObject *executor) } bool ended = false; uint32_t i = 0; - CHECK(executor->trace[0].opcode == _START_EXECUTOR); - if (executor->trace[1].opcode == _COLD_EXIT) { - return; - } + CHECK(executor->trace[0].opcode == _START_EXECUTOR || executor->trace[0].opcode == _COLD_EXIT); for (; i < executor->code_size; i++) { const _PyUOpInstruction *inst = &executor->trace[i]; uint16_t opcode = inst->opcode; @@ -1016,7 +1013,7 @@ sanity_check(_PyExecutorObject *executor) CHECK(inst->format == UOP_FORMAT_JUMP); CHECK(inst->error_target < executor->code_size); } - if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE) { + if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE || opcode == _COLD_EXIT) { ended = true; i++; break; @@ -1122,6 +1119,9 @@ init_cold_exit_executor(_PyExecutorObject *executor, int oparg) for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { assert(executor->vm_data.bloom.bits[i] == 0); } +#ifdef Py_DEBUG + sanity_check(executor); +#endif #ifdef _Py_JIT executor->jit_code = NULL; executor->jit_size = 0; From c19a18a3644bcf6fd2e43a6d3354e91827d587cd Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 19 Mar 2024 17:46:52 +0000 Subject: [PATCH 15/24] Remove NOPs before interpreting in T2. --- Python/optimizer.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index a93ca57b871fef..d45f90a487b288 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -915,12 +915,26 @@ static void make_exit(_PyUOpInstruction *inst, int opcode, int target) static int prepare_for_execution(_PyUOpInstruction *buffer, int length) { - int next_spare = length; int32_t current_jump = -1; int32_t current_jump_target = -1; int32_t current_error = -1; int32_t current_error_target = -1; int32_t current_popped = -1; + /* Leaving in NOPs slows down the interpreter and messes up the stats */ +#ifndef _Py_JIT + _PyUOpInstruction *copy_to = &buffer[0]; + for (int i = 0; i < length; i++) { + _PyUOpInstruction *inst = &buffer[i]; + if (inst->opcode != _NOP) { + if (copy_to != inst) { + *copy_to = *inst; + } + copy_to++; + } + } + length = copy_to - buffer; +#endif + int next_spare = length; for (int i = 0; i < length; i++) { _PyUOpInstruction *inst = &buffer[i]; int opcode = inst->opcode; From 7ab86e2b116abfdb0e4a3dfff0ffe252f8ca0c87 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 20 Mar 2024 09:14:21 +0000 Subject: [PATCH 16/24] Rename _PyUop_Popped --- Include/internal/pycore_uop_metadata.h | 4 ++-- Python/optimizer.c | 2 +- Tools/cases_generator/uop_metadata_generator.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index b78d519271be4d..3ee76c622a27b1 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -15,7 +15,7 @@ extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1]; extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1]; extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; -extern int _PyUop_Popped(int opcode, int oparg); +extern int _PyUop_num_popped(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { @@ -483,7 +483,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_UNPACK_SEQUENCE_TWO_TUPLE] = "_UNPACK_SEQUENCE_TWO_TUPLE", [_WITH_EXCEPT_START] = "_WITH_EXCEPT_START", }; -int _PyUop_Popped(int opcode, int oparg) +int _PyUop_num_popped(int opcode, int oparg) { switch(opcode) { case _NOP: diff --git a/Python/optimizer.c b/Python/optimizer.c index d45f90a487b288..88c5c3b548aab6 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -952,7 +952,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) } if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) { int popped = (_PyUop_Flags[opcode] & HAS_NO_POP_ERROR_FLAG) ? - 0 : _PyUop_Popped(opcode, inst->oparg); + 0 : _PyUop_num_popped(opcode, inst->oparg); if (target != current_error_target || popped != current_popped) { current_popped = popped; current_error = next_spare; diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py index 3feccbe8b962fa..7b3325ada4a49f 100644 --- a/Tools/cases_generator/uop_metadata_generator.py +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -26,7 +26,7 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: out.emit("extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1];\n") out.emit("extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1];\n") out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];\n\n") - out.emit("extern int _PyUop_Popped(int opcode, int oparg);\n\n") + out.emit("extern int _PyUop_num_popped(int opcode, int oparg);\n\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") out.emit("const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") for uop in analysis.uops.values(): @@ -45,7 +45,7 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: if uop.is_viable() and uop.properties.tier != 1: out.emit(f'[{uop.name}] = "{uop.name}",\n') out.emit("};\n") - out.emit("int _PyUop_Popped(int opcode, int oparg)\n{\n") + out.emit("int _PyUop_num_popped(int opcode, int oparg)\n{\n") out.emit("switch(opcode) {\n") for uop in analysis.uops.values(): if uop.is_viable() and uop.properties.tier != 1: From fa5d14b10b8c5de688d90118847f20de9f511da8 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 20 Mar 2024 09:50:15 +0000 Subject: [PATCH 17/24] Address review comments --- Include/internal/pycore_uop_ids.h | 197 ++++++++++++------------- Include/internal/pycore_uop_metadata.h | 10 +- Python/bytecodes.c | 6 +- Python/executor_cases.c.h | 7 +- Python/optimizer.c | 23 ++- Python/optimizer_analysis.c | 5 +- Python/optimizer_cases.c.h | 6 +- 7 files changed, 118 insertions(+), 136 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index cf5577d1ad4d9f..bcb10ab723ecba 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -91,44 +91,43 @@ extern "C" { #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE #define _END_SEND END_SEND -#define _ERROR_0 342 -#define _ERROR_N 343 +#define _ERROR_POP_N 342 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _FATAL_ERROR 344 +#define _FATAL_ERROR 343 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 345 +#define _FOR_ITER 344 #define _FOR_ITER_GEN FOR_ITER_GEN -#define _FOR_ITER_TIER_TWO 346 +#define _FOR_ITER_TIER_TWO 345 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 347 -#define _GUARD_BOTH_INT 348 -#define _GUARD_BOTH_UNICODE 349 -#define _GUARD_BUILTINS_VERSION 350 -#define _GUARD_DORV_VALUES 351 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 352 -#define _GUARD_GLOBALS_VERSION 353 -#define _GUARD_IS_FALSE_POP 354 -#define _GUARD_IS_NONE_POP 355 -#define _GUARD_IS_NOT_NONE_POP 356 -#define _GUARD_IS_TRUE_POP 357 -#define _GUARD_KEYS_VERSION 358 -#define _GUARD_NOT_EXHAUSTED_LIST 359 -#define _GUARD_NOT_EXHAUSTED_RANGE 360 -#define _GUARD_NOT_EXHAUSTED_TUPLE 361 -#define _GUARD_TYPE_VERSION 362 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 363 -#define _INIT_CALL_PY_EXACT_ARGS 364 -#define _INIT_CALL_PY_EXACT_ARGS_0 365 -#define _INIT_CALL_PY_EXACT_ARGS_1 366 -#define _INIT_CALL_PY_EXACT_ARGS_2 367 -#define _INIT_CALL_PY_EXACT_ARGS_3 368 -#define _INIT_CALL_PY_EXACT_ARGS_4 369 +#define _GUARD_BOTH_FLOAT 346 +#define _GUARD_BOTH_INT 347 +#define _GUARD_BOTH_UNICODE 348 +#define _GUARD_BUILTINS_VERSION 349 +#define _GUARD_DORV_VALUES 350 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 351 +#define _GUARD_GLOBALS_VERSION 352 +#define _GUARD_IS_FALSE_POP 353 +#define _GUARD_IS_NONE_POP 354 +#define _GUARD_IS_NOT_NONE_POP 355 +#define _GUARD_IS_TRUE_POP 356 +#define _GUARD_KEYS_VERSION 357 +#define _GUARD_NOT_EXHAUSTED_LIST 358 +#define _GUARD_NOT_EXHAUSTED_RANGE 359 +#define _GUARD_NOT_EXHAUSTED_TUPLE 360 +#define _GUARD_TYPE_VERSION 361 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 362 +#define _INIT_CALL_PY_EXACT_ARGS 363 +#define _INIT_CALL_PY_EXACT_ARGS_0 364 +#define _INIT_CALL_PY_EXACT_ARGS_1 365 +#define _INIT_CALL_PY_EXACT_ARGS_2 366 +#define _INIT_CALL_PY_EXACT_ARGS_3 367 +#define _INIT_CALL_PY_EXACT_ARGS_4 368 #define _INSTRUMENTED_CALL INSTRUMENTED_CALL #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW @@ -145,65 +144,65 @@ extern "C" { #define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST #define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE #define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 370 -#define _IS_NONE 371 +#define _INTERNAL_INCREMENT_OPT_COUNTER 369 +#define _IS_NONE 370 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 372 -#define _ITER_CHECK_RANGE 373 -#define _ITER_CHECK_TUPLE 374 -#define _ITER_JUMP_LIST 375 -#define _ITER_JUMP_RANGE 376 -#define _ITER_JUMP_TUPLE 377 -#define _ITER_NEXT_LIST 378 -#define _ITER_NEXT_RANGE 379 -#define _ITER_NEXT_TUPLE 380 -#define _JUMP_TO_TOP 381 +#define _ITER_CHECK_LIST 371 +#define _ITER_CHECK_RANGE 372 +#define _ITER_CHECK_TUPLE 373 +#define _ITER_JUMP_LIST 374 +#define _ITER_JUMP_RANGE 375 +#define _ITER_JUMP_TUPLE 376 +#define _ITER_NEXT_LIST 377 +#define _ITER_NEXT_RANGE 378 +#define _ITER_NEXT_TUPLE 379 +#define _JUMP_TO_TOP 380 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND #define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR -#define _LOAD_ATTR 382 -#define _LOAD_ATTR_CLASS 383 -#define _LOAD_ATTR_CLASS_0 384 -#define _LOAD_ATTR_CLASS_1 385 +#define _LOAD_ATTR 381 +#define _LOAD_ATTR_CLASS 382 +#define _LOAD_ATTR_CLASS_0 383 +#define _LOAD_ATTR_CLASS_1 384 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 386 -#define _LOAD_ATTR_INSTANCE_VALUE_0 387 -#define _LOAD_ATTR_INSTANCE_VALUE_1 388 -#define _LOAD_ATTR_METHOD_LAZY_DICT 389 -#define _LOAD_ATTR_METHOD_NO_DICT 390 -#define _LOAD_ATTR_METHOD_WITH_VALUES 391 -#define _LOAD_ATTR_MODULE 392 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 393 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 394 +#define _LOAD_ATTR_INSTANCE_VALUE 385 +#define _LOAD_ATTR_INSTANCE_VALUE_0 386 +#define _LOAD_ATTR_INSTANCE_VALUE_1 387 +#define _LOAD_ATTR_METHOD_LAZY_DICT 388 +#define _LOAD_ATTR_METHOD_NO_DICT 389 +#define _LOAD_ATTR_METHOD_WITH_VALUES 390 +#define _LOAD_ATTR_MODULE 391 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 392 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 393 #define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_SLOT 395 -#define _LOAD_ATTR_SLOT_0 396 -#define _LOAD_ATTR_SLOT_1 397 -#define _LOAD_ATTR_WITH_HINT 398 +#define _LOAD_ATTR_SLOT 394 +#define _LOAD_ATTR_SLOT_0 395 +#define _LOAD_ATTR_SLOT_1 396 +#define _LOAD_ATTR_WITH_HINT 397 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 399 -#define _LOAD_CONST_INLINE_BORROW 400 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 401 -#define _LOAD_CONST_INLINE_WITH_NULL 402 +#define _LOAD_CONST_INLINE 398 +#define _LOAD_CONST_INLINE_BORROW 399 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 400 +#define _LOAD_CONST_INLINE_WITH_NULL 401 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 403 -#define _LOAD_FAST_0 404 -#define _LOAD_FAST_1 405 -#define _LOAD_FAST_2 406 -#define _LOAD_FAST_3 407 -#define _LOAD_FAST_4 408 -#define _LOAD_FAST_5 409 -#define _LOAD_FAST_6 410 -#define _LOAD_FAST_7 411 +#define _LOAD_FAST 402 +#define _LOAD_FAST_0 403 +#define _LOAD_FAST_1 404 +#define _LOAD_FAST_2 405 +#define _LOAD_FAST_3 406 +#define _LOAD_FAST_4 407 +#define _LOAD_FAST_5 408 +#define _LOAD_FAST_6 409 +#define _LOAD_FAST_7 410 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 412 -#define _LOAD_GLOBAL_BUILTINS 413 -#define _LOAD_GLOBAL_MODULE 414 +#define _LOAD_GLOBAL 411 +#define _LOAD_GLOBAL_BUILTINS 412 +#define _LOAD_GLOBAL_MODULE 413 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR @@ -217,49 +216,49 @@ extern "C" { #define _MATCH_SEQUENCE MATCH_SEQUENCE #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_FRAME 415 -#define _POP_JUMP_IF_FALSE 416 -#define _POP_JUMP_IF_TRUE 417 +#define _POP_FRAME 414 +#define _POP_JUMP_IF_FALSE 415 +#define _POP_JUMP_IF_TRUE 416 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 418 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 417 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 419 +#define _PUSH_FRAME 418 #define _PUSH_NULL PUSH_NULL -#define _REPLACE_WITH_TRUE 420 +#define _REPLACE_WITH_TRUE 419 #define _RESUME_CHECK RESUME_CHECK -#define _SAVE_RETURN_OFFSET 421 -#define _SEND 422 +#define _SAVE_RETURN_OFFSET 420 +#define _SEND 421 #define _SEND_GEN SEND_GEN #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _SIDE_EXIT 423 -#define _START_EXECUTOR 424 -#define _STORE_ATTR 425 -#define _STORE_ATTR_INSTANCE_VALUE 426 -#define _STORE_ATTR_SLOT 427 +#define _SIDE_EXIT 422 +#define _START_EXECUTOR 423 +#define _STORE_ATTR 424 +#define _STORE_ATTR_INSTANCE_VALUE 425 +#define _STORE_ATTR_SLOT 426 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 428 -#define _STORE_FAST_0 429 -#define _STORE_FAST_1 430 -#define _STORE_FAST_2 431 -#define _STORE_FAST_3 432 -#define _STORE_FAST_4 433 -#define _STORE_FAST_5 434 -#define _STORE_FAST_6 435 -#define _STORE_FAST_7 436 +#define _STORE_FAST 427 +#define _STORE_FAST_0 428 +#define _STORE_FAST_1 429 +#define _STORE_FAST_2 430 +#define _STORE_FAST_3 431 +#define _STORE_FAST_4 432 +#define _STORE_FAST_5 433 +#define _STORE_FAST_6 434 +#define _STORE_FAST_7 435 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 437 +#define _STORE_SUBSCR 436 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TO_BOOL 438 +#define _TO_BOOL 437 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -269,12 +268,12 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 439 +#define _UNPACK_SEQUENCE 438 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define MAX_UOP_ID 439 +#define MAX_UOP_ID 438 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 3ee76c622a27b1..d5f8483d36f488 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -244,8 +244,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, [_DEOPT] = 0, [_SIDE_EXIT] = 0, - [_ERROR_0] = 0, - [_ERROR_N] = HAS_ARG_FLAG, + [_ERROR_POP_N] = HAS_ARG_FLAG, }; const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { @@ -326,8 +325,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_DICT_MERGE] = "_DICT_MERGE", [_DICT_UPDATE] = "_DICT_UPDATE", [_END_SEND] = "_END_SEND", - [_ERROR_0] = "_ERROR_0", - [_ERROR_N] = "_ERROR_N", + [_ERROR_POP_N] = "_ERROR_POP_N", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", [_EXIT_TRACE] = "_EXIT_TRACE", [_FATAL_ERROR] = "_FATAL_ERROR", @@ -936,9 +934,7 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _SIDE_EXIT: return 0; - case _ERROR_0: - return 0; - case _ERROR_N: + case _ERROR_POP_N: return oparg; default: return -1; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4496df9e8798f2..34f7851c944a12 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4215,11 +4215,7 @@ dummy_func( EXIT_TO_TRACE(); } - tier2 op(_ERROR_0, (--)) { - GOTO_UNWIND(); - } - - tier2 op(_ERROR_N, (values[oparg] --)) { + tier2 op(_ERROR_POP_N, (values[oparg] --)) { (void)values; SYNC_SP(); GOTO_UNWIND(); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index dd3af5a54fd144..e6f34a2b1ffbb3 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3759,12 +3759,7 @@ break; } - case _ERROR_0: { - GOTO_UNWIND(); - break; - } - - case _ERROR_N: { + case _ERROR_POP_N: { PyObject **values; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg]; diff --git a/Python/optimizer.c b/Python/optimizer.c index 88c5c3b548aab6..e5ca7df533a785 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -159,8 +159,11 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil static int init_cold_exit_executor(_PyExecutorObject *executor, int oparg); +/* The maximum number of exits in a trace cannot reach 1/4 of its length */ +#define COLD_EXIT_COUNT (UOP_MAX_TRACE_LENGTH/4) + static int cold_exits_initialized = 0; -static _PyExecutorObject COLD_EXITS[UOP_MAX_TRACE_LENGTH/4] = { 0 }; +static _PyExecutorObject COLD_EXITS[COLD_EXIT_COUNT] = { 0 }; static const _PyBloomFilter EMPTY_FILTER = { 0 }; @@ -172,7 +175,7 @@ _Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer) } else if (cold_exits_initialized == 0) { cold_exits_initialized = 1; - for (int i = 0; i < UOP_MAX_TRACE_LENGTH/4; i++) { + for (int i = 0; i < COLD_EXIT_COUNT; i++) { if (init_cold_exit_executor(&COLD_EXITS[i], i)) { return NULL; } @@ -921,7 +924,6 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) int32_t current_error_target = -1; int32_t current_popped = -1; /* Leaving in NOPs slows down the interpreter and messes up the stats */ -#ifndef _Py_JIT _PyUOpInstruction *copy_to = &buffer[0]; for (int i = 0; i < length; i++) { _PyUOpInstruction *inst = &buffer[i]; @@ -932,8 +934,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) copy_to++; } } - length = copy_to - buffer; -#endif + length = (int)(copy_to - buffer); int next_spare = length; for (int i = 0; i < length; i++) { _PyUOpInstruction *inst = &buffer[i]; @@ -957,8 +958,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) current_popped = popped; current_error = next_spare; current_error_target = target; - uint16_t error_op = popped ? _ERROR_N : _ERROR_0; - make_exit(&buffer[next_spare], error_op, 0); + make_exit(&buffer[next_spare], _ERROR_POP_N, 0); buffer[next_spare].oparg = popped; next_spare++; } @@ -1049,8 +1049,7 @@ sanity_check(_PyExecutorObject *executor) CHECK( opcode == _DEOPT || opcode == _SIDE_EXIT || - opcode == _ERROR_0 || - opcode == _ERROR_N); + opcode == _ERROR_POP_N); if (opcode == _SIDE_EXIT) { CHECK(inst->format == UOP_FORMAT_EXIT); } @@ -1075,7 +1074,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil } /* Initialize exits */ - assert(exit_count < UOP_MAX_TRACE_LENGTH/4); + assert(exit_count < COLD_EXIT_COUNT); for (int i = 0; i < exit_count; i++) { executor->exits[i].executor = &COLD_EXITS[i]; executor->exits[i].temperature = 0; @@ -1175,8 +1174,8 @@ int effective_trace_length(_PyUOpInstruction *buffer, int length) return i+1-nop_count; } } - assert(0 && "No terminating instruction"); - return length-nop_count; + Py_FatalError("No terminating instruction"); + Py_UNREACHABLE(); } #endif diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 7ad7c9321ffe9b..578406bbc39998 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -439,8 +439,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) * instruction could have escaped. */ int last_set_ip = -1; bool may_have_escaped = true; - for (int pc = 0; ; pc++) { - assert(pc < buffer_size && "No terminating uop"); + for (int pc = 0; pc < buffer_size; pc++) { int opcode = buffer[pc].opcode; switch (opcode) { case _SET_IP: @@ -510,6 +509,8 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } } } + Py_FatalError("No terminating instruction"); + Py_UNREACHABLE(); } static void diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b172c15be5a72e..5014a76d8e1821 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1988,11 +1988,7 @@ break; } - case _ERROR_0: { - break; - } - - case _ERROR_N: { + case _ERROR_POP_N: { stack_pointer += -oparg; break; } From 5217c1292ba2fd0d146f1b53a43fcfda51bc5377 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 20 Mar 2024 09:59:06 +0000 Subject: [PATCH 18/24] Add comment --- Include/cpython/optimizer.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 9f0fff34fc38e4..bc960c583782c5 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -35,6 +35,17 @@ typedef struct { #define UOP_FORMAT_JUMP 2 #define UOP_FORMAT_UNUSED 3 +/* Depending on the format, + * the 32 bits between the oparg and operand are: + * UOP_FORMAT_TARGET: + * uint32_t target; + * UOP_FORMAT_EXIT + * uint16_t exit_index; + * uint16_t error_target; + * UOP_FORMAT_JUMP + * uint16_t jump_target; + * uint16_t error_target; + */ typedef struct { uint16_t opcode:14; uint16_t format:2; From 43ba205c7bd2e6a41da9c9cd97e571bbc710ecc2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 20 Mar 2024 10:23:28 +0000 Subject: [PATCH 19/24] Rename and move macros --- Include/internal/pycore_opcode_metadata.h | 74 +-- Include/internal/pycore_uop_metadata.h | 26 +- Python/bytecodes.c | 100 ++-- Python/ceval.c | 6 - Python/ceval_macros.h | 6 + Python/executor_cases.c.h | 490 +++++++++--------- Python/optimizer.c | 2 +- Tools/cases_generator/analyzer.py | 2 +- Tools/cases_generator/generators_common.py | 6 +- .../opcode_metadata_generator.py | 2 +- Tools/cases_generator/tier2_generator.py | 12 +- Tools/jit/template.c | 4 +- 12 files changed, 365 insertions(+), 365 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index d6cfc07cc3bb1d..de525f72d3523e 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -921,7 +921,7 @@ enum InstructionFormat { #define HAS_PURE_FLAG (2048) #define HAS_PASSTHROUGH_FLAG (4096) #define HAS_OPARG_AND_1_FLAG (8192) -#define HAS_NO_POP_ERROR_FLAG (16384) +#define HAS_ERROR_NO_POP_FLAG (16384) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -936,7 +936,7 @@ enum InstructionFormat { #define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG)) #define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) #define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) -#define OPCODE_HAS_NO_POP_ERROR(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NO_POP_ERROR_FLAG)) +#define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) #define OPARG_FULL 0 #define OPARG_CACHE_1 1 @@ -956,8 +956,8 @@ struct opcode_metadata { extern const struct opcode_metadata _PyOpcode_opcode_metadata[268]; #ifdef NEED_OPCODE_METADATA const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { - [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG }, [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, @@ -977,25 +977,25 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG }, + [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1007,7 +1007,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, @@ -1019,40 +1019,40 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1061,10 +1061,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1093,18 +1093,18 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1119,15 +1119,15 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [POP_TOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, - [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ESCAPES_FLAG }, - [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index d5f8483d36f488..51206cd4ca2fdf 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -85,13 +85,13 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_POP_FRAME] = HAS_ESCAPES_FLAG, [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_ASSERTION_ERROR] = 0, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, @@ -100,18 +100,18 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_LOCALS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_FROM_DICT_OR_GLOBALS] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_FROM_DICT_OR_GLOBALS] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_GLOBALS_VERSION] = HAS_DEOPT_FLAG, [_GUARD_BUILTINS_VERSION] = HAS_DEOPT_FLAG, [_LOAD_GLOBAL_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_LOAD_GLOBAL_BUILTINS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG, [_COPY_FREE_VARS] = HAS_ARG_FLAG, @@ -165,8 +165,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MATCH_SEQUENCE] = 0, [_MATCH_KEYS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_FOR_ITER_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_FOR_ITER_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_ITER_NEXT_LIST] = 0, @@ -202,18 +202,18 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CALL_STR_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_NO_POP_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 34f7851c944a12..0c52271ef06c48 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -179,7 +179,7 @@ dummy_func( uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; if (code_version != global_version) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { - NO_POP_ERROR(); + ERROR_NO_POP(); } next_instr = this_instr; } @@ -281,7 +281,7 @@ dummy_func( if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { - NO_POP_ERROR(); + ERROR_NO_POP(); } PyErr_SetRaisedException(NULL); } @@ -296,7 +296,7 @@ dummy_func( if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { - NO_POP_ERROR(); + ERROR_NO_POP(); } PyErr_SetRaisedException(NULL); } @@ -832,7 +832,7 @@ dummy_func( int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); - if (err) NO_POP_ERROR(); + if (err) ERROR_NO_POP(); STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -856,7 +856,7 @@ dummy_func( int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); - if (err) NO_POP_ERROR(); + if (err) ERROR_NO_POP(); Py_INCREF(retval); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -912,7 +912,7 @@ dummy_func( if (PyAsyncGen_CheckExact(aiter)) { awaitable = type->tp_as_async->am_anext(aiter); if (awaitable == NULL) { - NO_POP_ERROR(); + ERROR_NO_POP(); } } else { if (type->tp_as_async != NULL){ @@ -922,7 +922,7 @@ dummy_func( if (getter != NULL) { next_iter = (*getter)(aiter); if (next_iter == NULL) { - NO_POP_ERROR(); + ERROR_NO_POP(); } } else { @@ -930,7 +930,7 @@ dummy_func( "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); - NO_POP_ERROR(); + ERROR_NO_POP(); } awaitable = _PyCoro_GetAwaitableIter(next_iter); @@ -942,7 +942,7 @@ dummy_func( Py_TYPE(next_iter)->tp_name); Py_DECREF(next_iter); - NO_POP_ERROR(); + ERROR_NO_POP(); } else { Py_DECREF(next_iter); } @@ -1024,7 +1024,7 @@ dummy_func( JUMPBY(oparg); } else { - NO_POP_ERROR(); + ERROR_NO_POP(); } } Py_DECREF(v); @@ -1060,7 +1060,7 @@ dummy_func( int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, this_instr, retval); - if (err) NO_POP_ERROR(); + if (err) ERROR_NO_POP(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -1114,7 +1114,7 @@ dummy_func( else { assert(PyLong_Check(lasti)); _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - NO_POP_ERROR(); + ERROR_NO_POP(); } } assert(exc && PyExceptionInstance_Check(exc)); @@ -1190,7 +1190,7 @@ dummy_func( if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - NO_POP_ERROR(); + ERROR_NO_POP(); } err = PyObject_DelItem(ns, name); // Can't use ERROR_IF here. @@ -1198,7 +1198,7 @@ dummy_func( _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - NO_POP_ERROR(); + ERROR_NO_POP(); } } @@ -1318,12 +1318,12 @@ dummy_func( int err = PyDict_Pop(GLOBALS(), name, NULL); // Can't use ERROR_IF here. if (err < 0) { - NO_POP_ERROR(); + ERROR_NO_POP(); } if (err == 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - NO_POP_ERROR(); + ERROR_NO_POP(); } } @@ -1340,21 +1340,21 @@ dummy_func( inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - NO_POP_ERROR(); + ERROR_NO_POP(); } if (v == NULL) { if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - NO_POP_ERROR(); + ERROR_NO_POP(); } if (v == NULL) { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - NO_POP_ERROR(); + ERROR_NO_POP(); } if (v == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - NO_POP_ERROR(); + ERROR_NO_POP(); } } } @@ -1370,21 +1370,21 @@ dummy_func( } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - NO_POP_ERROR(); + ERROR_NO_POP(); } if (v == NULL) { if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - NO_POP_ERROR(); + ERROR_NO_POP(); } if (v == NULL) { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - NO_POP_ERROR(); + ERROR_NO_POP(); } if (v == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - NO_POP_ERROR(); + ERROR_NO_POP(); } } } @@ -1516,7 +1516,7 @@ dummy_func( PyObject *initial = GETLOCAL(oparg); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - NO_POP_ERROR(); + ERROR_NO_POP(); } SETLOCAL(oparg, cell); } @@ -1528,7 +1528,7 @@ dummy_func( // Fortunately we don't need its superpower. if (oldobj == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - NO_POP_ERROR(); + ERROR_NO_POP(); } PyCell_SET(cell, NULL); Py_DECREF(oldobj); @@ -1540,14 +1540,14 @@ dummy_func( assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { - NO_POP_ERROR(); + ERROR_NO_POP(); } if (!value) { PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - NO_POP_ERROR(); + ERROR_NO_POP(); } Py_INCREF(value); } @@ -1627,7 +1627,7 @@ dummy_func( inst(BUILD_SET, (values[oparg] -- set)) { set = PySet_New(NULL); if (set == NULL) - NO_POP_ERROR(); + ERROR_NO_POP(); int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; @@ -2510,7 +2510,7 @@ dummy_func( _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); - NO_POP_ERROR(); + ERROR_NO_POP(); } iter = iterable; } @@ -2521,7 +2521,7 @@ dummy_func( /* `iterable` is not a generator. */ iter = PyObject_GetIter(iterable); if (iter == NULL) { - NO_POP_ERROR(); + ERROR_NO_POP(); } DECREF_INPUTS(); } @@ -2558,7 +2558,7 @@ dummy_func( if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - NO_POP_ERROR(); + ERROR_NO_POP(); } monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); @@ -2581,7 +2581,7 @@ dummy_func( if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - NO_POP_ERROR(); + ERROR_NO_POP(); } _PyErr_Clear(tstate); } @@ -2607,7 +2607,7 @@ dummy_func( else { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - NO_POP_ERROR(); + ERROR_NO_POP(); } monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); @@ -2787,7 +2787,7 @@ dummy_func( "asynchronous context manager protocol", Py_TYPE(mgr)->tp_name); } - NO_POP_ERROR(); + ERROR_NO_POP(); } exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); if (exit == NULL) { @@ -2799,7 +2799,7 @@ dummy_func( Py_TYPE(mgr)->tp_name); } Py_DECREF(enter); - NO_POP_ERROR(); + ERROR_NO_POP(); } DECREF_INPUTS(); res = PyObject_CallNoArgs(enter); @@ -2822,7 +2822,7 @@ dummy_func( "context manager protocol", Py_TYPE(mgr)->tp_name); } - NO_POP_ERROR(); + ERROR_NO_POP(); } exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); if (exit == NULL) { @@ -2834,7 +2834,7 @@ dummy_func( Py_TYPE(mgr)->tp_name); } Py_DECREF(enter); - NO_POP_ERROR(); + ERROR_NO_POP(); } DECREF_INPUTS(); res = PyObject_CallNoArgs(enter); @@ -3083,7 +3083,7 @@ dummy_func( // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - NO_POP_ERROR(); + ERROR_NO_POP(); } frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); @@ -3306,7 +3306,7 @@ dummy_func( STAT_INC(CALL, hit); PyObject *self = _PyType_NewManagedObject(tp); if (self == NULL) { - NO_POP_ERROR(); + ERROR_NO_POP(); } Py_DECREF(tp); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( @@ -3343,7 +3343,7 @@ dummy_func( PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(should_be_none)->tp_name); - NO_POP_ERROR(); + ERROR_NO_POP(); } } @@ -3480,7 +3480,7 @@ dummy_func( PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { - NO_POP_ERROR(); + ERROR_NO_POP(); } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3506,7 +3506,7 @@ dummy_func( PyObject *inst = args[0]; int retval = PyObject_IsInstance(inst, cls); if (retval < 0) { - NO_POP_ERROR(); + ERROR_NO_POP(); } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3720,7 +3720,7 @@ dummy_func( // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - NO_POP_ERROR(); + ERROR_NO_POP(); } assert(next_instr - this_instr == 1); frame->return_offset = 1; @@ -3768,11 +3768,11 @@ dummy_func( assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (!PyTuple_CheckExact(callargs)) { if (check_args_iterable(tstate, func, callargs) < 0) { - NO_POP_ERROR(); + ERROR_NO_POP(); } PyObject *tuple = PySequence_Tuple(callargs); if (tuple == NULL) { - NO_POP_ERROR(); + ERROR_NO_POP(); } Py_SETREF(callargs, tuple); } @@ -3784,7 +3784,7 @@ dummy_func( int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); - if (err) NO_POP_ERROR(); + if (err) ERROR_NO_POP(); result = PyObject_Call(func, callargs, kwargs); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { @@ -3818,7 +3818,7 @@ dummy_func( // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { - NO_POP_ERROR(); + ERROR_NO_POP(); } assert(next_instr - this_instr == 1); frame->return_offset = 1; @@ -3839,7 +3839,7 @@ dummy_func( Py_DECREF(codeobj); if (func_obj == NULL) { - NO_POP_ERROR(); + ERROR_NO_POP(); } _PyFunction_SetVersion( @@ -3879,7 +3879,7 @@ dummy_func( PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); if (gen == NULL) { - NO_POP_ERROR(); + ERROR_NO_POP(); } assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Python/ceval.c b/Python/ceval.c index 6d4e98fd494b71..af4e4153a4ecfe 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -971,12 +971,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #define GOTO_ERROR(LABEL) goto LABEL ## _tier_two #undef DEOPT_IF -#define JUMP_TO_JUMP_TARGET goto jump_to_jump_target -#define JUMP_TO_ERROR goto jump_to_error_target -#define NO_POP_ERROR() goto jump_to_error_target -#define GOTO_UNWIND() goto error_tier_two -#define EXIT_TO_TRACE() goto exit_to_trace -#define EXIT_TO_TIER1() goto exit_to_tier1 #ifdef Py_STATS // Disable these macros that apply to Tier 1 stats when we are in Tier 2 diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 22992aa09e1f38..f2536ed3602c69 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -423,3 +423,9 @@ do { \ #define CURRENT_OPARG() (next_uop[-1].oparg) #define CURRENT_OPERAND() (next_uop[-1].operand) + +#define JUMP_TO_JUMP_TARGET() goto jump_to_jump_target +#define JUMP_TO_ERROR() goto jump_to_error_target +#define GOTO_UNWIND() goto error_tier_two +#define EXIT_TO_TRACE() goto exit_to_trace +#define EXIT_TO_TIER1() goto exit_to_tier1 diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e6f34a2b1ffbb3..dcbd973c3721a6 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -14,13 +14,13 @@ case _RESUME_CHECK: { #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) JUMP_TO_JUMP_TARGET; + if (_Py_emscripten_signal_clock == 0) JUMP_TO_JUMP_TARGET(); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) JUMP_TO_JUMP_TARGET; + if (eval_breaker != version) JUMP_TO_JUMP_TARGET(); break; } @@ -35,7 +35,7 @@ UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); - if (1) JUMP_TO_ERROR; + if (1) JUMP_TO_ERROR(); } Py_INCREF(value); stack_pointer[0] = value; @@ -293,7 +293,7 @@ value = stack_pointer[-1]; res = PyNumber_Negative(value); Py_DECREF(value); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = res; break; } @@ -314,7 +314,7 @@ value = stack_pointer[-1]; int err = PyObject_IsTrue(value); Py_DECREF(value); - if (err < 0) JUMP_TO_ERROR; + if (err < 0) JUMP_TO_ERROR(); res = err ? Py_True : Py_False; stack_pointer[-1] = res; break; @@ -323,7 +323,7 @@ case _TO_BOOL_BOOL: { PyObject *value; value = stack_pointer[-1]; - if (!PyBool_Check(value)) JUMP_TO_JUMP_TARGET; + if (!PyBool_Check(value)) JUMP_TO_JUMP_TARGET(); STAT_INC(TO_BOOL, hit); break; } @@ -332,7 +332,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyLong_CheckExact(value)) JUMP_TO_JUMP_TARGET; + if (!PyLong_CheckExact(value)) JUMP_TO_JUMP_TARGET(); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { assert(_Py_IsImmortal(value)); @@ -350,7 +350,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyList_CheckExact(value)) JUMP_TO_JUMP_TARGET; + if (!PyList_CheckExact(value)) JUMP_TO_JUMP_TARGET(); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; Py_DECREF(value); @@ -363,7 +363,7 @@ PyObject *res; value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - if (!Py_IsNone(value)) JUMP_TO_JUMP_TARGET; + if (!Py_IsNone(value)) JUMP_TO_JUMP_TARGET(); STAT_INC(TO_BOOL, hit); res = Py_False; stack_pointer[-1] = res; @@ -374,7 +374,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyUnicode_CheckExact(value)) JUMP_TO_JUMP_TARGET; + if (!PyUnicode_CheckExact(value)) JUMP_TO_JUMP_TARGET(); STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { assert(_Py_IsImmortal(value)); @@ -405,7 +405,7 @@ value = stack_pointer[-1]; res = PyNumber_Invert(value); Py_DECREF(value); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = res; break; } @@ -415,8 +415,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyLong_CheckExact(left)) JUMP_TO_JUMP_TARGET; - if (!PyLong_CheckExact(right)) JUMP_TO_JUMP_TARGET; + if (!PyLong_CheckExact(left)) JUMP_TO_JUMP_TARGET(); + if (!PyLong_CheckExact(right)) JUMP_TO_JUMP_TARGET(); break; } @@ -430,7 +430,7 @@ res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -446,7 +446,7 @@ res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -462,7 +462,7 @@ res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -473,8 +473,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyFloat_CheckExact(left)) JUMP_TO_JUMP_TARGET; - if (!PyFloat_CheckExact(right)) JUMP_TO_JUMP_TARGET; + if (!PyFloat_CheckExact(left)) JUMP_TO_JUMP_TARGET(); + if (!PyFloat_CheckExact(right)) JUMP_TO_JUMP_TARGET(); break; } @@ -531,8 +531,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyUnicode_CheckExact(left)) JUMP_TO_JUMP_TARGET; - if (!PyUnicode_CheckExact(right)) JUMP_TO_JUMP_TARGET; + if (!PyUnicode_CheckExact(left)) JUMP_TO_JUMP_TARGET(); + if (!PyUnicode_CheckExact(right)) JUMP_TO_JUMP_TARGET(); break; } @@ -546,7 +546,7 @@ res = PyUnicode_Concat(left, right); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -561,7 +561,7 @@ res = PyObject_GetItem(container, sub); Py_DECREF(container); Py_DECREF(sub); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -586,7 +586,7 @@ Py_DECREF(slice); } Py_DECREF(container); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-3] = res; stack_pointer += -2; break; @@ -612,7 +612,7 @@ } Py_DECREF(v); Py_DECREF(container); - if (err) JUMP_TO_ERROR; + if (err) JUMP_TO_ERROR(); stack_pointer += -4; break; } @@ -623,12 +623,12 @@ PyObject *res; sub = stack_pointer[-1]; list = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET; - if (!PyList_CheckExact(list)) JUMP_TO_JUMP_TARGET; + if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); + if (!PyList_CheckExact(list)) JUMP_TO_JUMP_TARGET(); // Deopt unless 0 <= sub < PyList_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET; + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyList_GET_SIZE(list)) JUMP_TO_JUMP_TARGET; + if (index >= PyList_GET_SIZE(list)) JUMP_TO_JUMP_TARGET(); STAT_INC(BINARY_SUBSCR, hit); res = PyList_GET_ITEM(list, index); assert(res != NULL); @@ -646,14 +646,14 @@ PyObject *res; sub = stack_pointer[-1]; str = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET; - if (!PyUnicode_CheckExact(str)) JUMP_TO_JUMP_TARGET; - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET; + if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); + if (!PyUnicode_CheckExact(str)) JUMP_TO_JUMP_TARGET(); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) JUMP_TO_JUMP_TARGET; + if (PyUnicode_GET_LENGTH(str) <= index) JUMP_TO_JUMP_TARGET(); // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) JUMP_TO_JUMP_TARGET; + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) JUMP_TO_JUMP_TARGET(); STAT_INC(BINARY_SUBSCR, hit); res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); @@ -669,12 +669,12 @@ PyObject *res; sub = stack_pointer[-1]; tuple = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET; - if (!PyTuple_CheckExact(tuple)) JUMP_TO_JUMP_TARGET; + if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); + if (!PyTuple_CheckExact(tuple)) JUMP_TO_JUMP_TARGET(); // Deopt unless 0 <= sub < PyTuple_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET; + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) JUMP_TO_JUMP_TARGET; + if (index >= PyTuple_GET_SIZE(tuple)) JUMP_TO_JUMP_TARGET(); STAT_INC(BINARY_SUBSCR, hit); res = PyTuple_GET_ITEM(tuple, index); assert(res != NULL); @@ -692,7 +692,7 @@ PyObject *res; sub = stack_pointer[-1]; dict = stack_pointer[-2]; - if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET; + if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET(); STAT_INC(BINARY_SUBSCR, hit); int rc = PyDict_GetItemRef(dict, sub, &res); if (rc == 0) { @@ -700,7 +700,7 @@ } Py_DECREF(dict); Py_DECREF(sub); - if (rc <= 0) JUMP_TO_ERROR; + if (rc <= 0) JUMP_TO_ERROR(); // not found or error stack_pointer[-2] = res; stack_pointer += -1; @@ -715,7 +715,7 @@ oparg = CURRENT_OPARG(); v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) JUMP_TO_ERROR; + if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) JUMP_TO_ERROR(); stack_pointer += -1; break; } @@ -728,7 +728,7 @@ set = stack_pointer[-2 - (oparg-1)]; int err = PySet_Add(set, v); Py_DECREF(v); - if (err) JUMP_TO_ERROR; + if (err) JUMP_TO_ERROR(); stack_pointer += -1; break; } @@ -745,7 +745,7 @@ Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); - if (err) JUMP_TO_ERROR; + if (err) JUMP_TO_ERROR(); stack_pointer += -3; break; } @@ -757,13 +757,13 @@ sub = stack_pointer[-1]; list = stack_pointer[-2]; value = stack_pointer[-3]; - if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET; - if (!PyList_CheckExact(list)) JUMP_TO_JUMP_TARGET; + if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); + if (!PyList_CheckExact(list)) JUMP_TO_JUMP_TARGET(); // Ensure nonnegative, zero-or-one-digit ints. - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET; + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; // Ensure index < len(list) - if (index >= PyList_GET_SIZE(list)) JUMP_TO_JUMP_TARGET; + if (index >= PyList_GET_SIZE(list)) JUMP_TO_JUMP_TARGET(); STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); @@ -782,11 +782,11 @@ sub = stack_pointer[-1]; dict = stack_pointer[-2]; value = stack_pointer[-3]; - if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET; + if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET(); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); - if (err) JUMP_TO_ERROR; + if (err) JUMP_TO_ERROR(); stack_pointer += -3; break; } @@ -800,7 +800,7 @@ int err = PyObject_DelItem(container, sub); Py_DECREF(container); Py_DECREF(sub); - if (err) JUMP_TO_ERROR; + if (err) JUMP_TO_ERROR(); stack_pointer += -2; break; } @@ -813,7 +813,7 @@ assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); Py_DECREF(value); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = res; break; } @@ -829,7 +829,7 @@ res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); Py_DECREF(value2); Py_DECREF(value1); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -880,11 +880,11 @@ "__aiter__ method, got %.100s", type->tp_name); Py_DECREF(obj); - if (true) JUMP_TO_ERROR; + if (true) JUMP_TO_ERROR(); } iter = (*getter)(obj); Py_DECREF(obj); - if (iter == NULL) JUMP_TO_ERROR; + if (iter == NULL) JUMP_TO_ERROR(); if (Py_TYPE(iter)->tp_as_async == NULL || Py_TYPE(iter)->tp_as_async->am_anext == NULL) { _PyErr_Format(tstate, PyExc_TypeError, @@ -892,7 +892,7 @@ "that does not implement __anext__: %.100s", Py_TYPE(iter)->tp_name); Py_DECREF(iter); - if (true) JUMP_TO_ERROR; + if (true) JUMP_TO_ERROR(); } stack_pointer[-1] = iter; break; @@ -908,7 +908,7 @@ if (PyAsyncGen_CheckExact(aiter)) { awaitable = type->tp_as_async->am_anext(aiter); if (awaitable == NULL) { - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } } else { if (type->tp_as_async != NULL){ @@ -917,7 +917,7 @@ if (getter != NULL) { next_iter = (*getter)(aiter); if (next_iter == NULL) { - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } } else { @@ -925,7 +925,7 @@ "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } awaitable = _PyCoro_GetAwaitableIter(next_iter); if (awaitable == NULL) { @@ -935,7 +935,7 @@ "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); Py_DECREF(next_iter); - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } else { Py_DECREF(next_iter); } @@ -968,7 +968,7 @@ /* The code below jumps to `error` if `iter` is NULL. */ } } - if (iter == NULL) JUMP_TO_ERROR; + if (iter == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = iter; break; } @@ -998,11 +998,11 @@ case _LOAD_BUILD_CLASS: { PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) JUMP_TO_ERROR; + if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) JUMP_TO_ERROR(); if (bc == NULL) { _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); - if (true) JUMP_TO_ERROR; + if (true) JUMP_TO_ERROR(); } stack_pointer[0] = bc; stack_pointer += 1; @@ -1020,14 +1020,14 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); Py_DECREF(v); - if (true) JUMP_TO_ERROR; + if (true) JUMP_TO_ERROR(); } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); Py_DECREF(v); - if (err) JUMP_TO_ERROR; + if (err) JUMP_TO_ERROR(); stack_pointer += -1; break; } @@ -1040,7 +1040,7 @@ if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } err = PyObject_DelItem(ns, name); // Can't use ERROR_IF here. @@ -1048,7 +1048,7 @@ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } break; } @@ -1060,7 +1060,7 @@ PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); Py_DECREF(seq); - if (res == 0) JUMP_TO_ERROR; + if (res == 0) JUMP_TO_ERROR(); stack_pointer += -1 + oparg; break; } @@ -1072,8 +1072,8 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; assert(oparg == 2); - if (!PyTuple_CheckExact(seq)) JUMP_TO_JUMP_TARGET; - if (PyTuple_GET_SIZE(seq) != 2) JUMP_TO_JUMP_TARGET; + if (!PyTuple_CheckExact(seq)) JUMP_TO_JUMP_TARGET(); + if (PyTuple_GET_SIZE(seq) != 2) JUMP_TO_JUMP_TARGET(); STAT_INC(UNPACK_SEQUENCE, hit); val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); @@ -1090,8 +1090,8 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; values = &stack_pointer[-1]; - if (!PyTuple_CheckExact(seq)) JUMP_TO_JUMP_TARGET; - if (PyTuple_GET_SIZE(seq) != oparg) JUMP_TO_JUMP_TARGET; + if (!PyTuple_CheckExact(seq)) JUMP_TO_JUMP_TARGET(); + if (PyTuple_GET_SIZE(seq) != oparg) JUMP_TO_JUMP_TARGET(); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -1108,8 +1108,8 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; values = &stack_pointer[-1]; - if (!PyList_CheckExact(seq)) JUMP_TO_JUMP_TARGET; - if (PyList_GET_SIZE(seq) != oparg) JUMP_TO_JUMP_TARGET; + if (!PyList_CheckExact(seq)) JUMP_TO_JUMP_TARGET(); + if (PyList_GET_SIZE(seq) != oparg) JUMP_TO_JUMP_TARGET(); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -1128,7 +1128,7 @@ PyObject **top = stack_pointer + totalargs - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq); - if (res == 0) JUMP_TO_ERROR; + if (res == 0) JUMP_TO_ERROR(); stack_pointer += (oparg >> 8) + (oparg & 0xFF); break; } @@ -1143,7 +1143,7 @@ int err = PyObject_SetAttr(owner, name, v); Py_DECREF(v); Py_DECREF(owner); - if (err) JUMP_TO_ERROR; + if (err) JUMP_TO_ERROR(); stack_pointer += -2; break; } @@ -1155,7 +1155,7 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_DelAttr(owner, name); Py_DECREF(owner); - if (err) JUMP_TO_ERROR; + if (err) JUMP_TO_ERROR(); stack_pointer += -1; break; } @@ -1167,7 +1167,7 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); - if (err) JUMP_TO_ERROR; + if (err) JUMP_TO_ERROR(); stack_pointer += -1; break; } @@ -1178,12 +1178,12 @@ int err = PyDict_Pop(GLOBALS(), name, NULL); // Can't use ERROR_IF here. if (err < 0) { - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } if (err == 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } break; } @@ -1194,7 +1194,7 @@ if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - if (true) JUMP_TO_ERROR; + if (true) JUMP_TO_ERROR(); } Py_INCREF(locals); stack_pointer[0] = locals; @@ -1209,21 +1209,21 @@ mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } if (v == NULL) { if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } if (v == NULL) { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } if (v == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } } } @@ -1252,22 +1252,22 @@ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); } - if (true) JUMP_TO_ERROR; + if (true) JUMP_TO_ERROR(); } Py_INCREF(res); } else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) JUMP_TO_ERROR; + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) JUMP_TO_ERROR(); if (res == NULL) { /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) JUMP_TO_ERROR; + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) JUMP_TO_ERROR(); if (res == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - if (true) JUMP_TO_ERROR; + if (true) JUMP_TO_ERROR(); } } } @@ -1281,8 +1281,8 @@ case _GUARD_GLOBALS_VERSION: { uint16_t version = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET; - if (dict->ma_keys->dk_version != version) JUMP_TO_JUMP_TARGET; + if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET(); + if (dict->ma_keys->dk_version != version) JUMP_TO_JUMP_TARGET(); assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1290,8 +1290,8 @@ case _GUARD_BUILTINS_VERSION: { uint16_t version = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)BUILTINS(); - if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET; - if (dict->ma_keys->dk_version != version) JUMP_TO_JUMP_TARGET; + if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET(); + if (dict->ma_keys->dk_version != version) JUMP_TO_JUMP_TARGET(); assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1304,7 +1304,7 @@ PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; - if (res == NULL) JUMP_TO_JUMP_TARGET; + if (res == NULL) JUMP_TO_JUMP_TARGET(); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1322,7 +1322,7 @@ PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; - if (res == NULL) JUMP_TO_JUMP_TARGET; + if (res == NULL) JUMP_TO_JUMP_TARGET(); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1340,7 +1340,7 @@ UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); - if (1) JUMP_TO_ERROR; + if (1) JUMP_TO_ERROR(); } SETLOCAL(oparg, NULL); break; @@ -1353,7 +1353,7 @@ PyObject *initial = GETLOCAL(oparg); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } SETLOCAL(oparg, cell); break; @@ -1367,7 +1367,7 @@ // Fortunately we don't need its superpower. if (oldobj == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } PyCell_SET(cell, NULL); Py_DECREF(oldobj); @@ -1384,14 +1384,14 @@ assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } if (!value) { PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } Py_INCREF(value); } @@ -1407,7 +1407,7 @@ value = PyCell_GET(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - if (true) JUMP_TO_ERROR; + if (true) JUMP_TO_ERROR(); } Py_INCREF(value); stack_pointer[0] = value; @@ -1451,7 +1451,7 @@ for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - if (str == NULL) JUMP_TO_ERROR; + if (str == NULL) JUMP_TO_ERROR(); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; break; @@ -1463,7 +1463,7 @@ oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg]; tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) JUMP_TO_ERROR; + if (tup == NULL) JUMP_TO_ERROR(); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; break; @@ -1475,7 +1475,7 @@ oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg]; list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) JUMP_TO_ERROR; + if (list == NULL) JUMP_TO_ERROR(); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; break; @@ -1498,7 +1498,7 @@ Py_TYPE(iterable)->tp_name); } Py_DECREF(iterable); - if (true) JUMP_TO_ERROR; + if (true) JUMP_TO_ERROR(); } assert(Py_IsNone(none_val)); Py_DECREF(iterable); @@ -1514,7 +1514,7 @@ set = stack_pointer[-2 - (oparg-1)]; int err = _PySet_Update(set, iterable); Py_DECREF(iterable); - if (err < 0) JUMP_TO_ERROR; + if (err < 0) JUMP_TO_ERROR(); stack_pointer += -1; break; } @@ -1533,7 +1533,7 @@ for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - if (map == NULL) JUMP_TO_ERROR; + if (map == NULL) JUMP_TO_ERROR(); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; break; @@ -1545,17 +1545,17 @@ if (LOCALS() == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); - if (true) JUMP_TO_ERROR; + if (true) JUMP_TO_ERROR(); } /* check if __annotations__ in locals()... */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) JUMP_TO_ERROR; + if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) JUMP_TO_ERROR(); if (ann_dict == NULL) { ann_dict = PyDict_New(); - if (ann_dict == NULL) JUMP_TO_ERROR; + if (ann_dict == NULL) JUMP_TO_ERROR(); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); Py_DECREF(ann_dict); - if (err) JUMP_TO_ERROR; + if (err) JUMP_TO_ERROR(); } else { Py_DECREF(ann_dict); @@ -1579,7 +1579,7 @@ Py_DECREF(values[_i]); } Py_DECREF(keys); - if (map == NULL) JUMP_TO_ERROR; + if (map == NULL) JUMP_TO_ERROR(); stack_pointer[-1 - oparg] = map; stack_pointer += -oparg; break; @@ -1598,7 +1598,7 @@ Py_TYPE(update)->tp_name); } Py_DECREF(update); - if (true) JUMP_TO_ERROR; + if (true) JUMP_TO_ERROR(); } Py_DECREF(update); stack_pointer += -1; @@ -1616,7 +1616,7 @@ if (_PyDict_MergeEx(dict, update, 2) < 0) { _PyEval_FormatKwargsError(tstate, callable, update); Py_DECREF(update); - if (true) JUMP_TO_ERROR; + if (true) JUMP_TO_ERROR(); } Py_DECREF(update); stack_pointer += -1; @@ -1634,7 +1634,7 @@ assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) JUMP_TO_ERROR; + if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) JUMP_TO_ERROR(); stack_pointer += -2; break; } @@ -1651,15 +1651,15 @@ class = stack_pointer[-2]; global_super = stack_pointer[-3]; assert(!(oparg & 1)); - if (global_super != (PyObject *)&PySuper_Type) JUMP_TO_JUMP_TARGET; - if (!PyType_Check(class)) JUMP_TO_JUMP_TARGET; + if (global_super != (PyObject *)&PySuper_Type) JUMP_TO_JUMP_TARGET(); + if (!PyType_Check(class)) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); - if (attr == NULL) JUMP_TO_ERROR; + if (attr == NULL) JUMP_TO_ERROR(); stack_pointer[-3] = attr; stack_pointer += -2; break; @@ -1676,8 +1676,8 @@ class = stack_pointer[-2]; global_super = stack_pointer[-3]; assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) JUMP_TO_JUMP_TARGET; - if (!PyType_Check(class)) JUMP_TO_JUMP_TARGET; + if (global_super != (PyObject *)&PySuper_Type) JUMP_TO_JUMP_TARGET(); + if (!PyType_Check(class)) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -1688,7 +1688,7 @@ Py_DECREF(class); if (attr == NULL) { Py_DECREF(self); - if (true) JUMP_TO_ERROR; + if (true) JUMP_TO_ERROR(); } if (method_found) { self_or_null = self; // transfer ownership @@ -1728,7 +1728,7 @@ meth | NULL | arg1 | ... | argN */ Py_DECREF(owner); - if (attr == NULL) JUMP_TO_ERROR; + if (attr == NULL) JUMP_TO_ERROR(); self_or_null = NULL; } } @@ -1736,7 +1736,7 @@ /* Classic, pushes one value. */ attr = PyObject_GetAttr(owner, name); Py_DECREF(owner); - if (attr == NULL) JUMP_TO_ERROR; + if (attr == NULL) JUMP_TO_ERROR(); } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = self_or_null; @@ -1750,7 +1750,7 @@ uint32_t type_version = (uint32_t)CURRENT_OPERAND(); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - if (tp->tp_version_tag != type_version) JUMP_TO_JUMP_TARGET; + if (tp->tp_version_tag != type_version) JUMP_TO_JUMP_TARGET(); break; } @@ -1760,7 +1760,7 @@ assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET; + if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET(); break; } @@ -1773,7 +1773,7 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - if (attr == NULL) JUMP_TO_JUMP_TARGET; + if (attr == NULL) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1791,7 +1791,7 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - if (attr == NULL) JUMP_TO_JUMP_TARGET; + if (attr == NULL) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1808,10 +1808,10 @@ PyObject *owner; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)CURRENT_OPERAND(); - if (!PyModule_CheckExact(owner)) JUMP_TO_JUMP_TARGET; + if (!PyModule_CheckExact(owner)) JUMP_TO_JUMP_TARGET(); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - if (dict->ma_keys->dk_version != dict_version) JUMP_TO_JUMP_TARGET; + if (dict->ma_keys->dk_version != dict_version) JUMP_TO_JUMP_TARGET(); break; } @@ -1827,7 +1827,7 @@ assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; attr = ep->me_value; - if (attr == NULL) JUMP_TO_JUMP_TARGET; + if (attr == NULL) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1843,9 +1843,9 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - if (_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET; + if (_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET(); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - if (dict == NULL) JUMP_TO_JUMP_TARGET; + if (dict == NULL) JUMP_TO_JUMP_TARGET(); assert(PyDict_CheckExact((PyObject *)dict)); break; } @@ -1859,19 +1859,19 @@ uint16_t hint = (uint16_t)CURRENT_OPERAND(); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - if (hint >= (size_t)dict->ma_keys->dk_nentries) JUMP_TO_JUMP_TARGET; + if (hint >= (size_t)dict->ma_keys->dk_nentries) JUMP_TO_JUMP_TARGET(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) JUMP_TO_JUMP_TARGET; + if (ep->me_key != name) JUMP_TO_JUMP_TARGET(); attr = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) JUMP_TO_JUMP_TARGET; + if (ep->me_key != name) JUMP_TO_JUMP_TARGET(); attr = ep->me_value; } - if (attr == NULL) JUMP_TO_JUMP_TARGET; + if (attr == NULL) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1891,7 +1891,7 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); char *addr = (char *)owner + index; attr = *(PyObject **)addr; - if (attr == NULL) JUMP_TO_JUMP_TARGET; + if (attr == NULL) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1909,7 +1909,7 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); char *addr = (char *)owner + index; attr = *(PyObject **)addr; - if (attr == NULL) JUMP_TO_JUMP_TARGET; + if (attr == NULL) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1926,9 +1926,9 @@ PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)CURRENT_OPERAND(); - if (!PyType_Check(owner)) JUMP_TO_JUMP_TARGET; + if (!PyType_Check(owner)) JUMP_TO_JUMP_TARGET(); assert(type_version != 0); - if (((PyTypeObject *)owner)->tp_version_tag != type_version) JUMP_TO_JUMP_TARGET; + if (((PyTypeObject *)owner)->tp_version_tag != type_version) JUMP_TO_JUMP_TARGET(); break; } @@ -1977,7 +1977,7 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET; + if (!_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET(); break; } @@ -2032,11 +2032,11 @@ res = PyObject_RichCompare(left, right, oparg >> 5); Py_DECREF(left); Py_DECREF(right); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); if (oparg & 16) { int res_bool = PyObject_IsTrue(res); Py_DECREF(res); - if (res_bool < 0) JUMP_TO_ERROR; + if (res_bool < 0) JUMP_TO_ERROR(); res = res_bool ? Py_True : Py_False; } stack_pointer[-2] = res; @@ -2072,8 +2072,8 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!_PyLong_IsCompact((PyLongObject *)left)) JUMP_TO_JUMP_TARGET; - if (!_PyLong_IsCompact((PyLongObject *)right)) JUMP_TO_JUMP_TARGET; + if (!_PyLong_IsCompact((PyLongObject *)left)) JUMP_TO_JUMP_TARGET(); + if (!_PyLong_IsCompact((PyLongObject *)right)) JUMP_TO_JUMP_TARGET(); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && _PyLong_DigitCount((PyLongObject *)right) <= 1); @@ -2138,7 +2138,7 @@ int res = PySequence_Contains(right, left); Py_DECREF(left); Py_DECREF(right); - if (res < 0) JUMP_TO_ERROR; + if (res < 0) JUMP_TO_ERROR(); b = (res ^ oparg) ? Py_True : Py_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -2152,13 +2152,13 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))) JUMP_TO_JUMP_TARGET; + if (!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))) JUMP_TO_JUMP_TARGET(); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! int res = _PySet_Contains((PySetObject *)right, left); Py_DECREF(left); Py_DECREF(right); - if (res < 0) JUMP_TO_ERROR; + if (res < 0) JUMP_TO_ERROR(); b = (res ^ oparg) ? Py_True : Py_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -2172,12 +2172,12 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyDict_CheckExact(right)) JUMP_TO_JUMP_TARGET; + if (!PyDict_CheckExact(right)) JUMP_TO_JUMP_TARGET(); STAT_INC(CONTAINS_OP, hit); int res = PyDict_Contains(right, left); Py_DECREF(left); Py_DECREF(right); - if (res < 0) JUMP_TO_ERROR; + if (res < 0) JUMP_TO_ERROR(); b = (res ^ oparg) ? Py_True : Py_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -2194,7 +2194,7 @@ if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { Py_DECREF(exc_value); Py_DECREF(match_type); - if (true) JUMP_TO_ERROR; + if (true) JUMP_TO_ERROR(); } match = NULL; rest = NULL; @@ -2202,9 +2202,9 @@ &match, &rest); Py_DECREF(exc_value); Py_DECREF(match_type); - if (res < 0) JUMP_TO_ERROR; + if (res < 0) JUMP_TO_ERROR(); assert((match == NULL) == (rest == NULL)); - if (match == NULL) JUMP_TO_ERROR; + if (match == NULL) JUMP_TO_ERROR(); if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } @@ -2222,7 +2222,7 @@ assert(PyExceptionInstance_Check(left)); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { Py_DECREF(right); - if (true) JUMP_TO_ERROR; + if (true) JUMP_TO_ERROR(); } int res = PyErr_GivenExceptionMatches(left, right); Py_DECREF(right); @@ -2256,9 +2256,9 @@ obj = stack_pointer[-1]; // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); - if (len_i < 0) JUMP_TO_ERROR; + if (len_i < 0) JUMP_TO_ERROR(); len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) JUMP_TO_ERROR; + if (len_o == NULL) JUMP_TO_ERROR(); stack_pointer[0] = len_o; stack_pointer += 1; break; @@ -2284,7 +2284,7 @@ assert(PyTuple_CheckExact(attrs)); // Success! } else { - if (_PyErr_Occurred(tstate)) JUMP_TO_ERROR; + if (_PyErr_Occurred(tstate)) JUMP_TO_ERROR(); // Error! attrs = Py_None; // Failure! } @@ -2323,7 +2323,7 @@ subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = _PyEval_MatchKeys(tstate, subject, keys); - if (values_or_none == NULL) JUMP_TO_ERROR; + if (values_or_none == NULL) JUMP_TO_ERROR(); stack_pointer[0] = values_or_none; stack_pointer += 1; break; @@ -2336,7 +2336,7 @@ /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); Py_DECREF(iterable); - if (iter == NULL) JUMP_TO_ERROR; + if (iter == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = iter; break; } @@ -2354,7 +2354,7 @@ _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } iter = iterable; } @@ -2365,7 +2365,7 @@ /* `iterable` is not a generator. */ iter = PyObject_GetIter(iterable); if (iter == NULL) { - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } Py_DECREF(iterable); } @@ -2384,7 +2384,7 @@ if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } _PyErr_Clear(tstate); } @@ -2392,7 +2392,7 @@ Py_DECREF(iter); STACK_SHRINK(1); /* The translator sets the deopt target just past END_FOR */ - if (true) JUMP_TO_JUMP_TARGET; + if (true) JUMP_TO_JUMP_TARGET(); } // Common case: no jump, leave it to the code generator stack_pointer[0] = next; @@ -2405,7 +2405,7 @@ case _ITER_CHECK_LIST: { PyObject *iter; iter = stack_pointer[-1]; - if (Py_TYPE(iter) != &PyListIter_Type) JUMP_TO_JUMP_TARGET; + if (Py_TYPE(iter) != &PyListIter_Type) JUMP_TO_JUMP_TARGET(); break; } @@ -2417,8 +2417,8 @@ _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; - if (seq == NULL) JUMP_TO_JUMP_TARGET; - if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) JUMP_TO_JUMP_TARGET; + if (seq == NULL) JUMP_TO_JUMP_TARGET(); + if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) JUMP_TO_JUMP_TARGET(); break; } @@ -2440,7 +2440,7 @@ case _ITER_CHECK_TUPLE: { PyObject *iter; iter = stack_pointer[-1]; - if (Py_TYPE(iter) != &PyTupleIter_Type) JUMP_TO_JUMP_TARGET; + if (Py_TYPE(iter) != &PyTupleIter_Type) JUMP_TO_JUMP_TARGET(); break; } @@ -2452,8 +2452,8 @@ _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; - if (seq == NULL) JUMP_TO_JUMP_TARGET; - if (it->it_index >= PyTuple_GET_SIZE(seq)) JUMP_TO_JUMP_TARGET; + if (seq == NULL) JUMP_TO_JUMP_TARGET(); + if (it->it_index >= PyTuple_GET_SIZE(seq)) JUMP_TO_JUMP_TARGET(); break; } @@ -2476,7 +2476,7 @@ PyObject *iter; iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - if (Py_TYPE(r) != &PyRangeIter_Type) JUMP_TO_JUMP_TARGET; + if (Py_TYPE(r) != &PyRangeIter_Type) JUMP_TO_JUMP_TARGET(); break; } @@ -2487,7 +2487,7 @@ iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); - if (r->len <= 0) JUMP_TO_JUMP_TARGET; + if (r->len <= 0) JUMP_TO_JUMP_TARGET(); break; } @@ -2502,7 +2502,7 @@ r->start = value + r->step; r->len--; next = PyLong_FromLong(value); - if (next == NULL) JUMP_TO_ERROR; + if (next == NULL) JUMP_TO_ERROR(); stack_pointer[0] = next; stack_pointer += 1; break; @@ -2545,7 +2545,7 @@ PyObject *stack[4] = {NULL, exc, val, tb}; res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[0] = res; stack_pointer += 1; break; @@ -2575,7 +2575,7 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET; + if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET(); break; } @@ -2585,7 +2585,7 @@ uint32_t keys_version = (uint32_t)CURRENT_OPERAND(); PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - if (owner_heap_type->ht_cached_keys->dk_version != keys_version) JUMP_TO_JUMP_TARGET; + if (owner_heap_type->ht_cached_keys->dk_version != keys_version) JUMP_TO_JUMP_TARGET(); break; } @@ -2667,7 +2667,7 @@ assert(dictoffset > 0); PyObject *dict = *(PyObject **)((char *)owner + dictoffset); /* This object has a __dict__, just not yet created */ - if (dict != NULL) JUMP_TO_JUMP_TARGET; + if (dict != NULL) JUMP_TO_JUMP_TARGET(); break; } @@ -2705,8 +2705,8 @@ oparg = CURRENT_OPARG(); null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - if (null != NULL) JUMP_TO_JUMP_TARGET; - if (Py_TYPE(callable) != &PyMethod_Type) JUMP_TO_JUMP_TARGET; + if (null != NULL) JUMP_TO_JUMP_TARGET(); + if (Py_TYPE(callable) != &PyMethod_Type) JUMP_TO_JUMP_TARGET(); break; } @@ -2728,7 +2728,7 @@ } case _CHECK_PEP_523: { - if (tstate->interp->eval_frame) JUMP_TO_JUMP_TARGET; + if (tstate->interp->eval_frame) JUMP_TO_JUMP_TARGET(); break; } @@ -2739,11 +2739,11 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND(); - if (!PyFunction_Check(callable)) JUMP_TO_JUMP_TARGET; + if (!PyFunction_Check(callable)) JUMP_TO_JUMP_TARGET(); PyFunctionObject *func = (PyFunctionObject *)callable; - if (func->func_version != func_version) JUMP_TO_JUMP_TARGET; + if (func->func_version != func_version) JUMP_TO_JUMP_TARGET(); PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (self_or_null != NULL)) JUMP_TO_JUMP_TARGET; + if (code->co_argcount != oparg + (self_or_null != NULL)) JUMP_TO_JUMP_TARGET(); break; } @@ -2753,8 +2753,8 @@ callable = stack_pointer[-2 - oparg]; PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) JUMP_TO_JUMP_TARGET; - if (tstate->py_recursion_remaining <= 1) JUMP_TO_JUMP_TARGET; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) JUMP_TO_JUMP_TARGET(); + if (tstate->py_recursion_remaining <= 1) JUMP_TO_JUMP_TARGET(); break; } @@ -2936,8 +2936,8 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; assert(oparg == 1); - if (null != NULL) JUMP_TO_JUMP_TARGET; - if (callable != (PyObject *)&PyType_Type) JUMP_TO_JUMP_TARGET; + if (null != NULL) JUMP_TO_JUMP_TARGET(); + if (callable != (PyObject *)&PyType_Type) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); res = Py_NewRef(Py_TYPE(arg)); Py_DECREF(arg); @@ -2956,12 +2956,12 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; assert(oparg == 1); - if (null != NULL) JUMP_TO_JUMP_TARGET; - if (callable != (PyObject *)&PyUnicode_Type) JUMP_TO_JUMP_TARGET; + if (null != NULL) JUMP_TO_JUMP_TARGET(); + if (callable != (PyObject *)&PyUnicode_Type) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); res = PyObject_Str(arg); Py_DECREF(arg); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-3] = res; stack_pointer += -2; break; @@ -2977,12 +2977,12 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; assert(oparg == 1); - if (null != NULL) JUMP_TO_JUMP_TARGET; - if (callable != (PyObject *)&PyTuple_Type) JUMP_TO_JUMP_TARGET; + if (null != NULL) JUMP_TO_JUMP_TARGET(); + if (callable != (PyObject *)&PyTuple_Type) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); res = PySequence_Tuple(arg); Py_DECREF(arg); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-3] = res; stack_pointer += -2; break; @@ -2998,7 +2998,7 @@ PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(should_be_none)->tp_name); - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } stack_pointer += -1; break; @@ -3018,9 +3018,9 @@ args--; total_args++; } - if (!PyType_Check(callable)) JUMP_TO_JUMP_TARGET; + if (!PyType_Check(callable)) JUMP_TO_JUMP_TARGET(); PyTypeObject *tp = (PyTypeObject *)callable; - if (tp->tp_vectorcall == NULL) JUMP_TO_JUMP_TARGET; + if (tp->tp_vectorcall == NULL) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); /* Free the arguments. */ @@ -3028,7 +3028,7 @@ Py_DECREF(args[i]); } Py_DECREF(tp); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3049,11 +3049,11 @@ args--; total_args++; } - if (total_args != 1) JUMP_TO_JUMP_TARGET; - if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET; - if (PyCFunction_GET_FLAGS(callable) != METH_O) JUMP_TO_JUMP_TARGET; + if (total_args != 1) JUMP_TO_JUMP_TARGET(); + if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET(); + if (PyCFunction_GET_FLAGS(callable) != METH_O) JUMP_TO_JUMP_TARGET(); // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET; + if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); PyObject *arg = args[0]; @@ -3063,7 +3063,7 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3084,8 +3084,8 @@ args--; total_args++; } - if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET; - if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) JUMP_TO_JUMP_TARGET; + if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET(); + if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ @@ -3099,7 +3099,7 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3120,8 +3120,8 @@ args--; total_args++; } - if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET; - if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) JUMP_TO_JUMP_TARGET; + if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET(); + if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ PyCFunctionFastWithKeywords cfunc = @@ -3134,7 +3134,7 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3155,14 +3155,14 @@ args--; total_args++; } - if (total_args != 1) JUMP_TO_JUMP_TARGET; + if (total_args != 1) JUMP_TO_JUMP_TARGET(); PyInterpreterState *interp = tstate->interp; - if (callable != interp->callable_cache.len) JUMP_TO_JUMP_TARGET; + if (callable != interp->callable_cache.len) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3191,15 +3191,15 @@ args--; total_args++; } - if (total_args != 2) JUMP_TO_JUMP_TARGET; + if (total_args != 2) JUMP_TO_JUMP_TARGET(); PyInterpreterState *interp = tstate->interp; - if (callable != interp->callable_cache.isinstance) JUMP_TO_JUMP_TARGET; + if (callable != interp->callable_cache.isinstance) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); PyObject *cls = args[1]; PyObject *inst = args[0]; int retval = PyObject_IsInstance(inst, cls); if (retval < 0) { - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3229,15 +3229,15 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (total_args != 2) JUMP_TO_JUMP_TARGET; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET; + if (total_args != 2) JUMP_TO_JUMP_TARGET(); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_O) JUMP_TO_JUMP_TARGET; + if (meth->ml_flags != METH_O) JUMP_TO_JUMP_TARGET(); // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET; + if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET(); PyObject *arg = args[1]; PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET; + if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -3247,7 +3247,7 @@ Py_DECREF(self); Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3268,12 +3268,12 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); PyMethodDef *meth = method->d_method; - if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) JUMP_TO_JUMP_TARGET; + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) JUMP_TO_JUMP_TARGET(); PyTypeObject *d_type = method->d_common.d_type; PyObject *self = args[0]; - if (!Py_IS_TYPE(self, d_type)) JUMP_TO_JUMP_TARGET; + if (!Py_IS_TYPE(self, d_type)) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); int nargs = total_args - 1; PyCFunctionFastWithKeywords cfunc = @@ -3285,7 +3285,7 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3306,15 +3306,15 @@ args--; total_args++; } - if (total_args != 1) JUMP_TO_JUMP_TARGET; + if (total_args != 1) JUMP_TO_JUMP_TARGET(); PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); PyMethodDef *meth = method->d_method; PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET; - if (meth->ml_flags != METH_NOARGS) JUMP_TO_JUMP_TARGET; + if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET(); + if (meth->ml_flags != METH_NOARGS) JUMP_TO_JUMP_TARGET(); // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET; + if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -3323,7 +3323,7 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3345,11 +3345,11 @@ } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; /* Builtin METH_FASTCALL methods, without keywords */ - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_FASTCALL) JUMP_TO_JUMP_TARGET; + if (meth->ml_flags != METH_FASTCALL) JUMP_TO_JUMP_TARGET(); PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET; + if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; @@ -3361,7 +3361,7 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3383,7 +3383,7 @@ PyFunction_New(codeobj, GLOBALS()); Py_DECREF(codeobj); if (func_obj == NULL) { - JUMP_TO_ERROR; + JUMP_TO_ERROR(); } _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); @@ -3440,7 +3440,7 @@ Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - if (slice == NULL) JUMP_TO_ERROR; + if (slice == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); break; @@ -3456,7 +3456,7 @@ conv_fn = _PyEval_ConversionFuncs[oparg]; result = conv_fn(value); Py_DECREF(value); - if (result == NULL) JUMP_TO_ERROR; + if (result == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = result; break; } @@ -3470,7 +3470,7 @@ if (!PyUnicode_CheckExact(value)) { res = PyObject_Format(value, NULL); Py_DECREF(value); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); } else { res = value; @@ -3488,7 +3488,7 @@ res = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_DECREF(fmt_spec); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -3517,7 +3517,7 @@ res = _PyEval_BinaryOps[oparg](lhs, rhs); Py_DECREF(lhs); Py_DECREF(rhs); - if (res == NULL) JUMP_TO_ERROR; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -3553,7 +3553,7 @@ PyObject *flag; flag = stack_pointer[-1]; stack_pointer += -1; - if (!Py_IsTrue(flag)) JUMP_TO_JUMP_TARGET; + if (!Py_IsTrue(flag)) JUMP_TO_JUMP_TARGET(); assert(Py_IsTrue(flag)); break; } @@ -3562,7 +3562,7 @@ PyObject *flag; flag = stack_pointer[-1]; stack_pointer += -1; - if (!Py_IsFalse(flag)) JUMP_TO_JUMP_TARGET; + if (!Py_IsFalse(flag)) JUMP_TO_JUMP_TARGET(); assert(Py_IsFalse(flag)); break; } @@ -3573,7 +3573,7 @@ stack_pointer += -1; if (!Py_IsNone(val)) { Py_DECREF(val); - if (1) JUMP_TO_JUMP_TARGET; + if (1) JUMP_TO_JUMP_TARGET(); } break; } @@ -3582,7 +3582,7 @@ PyObject *val; val = stack_pointer[-1]; stack_pointer += -1; - if (Py_IsNone(val)) JUMP_TO_JUMP_TARGET; + if (Py_IsNone(val)) JUMP_TO_JUMP_TARGET(); Py_DECREF(val); break; } @@ -3613,12 +3613,12 @@ } case _EXIT_TRACE: { - if (1) JUMP_TO_JUMP_TARGET; + if (1) JUMP_TO_JUMP_TARGET(); break; } case _CHECK_VALIDITY: { - if (!current_executor->vm_data.valid) JUMP_TO_JUMP_TARGET; + if (!current_executor->vm_data.valid) JUMP_TO_JUMP_TARGET(); break; } @@ -3678,7 +3678,7 @@ case _CHECK_FUNCTION: { uint32_t func_version = (uint32_t)CURRENT_OPERAND(); assert(PyFunction_Check(frame->f_funcobj)); - if (((PyFunctionObject *)frame->f_funcobj)->func_version != func_version) JUMP_TO_JUMP_TARGET; + if (((PyFunctionObject *)frame->f_funcobj)->func_version != func_version) JUMP_TO_JUMP_TARGET(); break; } @@ -3744,7 +3744,7 @@ case _CHECK_VALIDITY_AND_SET_IP: { PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND(); - if (!current_executor->vm_data.valid) JUMP_TO_JUMP_TARGET; + if (!current_executor->vm_data.valid) JUMP_TO_JUMP_TARGET(); frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; break; } diff --git a/Python/optimizer.c b/Python/optimizer.c index e5ca7df533a785..c1a4e3534f62ca 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -952,7 +952,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) buffer[i].format = UOP_FORMAT_JUMP; } if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) { - int popped = (_PyUop_Flags[opcode] & HAS_NO_POP_ERROR_FLAG) ? + int popped = (_PyUop_Flags[opcode] & HAS_ERROR_NO_POP_FLAG) ? 0 : _PyUop_num_popped(opcode, inst->oparg); if (target != current_error_target || popped != current_popped) { current_popped = popped; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 219d023e01cd76..45c7991064efa7 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -350,7 +350,7 @@ def has_pop_error(op: parser.InstDef) -> bool: def has_no_pop_error(op: parser.InstDef) -> bool: return ( - variable_used(op, "NO_POP_ERROR") + variable_used(op, "ERROR_NO_POP") or variable_used(op, "pop_1_error") or variable_used(op, "exception_unwind") or variable_used(op, "resume_with_error") diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 679ecd1197b3d5..c9ea162575693f 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -99,7 +99,7 @@ def replace_error( out.emit(close) -def replace_no_pop_error( +def replace_error_no_pop( out: CWriter, tkn: Token, tkn_iter: Iterator[Token], @@ -174,7 +174,7 @@ def replace_check_eval_breaker( "EXIT_IF": replace_deopt, "DEOPT_IF": replace_deopt, "ERROR_IF": replace_error, - "NO_POP_ERROR": replace_no_pop_error, + "ERROR_NO_POP": replace_error_no_pop, "DECREF_INPUTS": replace_decrefs, "CHECK_EVAL_BREAKER": replace_check_eval_breaker, "SYNC_SP": replace_sync_sp, @@ -229,7 +229,7 @@ def cflags(p: Properties) -> str: if not p.infallible: flags.append("HAS_ERROR_FLAG") if p.no_pop_error: - flags.append("HAS_NO_POP_ERROR_FLAG") + flags.append("HAS_ERROR_NO_POP_FLAG") if p.escapes: flags.append("HAS_ESCAPES_FLAG") if p.pure: diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 6df1852d8a46de..04fecb235f18cd 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -54,7 +54,7 @@ "PURE", "PASSTHROUGH", "OPARG_AND_1", - "NO_POP_ERROR", + "ERROR_NO_POP", ] diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 57ac730a1abfa6..114d28ee745632 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -72,10 +72,10 @@ def tier2_replace_error( label = next(tkn_iter).text next(tkn_iter) # RPAREN next(tkn_iter) # Semi colon - out.emit(") JUMP_TO_ERROR;\n") + out.emit(") JUMP_TO_ERROR();\n") -def tier2_replace_no_pop_error( +def tier2_replace_error_no_pop( out: CWriter, tkn: Token, tkn_iter: Iterator[Token], @@ -86,7 +86,7 @@ def tier2_replace_no_pop_error( next(tkn_iter) # LPAREN next(tkn_iter) # RPAREN next(tkn_iter) # Semi colon - out.emit_at("JUMP_TO_ERROR;", tkn) + out.emit_at("JUMP_TO_ERROR();", tkn) def tier2_replace_deopt( out: CWriter, @@ -100,7 +100,7 @@ def tier2_replace_deopt( out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "RPAREN") next(tkn_iter) # Semi colon - out.emit(") JUMP_TO_JUMP_TARGET;\n") + out.emit(") JUMP_TO_JUMP_TARGET();\n") def tier2_replace_exit_if( @@ -115,7 +115,7 @@ def tier2_replace_exit_if( out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "RPAREN") next(tkn_iter) # Semi colon - out.emit(") JUMP_TO_JUMP_TARGET;\n") + out.emit(") JUMP_TO_JUMP_TARGET();\n") def tier2_replace_oparg( @@ -141,7 +141,7 @@ def tier2_replace_oparg( TIER2_REPLACEMENT_FUNCTIONS = REPLACEMENT_FUNCTIONS.copy() TIER2_REPLACEMENT_FUNCTIONS["ERROR_IF"] = tier2_replace_error -TIER2_REPLACEMENT_FUNCTIONS["NO_POP_ERROR"] = tier2_replace_no_pop_error +TIER2_REPLACEMENT_FUNCTIONS["ERROR_NO_POP"] = tier2_replace_error_no_pop TIER2_REPLACEMENT_FUNCTIONS["DEOPT_IF"] = tier2_replace_deopt TIER2_REPLACEMENT_FUNCTIONS["oparg"] = tier2_replace_oparg TIER2_REPLACEMENT_FUNCTIONS["EXIT_IF"] = tier2_replace_exit_if diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 42137d9ce244a8..2166ecc6598d2d 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -75,10 +75,10 @@ do { \ } while (0) #undef JUMP_TO_JUMP_TARGET -#define JUMP_TO_JUMP_TARGET PATCH_JUMP(_JIT_JUMP_TARGET) +#define JUMP_TO_JUMP_TARGET() PATCH_JUMP(_JIT_JUMP_TARGET) #undef JUMP_TO_ERROR -#define JUMP_TO_ERROR PATCH_JUMP(_JIT_ERROR_TARGET) +#define JUMP_TO_ERROR() PATCH_JUMP(_JIT_ERROR_TARGET) _Py_CODEUNIT * _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate) From 3c4869b2b1577774743c0d56c2e291b4ed03a3fd Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 20 Mar 2024 10:31:18 +0000 Subject: [PATCH 20/24] Patch error jumps --- Python/jit.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Python/jit.c b/Python/jit.c index 78990805e435d9..937c932b51bdf2 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -440,7 +440,11 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size patches[HoleValue_TARGET] = instruction->target; break; case UOP_FORMAT_EXIT: + assert(instruction->exit_index < executor->exit_count); patches[HoleValue_EXIT_INDEX] = instruction->exit_index; + if (instruction->error_target < length) { + patches[HoleValue_ERROR_TARGET] = (uint64_t)memory + instruction_starts[instruction->error_target]; + } break; case UOP_FORMAT_JUMP: assert(instruction->jump_target < length); From b32493319b6828fe1fdcd7c5ddf7a339a5d88a34 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 20 Mar 2024 10:41:11 +0000 Subject: [PATCH 21/24] Make assert a fatal error --- Python/jit.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/jit.c b/Python/jit.c index 937c932b51bdf2..4e650df6b8c82d 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -455,6 +455,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size break; default: assert(0); + Py_FatalError("Illegal instruction format"); } patches[HoleValue_TOP] = (uint64_t)memory + instruction_starts[1]; patches[HoleValue_ZERO] = 0; From 71a2cae223d1873b049ea13574a1f7da7119415e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 20 Mar 2024 10:41:49 +0000 Subject: [PATCH 22/24] Rename analyzer attributes for clarity --- Python/optimizer_analysis.c | 3 +- Tools/cases_generator/analyzer.py | 32 +++++++++++----------- Tools/cases_generator/generators_common.py | 2 +- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 578406bbc39998..cdd46c61ee4a5b 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -400,7 +400,8 @@ optimize_uops( assert(STACK_LEVEL() >= 0); } if (this_instr != trace + trace_len) { - assert (this_instr < trace + trace_len && this_instr > trace); + assert(this_instr > trace); + assert(this_instr < trace + trace_len); trace_len = (int)(this_instr - trace + 1); } _Py_uop_abstractcontext_fini(ctx); diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 45c7991064efa7..2329205ad31d09 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -8,8 +8,8 @@ @dataclass class Properties: escapes: bool - pop_error: bool - no_pop_error: bool + error_with_pop: bool + error_without_pop: bool deopts: bool oparg: bool jumps: bool @@ -38,8 +38,8 @@ def dump(self, indent: str) -> None: def from_list(properties: list["Properties"]) -> "Properties": return Properties( escapes=any(p.escapes for p in properties), - pop_error=any(p.pop_error for p in properties), - no_pop_error=any(p.no_pop_error for p in properties), + error_with_pop=any(p.error_with_pop for p in properties), + error_without_pop=any(p.error_without_pop for p in properties), deopts=any(p.deopts for p in properties), oparg=any(p.oparg for p in properties), jumps=any(p.jumps for p in properties), @@ -59,14 +59,14 @@ def from_list(properties: list["Properties"]) -> "Properties": @property def infallible(self) -> bool: - return not self.pop_error and not self.no_pop_error + return not self.error_with_pop and not self.error_without_pop SKIP_PROPERTIES = Properties( escapes=False, - pop_error=False, - no_pop_error=False, + error_with_pop=False, + error_without_pop=False, deopts=False, oparg=False, jumps=False, @@ -178,10 +178,10 @@ def why_not_viable(self) -> str | None: return "uses the 'this_instr' variable" if len([c for c in self.caches if c.name != "unused"]) > 1: return "has unused cache entries" - if self.properties.pop_error and self.properties.no_pop_error: + if self.properties.error_with_pop and self.properties.error_without_pop: return "has both popping and not-popping errors" if self.properties.eval_breaker: - if self.properties.pop_error or self.properties.no_pop_error: + if self.properties.error_with_pop or self.properties.error_without_pop: return "has error handling and eval-breaker check" if self.properties.side_exit: return "exits and eval-breaker check" @@ -340,7 +340,7 @@ def tier_variable(node: parser.InstDef) -> int | None: return int(token.text[-1]) return None -def has_pop_error(op: parser.InstDef) -> bool: +def has_error_with_pop(op: parser.InstDef) -> bool: return ( variable_used(op, "ERROR_IF") or variable_used(op, "pop_1_error") @@ -348,7 +348,7 @@ def has_pop_error(op: parser.InstDef) -> bool: or variable_used(op, "resume_with_error") ) -def has_no_pop_error(op: parser.InstDef) -> bool: +def has_error_without_pop(op: parser.InstDef) -> bool: return ( variable_used(op, "ERROR_NO_POP") or variable_used(op, "pop_1_error") @@ -534,14 +534,14 @@ def compute_properties(op: parser.InstDef) -> Properties: tkn.column, op.name, ) - pop_error = has_pop_error(op) - no_pop_error = has_no_pop_error(op) - infallible = not pop_error and not no_pop_error + error_with_pop = has_error_with_pop(op) + error_without_pop = has_error_without_pop(op) + infallible = not error_with_pop and not error_without_pop passthrough = stack_effect_only_peeks(op) and infallible return Properties( escapes=makes_escaping_api_call(op), - pop_error=pop_error, - no_pop_error=no_pop_error, + error_with_pop=error_with_pop, + error_without_pop=error_without_pop, deopts=deopts_if, side_exit=exits_if, oparg=variable_used(op, "oparg"), diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index c9ea162575693f..0addcf0ab570f6 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -228,7 +228,7 @@ def cflags(p: Properties) -> str: flags.append("HAS_EXIT_FLAG") if not p.infallible: flags.append("HAS_ERROR_FLAG") - if p.no_pop_error: + if p.error_without_pop: flags.append("HAS_ERROR_NO_POP_FLAG") if p.escapes: flags.append("HAS_ESCAPES_FLAG") From 3f5dc4d306d965ae2763f4d85488d2dd30756eec Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 20 Mar 2024 10:47:08 +0000 Subject: [PATCH 23/24] Remove redundant macros --- Tools/jit/template.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 2166ecc6598d2d..9b4fc2af9671eb 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -31,10 +31,6 @@ } \ } while (0) -#define GOTO_UNWIND() goto error_tier_two -#define EXIT_TO_TRACE() goto exit_to_trace -#define EXIT_TO_TIER1() goto exit_to_tier1 - #undef ENABLE_SPECIALIZATION #define ENABLE_SPECIALIZATION (0) From 151db6a45ccb0cf676928058fc95870d189a2d32 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 25 Mar 2024 16:59:18 +0000 Subject: [PATCH 24/24] Address code review --- Python/bytecodes.c | 3 +-- Python/ceval.c | 2 -- Python/executor_cases.c.h | 3 --- Python/optimizer.c | 5 ++++- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0c52271ef06c48..5cd9db97c71e37 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4215,8 +4215,7 @@ dummy_func( EXIT_TO_TRACE(); } - tier2 op(_ERROR_POP_N, (values[oparg] --)) { - (void)values; + tier2 op(_ERROR_POP_N, (unused[oparg] --)) { SYNC_SP(); GOTO_UNWIND(); } diff --git a/Python/ceval.c b/Python/ceval.c index af4e4153a4ecfe..cd51011450c3d5 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -970,8 +970,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #undef GOTO_ERROR #define GOTO_ERROR(LABEL) goto LABEL ## _tier_two -#undef DEOPT_IF - #ifdef Py_STATS // Disable these macros that apply to Tier 1 stats when we are in Tier 2 #undef STAT_INC diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index dcbd973c3721a6..224b600b8f6a4a 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3760,10 +3760,7 @@ } case _ERROR_POP_N: { - PyObject **values; oparg = CURRENT_OPARG(); - values = &stack_pointer[-oparg]; - (void)values; stack_pointer += -oparg; GOTO_UNWIND(); break; diff --git a/Python/optimizer.c b/Python/optimizer.c index d6b41b08f2ce2f..92022651619990 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -159,7 +159,10 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil static int init_cold_exit_executor(_PyExecutorObject *executor, int oparg); -/* The maximum number of exits in a trace cannot reach 1/4 of its length */ +/* It is impossible for the number of exits to reach 1/4 of the total length, + * as the number of exits cannot reach 1/3 of the number of non-exits, due to + * the presence of CHECK_VALIDITY checks and instructions to produce the values + * being checked in exits. */ #define COLD_EXIT_COUNT (UOP_MAX_TRACE_LENGTH/4) static int cold_exits_initialized = 0;