From 2dc5e71d537bb215d343d4233d1b6f8b6a76d11f Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 20 Oct 2022 02:07:03 -0700 Subject: [PATCH 01/21] Combine all unquickened and adaptive instructions --- Include/internal/pycore_code.h | 2 - Include/internal/pycore_opcode.h | 120 ++++----- Include/opcode.h | 132 +++++----- Lib/dis.py | 5 +- Lib/opcode.py | 10 - Lib/test/test__opcode.py | 7 +- Lib/test/test_dis.py | 4 +- Python/bytecodes.c | 362 ++++++++++++--------------- Python/ceval.c | 4 +- Python/generated_cases.c.h | 341 +++++++++++-------------- Python/opcode_targets.h | 100 ++++---- Python/specialize.c | 59 ++--- Tools/c-analyzer/cpython/ignored.tsv | 1 - 13 files changed, 503 insertions(+), 644 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index cf8573aa9138b2..17a2f05072a5e9 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -91,8 +91,6 @@ typedef struct { #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache) -extern uint8_t _PyOpcode_Adaptive[256]; - // Borrowed references to common callables: struct callable_cache { PyObject *isinstance; diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 59276c8097116f..3ec937d975e51a 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -58,7 +58,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [BEFORE_ASYNC_WITH] = BEFORE_ASYNC_WITH, [BEFORE_WITH] = BEFORE_WITH, [BINARY_OP] = BINARY_OP, - [BINARY_OP_ADAPTIVE] = BINARY_OP, [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, [BINARY_OP_ADD_UNICODE] = BINARY_OP, @@ -69,7 +68,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [BINARY_OP_SUBTRACT_INT] = BINARY_OP, [BINARY_SLICE] = BINARY_SLICE, [BINARY_SUBSCR] = BINARY_SUBSCR, - [BINARY_SUBSCR_ADAPTIVE] = BINARY_SUBSCR, [BINARY_SUBSCR_DICT] = BINARY_SUBSCR, [BINARY_SUBSCR_GETITEM] = BINARY_SUBSCR, [BINARY_SUBSCR_LIST_INT] = BINARY_SUBSCR, @@ -83,7 +81,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [BUILD_TUPLE] = BUILD_TUPLE, [CACHE] = CACHE, [CALL] = CALL, - [CALL_ADAPTIVE] = CALL, [CALL_BOUND_METHOD_EXACT_ARGS] = CALL, [CALL_BUILTIN_CLASS] = CALL, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, @@ -106,7 +103,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [CHECK_EXC_MATCH] = CHECK_EXC_MATCH, [CLEANUP_THROW] = CLEANUP_THROW, [COMPARE_OP] = COMPARE_OP, - [COMPARE_OP_ADAPTIVE] = COMPARE_OP, [COMPARE_OP_FLOAT_JUMP] = COMPARE_OP, [COMPARE_OP_INT_JUMP] = COMPARE_OP, [COMPARE_OP_STR_JUMP] = COMPARE_OP, @@ -127,7 +123,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [EXTENDED_ARG_QUICK] = EXTENDED_ARG, [FORMAT_VALUE] = FORMAT_VALUE, [FOR_ITER] = FOR_ITER, - [FOR_ITER_ADAPTIVE] = FOR_ITER, [FOR_ITER_LIST] = FOR_ITER, [FOR_ITER_RANGE] = FOR_ITER, [GET_AITER] = GET_AITER, @@ -151,7 +146,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [LIST_TO_TUPLE] = LIST_TO_TUPLE, [LOAD_ASSERTION_ERROR] = LOAD_ASSERTION_ERROR, [LOAD_ATTR] = LOAD_ATTR, - [LOAD_ATTR_ADAPTIVE] = LOAD_ATTR, [LOAD_ATTR_CLASS] = LOAD_ATTR, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = LOAD_ATTR, [LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR, @@ -174,7 +168,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_FAST__LOAD_CONST] = LOAD_FAST, [LOAD_FAST__LOAD_FAST] = LOAD_FAST, [LOAD_GLOBAL] = LOAD_GLOBAL, - [LOAD_GLOBAL_ADAPTIVE] = LOAD_GLOBAL, [LOAD_GLOBAL_BUILTIN] = LOAD_GLOBAL, [LOAD_GLOBAL_MODULE] = LOAD_GLOBAL, [LOAD_NAME] = LOAD_NAME, @@ -207,7 +200,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [SET_UPDATE] = SET_UPDATE, [STOPITERATION_ERROR] = STOPITERATION_ERROR, [STORE_ATTR] = STORE_ATTR, - [STORE_ATTR_ADAPTIVE] = STORE_ATTR, [STORE_ATTR_INSTANCE_VALUE] = STORE_ATTR, [STORE_ATTR_SLOT] = STORE_ATTR, [STORE_ATTR_WITH_HINT] = STORE_ATTR, @@ -219,7 +211,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [STORE_NAME] = STORE_NAME, [STORE_SLICE] = STORE_SLICE, [STORE_SUBSCR] = STORE_SUBSCR, - [STORE_SUBSCR_ADAPTIVE] = STORE_SUBSCR, [STORE_SUBSCR_DICT] = STORE_SUBSCR, [STORE_SUBSCR_LIST_INT] = STORE_SUBSCR, [SWAP] = SWAP, @@ -229,7 +220,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [UNARY_POSITIVE] = UNARY_POSITIVE, [UNPACK_EX] = UNPACK_EX, [UNPACK_SEQUENCE] = UNPACK_SEQUENCE, - [UNPACK_SEQUENCE_ADAPTIVE] = UNPACK_SEQUENCE, [UNPACK_SEQUENCE_LIST] = UNPACK_SEQUENCE, [UNPACK_SEQUENCE_TUPLE] = UNPACK_SEQUENCE, [UNPACK_SEQUENCE_TWO_TUPLE] = UNPACK_SEQUENCE, @@ -243,44 +233,41 @@ static const char *const _PyOpcode_OpName[263] = { [CACHE] = "CACHE", [POP_TOP] = "POP_TOP", [PUSH_NULL] = "PUSH_NULL", - [BINARY_OP_ADAPTIVE] = "BINARY_OP_ADAPTIVE", - [END_FOR] = "END_FOR", [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", + [END_FOR] = "END_FOR", [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", + [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", [NOP] = "NOP", [UNARY_POSITIVE] = "UNARY_POSITIVE", [UNARY_NEGATIVE] = "UNARY_NEGATIVE", [UNARY_NOT] = "UNARY_NOT", - [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", - [UNARY_INVERT] = "UNARY_INVERT", [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", + [UNARY_INVERT] = "UNARY_INVERT", [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", - [BINARY_SUBSCR_ADAPTIVE] = "BINARY_SUBSCR_ADAPTIVE", [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", - [CALL_ADAPTIVE] = "CALL_ADAPTIVE", [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", + [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", + [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", + [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [BINARY_SUBSCR] = "BINARY_SUBSCR", [BINARY_SLICE] = "BINARY_SLICE", [STORE_SLICE] = "STORE_SLICE", - [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", - [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [GET_LEN] = "GET_LEN", [MATCH_MAPPING] = "MATCH_MAPPING", [MATCH_SEQUENCE] = "MATCH_SEQUENCE", [MATCH_KEYS] = "MATCH_KEYS", - [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", + [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", [CHECK_EG_MATCH] = "CHECK_EG_MATCH", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", @@ -289,6 +276,9 @@ static const char *const _PyOpcode_OpName[263] = { [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O", [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", + [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", + [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", + [COMPARE_OP_FLOAT_JUMP] = "COMPARE_OP_FLOAT_JUMP", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", @@ -296,37 +286,37 @@ static const char *const _PyOpcode_OpName[263] = { [BEFORE_WITH] = "BEFORE_WITH", [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", - [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", - [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", - [COMPARE_OP_ADAPTIVE] = "COMPARE_OP_ADAPTIVE", - [COMPARE_OP_FLOAT_JUMP] = "COMPARE_OP_FLOAT_JUMP", - [STORE_SUBSCR] = "STORE_SUBSCR", - [DELETE_SUBSCR] = "DELETE_SUBSCR", [COMPARE_OP_INT_JUMP] = "COMPARE_OP_INT_JUMP", - [STOPITERATION_ERROR] = "STOPITERATION_ERROR", [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP", [EXTENDED_ARG_QUICK] = "EXTENDED_ARG_QUICK", - [FOR_ITER_ADAPTIVE] = "FOR_ITER_ADAPTIVE", [FOR_ITER_LIST] = "FOR_ITER_LIST", - [GET_ITER] = "GET_ITER", - [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", - [PRINT_EXPR] = "PRINT_EXPR", - [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [STORE_SUBSCR] = "STORE_SUBSCR", + [DELETE_SUBSCR] = "DELETE_SUBSCR", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", - [LOAD_ATTR_ADAPTIVE] = "LOAD_ATTR_ADAPTIVE", - [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", - [RETURN_GENERATOR] = "RETURN_GENERATOR", + [STOPITERATION_ERROR] = "STOPITERATION_ERROR", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", + [GET_ITER] = "GET_ITER", + [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", + [PRINT_EXPR] = "PRINT_EXPR", + [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", + [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", + [RETURN_GENERATOR] = "RETURN_GENERATOR", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", + [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", + [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", + [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [LIST_TO_TUPLE] = "LIST_TO_TUPLE", [RETURN_VALUE] = "RETURN_VALUE", [IMPORT_STAR] = "IMPORT_STAR", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", @@ -353,7 +343,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -361,7 +351,7 @@ static const char *const _PyOpcode_OpName[263] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -381,9 +371,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -393,34 +383,34 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", - [LOAD_GLOBAL_ADAPTIVE] = "LOAD_GLOBAL_ADAPTIVE", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", - [LIST_EXTEND] = "LIST_EXTEND", - [SET_UPDATE] = "SET_UPDATE", - [DICT_MERGE] = "DICT_MERGE", - [DICT_UPDATE] = "DICT_UPDATE", - [STORE_ATTR_ADAPTIVE] = "STORE_ATTR_ADAPTIVE", - [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", - [CALL] = "CALL", - [KW_NAMES] = "KW_NAMES", [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", - [STORE_SUBSCR_ADAPTIVE] = "STORE_SUBSCR_ADAPTIVE", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", - [UNPACK_SEQUENCE_ADAPTIVE] = "UNPACK_SEQUENCE_ADAPTIVE", + [LIST_EXTEND] = "LIST_EXTEND", + [SET_UPDATE] = "SET_UPDATE", + [DICT_MERGE] = "DICT_MERGE", + [DICT_UPDATE] = "DICT_UPDATE", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", + [169] = "<169>", + [170] = "<170>", + [CALL] = "CALL", + [KW_NAMES] = "KW_NAMES", + [173] = "<173>", + [174] = "<174>", + [175] = "<175>", + [176] = "<176>", + [177] = "<177>", + [178] = "<178>", + [179] = "<179>", + [180] = "<180>", [181] = "<181>", [182] = "<182>", [183] = "<183>", @@ -507,6 +497,16 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ + case 169: \ + case 170: \ + case 173: \ + case 174: \ + case 175: \ + case 176: \ + case 177: \ + case 178: \ + case 179: \ + case 180: \ case 181: \ case 182: \ case 183: \ diff --git a/Include/opcode.h b/Include/opcode.h index 4efa35779fedc1..26e7c63d40e796 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -128,77 +128,67 @@ extern "C" { #define JUMP_NO_INTERRUPT 261 #define LOAD_METHOD 262 #define MAX_PSEUDO_OPCODE 262 -#define BINARY_OP_ADAPTIVE 3 -#define BINARY_OP_ADD_FLOAT 5 -#define BINARY_OP_ADD_INT 6 -#define BINARY_OP_ADD_UNICODE 7 -#define BINARY_OP_INPLACE_ADD_UNICODE 8 -#define BINARY_OP_MULTIPLY_FLOAT 13 -#define BINARY_OP_MULTIPLY_INT 14 -#define BINARY_OP_SUBTRACT_FLOAT 16 -#define BINARY_OP_SUBTRACT_INT 17 -#define BINARY_SUBSCR_ADAPTIVE 18 -#define BINARY_SUBSCR_DICT 19 -#define BINARY_SUBSCR_GETITEM 20 -#define BINARY_SUBSCR_LIST_INT 21 -#define BINARY_SUBSCR_TUPLE_INT 22 -#define CALL_ADAPTIVE 23 -#define CALL_PY_EXACT_ARGS 24 -#define CALL_PY_WITH_DEFAULTS 28 -#define CALL_BOUND_METHOD_EXACT_ARGS 29 -#define CALL_BUILTIN_CLASS 34 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 38 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 39 -#define CALL_NO_KW_BUILTIN_FAST 40 -#define CALL_NO_KW_BUILTIN_O 41 -#define CALL_NO_KW_ISINSTANCE 42 -#define CALL_NO_KW_LEN 43 -#define CALL_NO_KW_LIST_APPEND 44 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 45 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 46 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 47 -#define CALL_NO_KW_STR_1 48 -#define CALL_NO_KW_TUPLE_1 56 -#define CALL_NO_KW_TYPE_1 57 -#define COMPARE_OP_ADAPTIVE 58 -#define COMPARE_OP_FLOAT_JUMP 59 -#define COMPARE_OP_INT_JUMP 62 -#define COMPARE_OP_STR_JUMP 64 -#define EXTENDED_ARG_QUICK 65 -#define FOR_ITER_ADAPTIVE 66 -#define FOR_ITER_LIST 67 -#define FOR_ITER_RANGE 72 -#define LOAD_ATTR_ADAPTIVE 73 -#define LOAD_ATTR_CLASS 76 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 77 -#define LOAD_ATTR_INSTANCE_VALUE 78 -#define LOAD_ATTR_MODULE 79 -#define LOAD_ATTR_PROPERTY 80 -#define LOAD_ATTR_SLOT 81 -#define LOAD_ATTR_WITH_HINT 86 -#define LOAD_ATTR_METHOD_LAZY_DICT 113 -#define LOAD_ATTR_METHOD_NO_DICT 121 -#define LOAD_ATTR_METHOD_WITH_DICT 141 -#define LOAD_ATTR_METHOD_WITH_VALUES 143 -#define LOAD_CONST__LOAD_FAST 153 -#define LOAD_FAST__LOAD_CONST 154 -#define LOAD_FAST__LOAD_FAST 158 -#define LOAD_GLOBAL_ADAPTIVE 159 -#define LOAD_GLOBAL_BUILTIN 160 -#define LOAD_GLOBAL_MODULE 161 -#define STORE_ATTR_ADAPTIVE 166 -#define STORE_ATTR_INSTANCE_VALUE 167 -#define STORE_ATTR_SLOT 168 -#define STORE_ATTR_WITH_HINT 169 -#define STORE_FAST__LOAD_FAST 170 -#define STORE_FAST__STORE_FAST 173 -#define STORE_SUBSCR_ADAPTIVE 174 -#define STORE_SUBSCR_DICT 175 -#define STORE_SUBSCR_LIST_INT 176 -#define UNPACK_SEQUENCE_ADAPTIVE 177 -#define UNPACK_SEQUENCE_LIST 178 -#define UNPACK_SEQUENCE_TUPLE 179 -#define UNPACK_SEQUENCE_TWO_TUPLE 180 +#define BINARY_OP_ADD_FLOAT 3 +#define BINARY_OP_ADD_INT 5 +#define BINARY_OP_ADD_UNICODE 6 +#define BINARY_OP_INPLACE_ADD_UNICODE 7 +#define BINARY_OP_MULTIPLY_FLOAT 8 +#define BINARY_OP_MULTIPLY_INT 13 +#define BINARY_OP_SUBTRACT_FLOAT 14 +#define BINARY_OP_SUBTRACT_INT 16 +#define BINARY_SUBSCR_DICT 17 +#define BINARY_SUBSCR_GETITEM 18 +#define BINARY_SUBSCR_LIST_INT 19 +#define BINARY_SUBSCR_TUPLE_INT 20 +#define CALL_PY_EXACT_ARGS 21 +#define CALL_PY_WITH_DEFAULTS 22 +#define CALL_BOUND_METHOD_EXACT_ARGS 23 +#define CALL_BUILTIN_CLASS 24 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 28 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 29 +#define CALL_NO_KW_BUILTIN_FAST 34 +#define CALL_NO_KW_BUILTIN_O 38 +#define CALL_NO_KW_ISINSTANCE 39 +#define CALL_NO_KW_LEN 40 +#define CALL_NO_KW_LIST_APPEND 41 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 42 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 43 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 44 +#define CALL_NO_KW_STR_1 45 +#define CALL_NO_KW_TUPLE_1 46 +#define CALL_NO_KW_TYPE_1 47 +#define COMPARE_OP_FLOAT_JUMP 48 +#define COMPARE_OP_INT_JUMP 56 +#define COMPARE_OP_STR_JUMP 57 +#define EXTENDED_ARG_QUICK 58 +#define FOR_ITER_LIST 59 +#define FOR_ITER_RANGE 62 +#define LOAD_ATTR_CLASS 64 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 65 +#define LOAD_ATTR_INSTANCE_VALUE 66 +#define LOAD_ATTR_MODULE 67 +#define LOAD_ATTR_PROPERTY 72 +#define LOAD_ATTR_SLOT 73 +#define LOAD_ATTR_WITH_HINT 76 +#define LOAD_ATTR_METHOD_LAZY_DICT 77 +#define LOAD_ATTR_METHOD_NO_DICT 78 +#define LOAD_ATTR_METHOD_WITH_DICT 79 +#define LOAD_ATTR_METHOD_WITH_VALUES 80 +#define LOAD_CONST__LOAD_FAST 81 +#define LOAD_FAST__LOAD_CONST 86 +#define LOAD_FAST__LOAD_FAST 113 +#define LOAD_GLOBAL_BUILTIN 121 +#define LOAD_GLOBAL_MODULE 141 +#define STORE_ATTR_INSTANCE_VALUE 143 +#define STORE_ATTR_SLOT 153 +#define STORE_ATTR_WITH_HINT 154 +#define STORE_FAST__LOAD_FAST 158 +#define STORE_FAST__STORE_FAST 159 +#define STORE_SUBSCR_DICT 160 +#define STORE_SUBSCR_LIST_INT 161 +#define UNPACK_SEQUENCE_LIST 166 +#define UNPACK_SEQUENCE_TUPLE 167 +#define UNPACK_SEQUENCE_TWO_TUPLE 168 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/dis.py b/Lib/dis.py index a045d18241b465..523bd01d929565 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -512,9 +512,8 @@ def _get_instructions_bytes(code, varname_from_oparg=None, for i in range(size): offset += 2 # Only show the fancy argrepr for a CACHE instruction when it's - # the first entry for a particular cache value and the - # instruction using it is actually quickened: - if i == 0 and op != deop: + # the first entry for a particular cache value: + if i == 0: data = code[offset: offset + 2 * size] argrepr = f"{name}: {int.from_bytes(data, sys.byteorder)}" else: diff --git a/Lib/opcode.py b/Lib/opcode.py index dfe06f8cdefd7f..fc0fb13a270a59 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -278,7 +278,6 @@ def pseudo_op(name, op, real_ops): _specializations = { "BINARY_OP": [ - "BINARY_OP_ADAPTIVE", "BINARY_OP_ADD_FLOAT", "BINARY_OP_ADD_INT", "BINARY_OP_ADD_UNICODE", @@ -289,14 +288,12 @@ def pseudo_op(name, op, real_ops): "BINARY_OP_SUBTRACT_INT", ], "BINARY_SUBSCR": [ - "BINARY_SUBSCR_ADAPTIVE", "BINARY_SUBSCR_DICT", "BINARY_SUBSCR_GETITEM", "BINARY_SUBSCR_LIST_INT", "BINARY_SUBSCR_TUPLE_INT", ], "CALL": [ - "CALL_ADAPTIVE", "CALL_PY_EXACT_ARGS", "CALL_PY_WITH_DEFAULTS", "CALL_BOUND_METHOD_EXACT_ARGS", @@ -316,7 +313,6 @@ def pseudo_op(name, op, real_ops): "CALL_NO_KW_TYPE_1", ], "COMPARE_OP": [ - "COMPARE_OP_ADAPTIVE", "COMPARE_OP_FLOAT_JUMP", "COMPARE_OP_INT_JUMP", "COMPARE_OP_STR_JUMP", @@ -325,12 +321,10 @@ def pseudo_op(name, op, real_ops): "EXTENDED_ARG_QUICK", ], "FOR_ITER": [ - "FOR_ITER_ADAPTIVE", "FOR_ITER_LIST", "FOR_ITER_RANGE", ], "LOAD_ATTR": [ - "LOAD_ATTR_ADAPTIVE", # These potentially push [NULL, bound method] onto the stack. "LOAD_ATTR_CLASS", "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", @@ -353,12 +347,10 @@ def pseudo_op(name, op, real_ops): "LOAD_FAST__LOAD_FAST", ], "LOAD_GLOBAL": [ - "LOAD_GLOBAL_ADAPTIVE", "LOAD_GLOBAL_BUILTIN", "LOAD_GLOBAL_MODULE", ], "STORE_ATTR": [ - "STORE_ATTR_ADAPTIVE", "STORE_ATTR_INSTANCE_VALUE", "STORE_ATTR_SLOT", "STORE_ATTR_WITH_HINT", @@ -368,12 +360,10 @@ def pseudo_op(name, op, real_ops): "STORE_FAST__STORE_FAST", ], "STORE_SUBSCR": [ - "STORE_SUBSCR_ADAPTIVE", "STORE_SUBSCR_DICT", "STORE_SUBSCR_LIST_INT", ], "UNPACK_SEQUENCE": [ - "UNPACK_SEQUENCE_ADAPTIVE", "UNPACK_SEQUENCE_LIST", "UNPACK_SEQUENCE_TUPLE", "UNPACK_SEQUENCE_TWO_TUPLE", diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 704d19fffd09e6..db831069c7aeb8 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -72,9 +72,10 @@ def test_specialization_stats(self): stat_names = opcode._specialization_stats specialized_opcodes = [ - op[:-len("_ADAPTIVE")].lower() for - op in opcode._specialized_instructions - if op.endswith("_ADAPTIVE")] + op.lower() + for op in opcode._specializations + if opcode._inline_cache_entries[opcode.opmap[op]] + ] self.assertIn('load_attr', specialized_opcodes) self.assertIn('binary_subscr', specialized_opcodes) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index ceba007ebc182e..5640bf265b0da1 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -741,7 +741,7 @@ def loop_test(): LOAD_CONST 1 ((1, 2, 3)) LIST_EXTEND 1 LOAD_CONST 2 (3) - BINARY_OP_ADAPTIVE 5 (*) + BINARY_OP 5 (*) GET_ITER >> FOR_ITER_LIST 15 (to 50) STORE_FAST 0 (i) @@ -1200,7 +1200,7 @@ def test_show_caches(self): for cache in caches: self.assertRegex(cache, pattern) total_caches = 23 - empty_caches = 8 if adaptive else total_caches + empty_caches = 8 self.assertEqual(caches.count(""), empty_caches) self.assertEqual(len(caches), total_caches) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b0d56279e04379..974d2518ff6b6b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -429,20 +429,6 @@ dummy_func( JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); } - // stack effect: (__0 -- ) - inst(BINARY_SUBSCR) { - PREDICTED(BINARY_SUBSCR); - PyObject *sub = POP(); - PyObject *container = TOP(); - PyObject *res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - SET_TOP(res); - if (res == NULL) - goto error; - JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - } - // stack effect: (__0, __1 -- ) inst(BINARY_SLICE) { PyObject *stop = POP(); @@ -484,9 +470,11 @@ dummy_func( } // stack effect: (__0 -- ) - inst(BINARY_SUBSCR_ADAPTIVE) { + inst(BINARY_SUBSCR) { _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (cframe.use_tracing || opcode != BINARY_SUBSCR) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { PyObject *sub = TOP(); PyObject *container = SECOND(); next_instr--; @@ -498,8 +486,16 @@ dummy_func( else { STAT_INC(BINARY_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(BINARY_SUBSCR); } + PyObject *sub = POP(); + PyObject *container = TOP(); + PyObject *res = PyObject_GetItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + SET_TOP(res); + if (res == NULL) + goto error; + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); } // stack effect: (__0 -- ) @@ -631,7 +627,22 @@ dummy_func( // stack effect: (__0, __1, __2 -- ) inst(STORE_SUBSCR) { - PREDICTED(STORE_SUBSCR); + _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; + if (cframe.use_tracing || opcode != STORE_SUBSCR) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *sub = TOP(); + PyObject *container = SECOND(); + next_instr--; + if (_Py_Specialize_StoreSubscr(container, sub, next_instr) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(STORE_SUBSCR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + } PyObject *sub = TOP(); PyObject *container = SECOND(); PyObject *v = THIRD(); @@ -648,25 +659,6 @@ dummy_func( JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); } - // stack effect: (__0, __1, __2 -- ) - inst(STORE_SUBSCR_ADAPTIVE) { - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *sub = TOP(); - PyObject *container = SECOND(); - next_instr--; - if (_Py_Specialize_StoreSubscr(container, sub, next_instr) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - else { - STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(STORE_SUBSCR); - } - } - // stack effect: (__0, __1, __2 -- ) inst(STORE_SUBSCR_LIST_INT) { assert(cframe.use_tracing == 0); @@ -1204,23 +1196,10 @@ dummy_func( // stack effect: (__0 -- __array[oparg]) inst(UNPACK_SEQUENCE) { - PREDICTED(UNPACK_SEQUENCE); - PyObject *seq = POP(); - PyObject **top = stack_pointer + oparg; - if (!unpack_iterable(tstate, seq, oparg, -1, top)) { - Py_DECREF(seq); - goto error; - } - STACK_GROW(oparg); - Py_DECREF(seq); - JUMPBY(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - } - - // stack effect: (__0 -- __array[oparg]) - inst(UNPACK_SEQUENCE_ADAPTIVE) { - assert(cframe.use_tracing == 0); _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (cframe.use_tracing || opcode != UNPACK_SEQUENCE) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { PyObject *seq = TOP(); next_instr--; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); @@ -1229,8 +1208,16 @@ dummy_func( else { STAT_INC(UNPACK_SEQUENCE, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(UNPACK_SEQUENCE); } + PyObject *seq = POP(); + PyObject **top = stack_pointer + oparg; + if (!unpack_iterable(tstate, seq, oparg, -1, top)) { + Py_DECREF(seq); + goto error; + } + STACK_GROW(oparg); + Py_DECREF(seq); + JUMPBY(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); } // stack effect: (__0 -- __array[oparg]) @@ -1288,23 +1275,6 @@ dummy_func( Py_DECREF(seq); } - // stack effect: (__0, __1 -- ) - inst(STORE_ATTR) { - PREDICTED(STORE_ATTR); - PyObject *name = GETITEM(names, oparg); - PyObject *owner = TOP(); - PyObject *v = SECOND(); - int err; - STACK_SHRINK(2); - err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); - if (err != 0) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); - } - // stack effect: (__0 -- ) inst(DELETE_ATTR) { PyObject *name = GETITEM(names, oparg); @@ -1407,7 +1377,21 @@ dummy_func( // error: LOAD_GLOBAL has irregular stack effect inst(LOAD_GLOBAL) { - PREDICTED(LOAD_GLOBAL); + _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; + if (cframe.use_tracing || opcode != LOAD_GLOBAL) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *name = GETITEM(names, oparg>>1); + next_instr--; + if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(LOAD_GLOBAL, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + } int push_null = oparg & 1; PEEK(0) = NULL; PyObject *name = GETITEM(names, oparg>>1); @@ -1458,25 +1442,6 @@ dummy_func( PUSH(v); } - // error: LOAD_GLOBAL has irregular stack effect - inst(LOAD_GLOBAL_ADAPTIVE) { - assert(cframe.use_tracing == 0); - _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *name = GETITEM(names, oparg>>1); - next_instr--; - if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - else { - STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(LOAD_GLOBAL); - } - } - // error: LOAD_GLOBAL has irregular stack effect inst(LOAD_GLOBAL_MODULE) { assert(cframe.use_tracing == 0); @@ -1871,7 +1836,22 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR) { - PREDICTED(LOAD_ATTR); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (cframe.use_tracing || opcode != LOAD_ATTR) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *owner = TOP(); + PyObject *name = GETITEM(names, oparg>>1); + next_instr--; + if (_Py_Specialize_LoadAttr(owner, next_instr, name) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(LOAD_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + } PyObject *name = GETITEM(names, oparg >> 1); PyObject *owner = TOP(); if (oparg & 1) { @@ -1918,26 +1898,6 @@ dummy_func( JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); } - // error: LOAD_ATTR has irregular stack effect - inst(LOAD_ATTR_ADAPTIVE) { - assert(cframe.use_tracing == 0); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *owner = TOP(); - PyObject *name = GETITEM(names, oparg>>1); - next_instr--; - if (_Py_Specialize_LoadAttr(owner, next_instr, name) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - else { - STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(LOAD_ATTR); - } - } - // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_INSTANCE_VALUE) { assert(cframe.use_tracing == 0); @@ -2154,10 +2114,11 @@ dummy_func( } // stack effect: (__0, __1 -- ) - inst(STORE_ATTR_ADAPTIVE) { - assert(cframe.use_tracing == 0); + inst(STORE_ATTR) { _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (cframe.use_tracing || opcode != STORE_ATTR) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg); next_instr--; @@ -2169,8 +2130,19 @@ dummy_func( else { STAT_INC(STORE_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(STORE_ATTR); } + PyObject *name = GETITEM(names, oparg); + PyObject *owner = TOP(); + PyObject *v = SECOND(); + int err; + STACK_SHRINK(2); + err = PyObject_SetAttr(owner, name, v); + Py_DECREF(v); + Py_DECREF(owner); + if (err != 0) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); } // stack effect: (__0, __1 -- ) @@ -2276,25 +2248,10 @@ dummy_func( // stack effect: (__0 -- ) inst(COMPARE_OP) { - PREDICTED(COMPARE_OP); - assert(oparg <= Py_GE); - PyObject *right = POP(); - PyObject *left = TOP(); - PyObject *res = PyObject_RichCompare(left, right, oparg); - SET_TOP(res); - Py_DECREF(left); - Py_DECREF(right); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); - } - - // stack effect: (__0 -- ) - inst(COMPARE_OP_ADAPTIVE) { - assert(cframe.use_tracing == 0); _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (cframe.use_tracing || opcode != COMPARE_OP) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { PyObject *right = TOP(); PyObject *left = SECOND(); next_instr--; @@ -2304,8 +2261,18 @@ dummy_func( else { STAT_INC(COMPARE_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(COMPARE_OP); } + assert(oparg <= Py_GE); + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyObject_RichCompare(left, right, oparg); + SET_TOP(res); + Py_DECREF(left); + Py_DECREF(right); + if (res == NULL) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); } // stack effect: (__0 -- ) @@ -2788,7 +2755,18 @@ dummy_func( // stack effect: ( -- __0) inst(FOR_ITER) { - PREDICTED(FOR_ITER); + _PyForIterCache *cache = (_PyForIterCache *)next_instr; + if (cframe.use_tracing || opcode != FOR_ITER) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + next_instr--; + _Py_Specialize_ForIter(TOP(), next_instr); + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(FOR_ITER, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + } /* before: [iter]; after: [iter, iter()] *or* [] */ PyObject *iter = TOP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); @@ -2814,22 +2792,6 @@ dummy_func( JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1); } - // stack effect: ( -- __0) - inst(FOR_ITER_ADAPTIVE) { - assert(cframe.use_tracing == 0); - _PyForIterCache *cache = (_PyForIterCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - next_instr--; - _Py_Specialize_ForIter(TOP(), next_instr); - DISPATCH_SAME_OPARG(); - } - else { - STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(FOR_ITER); - } - } - // stack effect: ( -- __0) inst(FOR_ITER_LIST) { assert(cframe.use_tracing == 0); @@ -3118,7 +3080,25 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL) { - PREDICTED(CALL); + _PyCallCache *cache = (_PyCallCache *)next_instr; + if (cframe.use_tracing || opcode != CALL) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + next_instr--; + int is_meth = is_method(stack_pointer, oparg); + int nargs = oparg + is_meth; + PyObject *callable = PEEK(nargs + 1); + int err = _Py_Specialize_Call(callable, next_instr, nargs, + call_shape.kwnames); + if (err < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(CALL, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + } int total_args, is_meth; is_meth = is_method(stack_pointer, oparg); PyObject *function = PEEK(oparg + 1); @@ -3192,28 +3172,6 @@ dummy_func( CHECK_EVAL_BREAKER(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_ADAPTIVE) { - _PyCallCache *cache = (_PyCallCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - next_instr--; - int is_meth = is_method(stack_pointer, oparg); - int nargs = oparg + is_meth; - PyObject *callable = PEEK(nargs + 1); - int err = _Py_Specialize_Call(callable, next_instr, nargs, - call_shape.kwnames); - if (err < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - else { - STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(CALL); - } - } - // stack effect: (__0, __array[oparg] -- ) inst(CALL_PY_EXACT_ARGS) { PREDICTED(CALL_PY_EXACT_ARGS); @@ -3910,10 +3868,22 @@ dummy_func( Py_INCREF(peek); PUSH(peek); } - // stack effect: (__0 -- ) inst(BINARY_OP) { - PREDICTED(BINARY_OP); + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; + if (cframe.use_tracing || opcode != BINARY_OP) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *lhs = SECOND(); + PyObject *rhs = TOP(); + next_instr--; + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(BINARY_OP, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + } PyObject *rhs = POP(); PyObject *lhs = TOP(); assert(0 <= oparg); @@ -3929,24 +3899,6 @@ dummy_func( JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); } - // stack effect: (__0 -- ) - inst(BINARY_OP_ADAPTIVE) { - assert(cframe.use_tracing == 0); - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *lhs = SECOND(); - PyObject *rhs = TOP(); - next_instr--; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); - DISPATCH_SAME_OPARG(); - } - else { - STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(BINARY_OP); - } - } - // stack effect: ( -- ) inst(SWAP) { assert(oparg != 0); @@ -4004,15 +3956,15 @@ dummy_func( // Families go below this point // family(binary_op) = { - BINARY_OP, BINARY_OP_ADAPTIVE, BINARY_OP_ADD_FLOAT, + BINARY_OP, BINARY_OP_ADD_FLOAT, BINARY_OP_ADD_INT, BINARY_OP_ADD_UNICODE, BINARY_OP_INPLACE_ADD_UNICODE, BINARY_OP_MULTIPLY_FLOAT, BINARY_OP_MULTIPLY_INT, BINARY_OP_SUBTRACT_FLOAT, BINARY_OP_SUBTRACT_INT }; family(binary_subscr) = { - BINARY_SUBSCR, BINARY_SUBSCR_ADAPTIVE, BINARY_SUBSCR_DICT, + BINARY_SUBSCR, BINARY_SUBSCR_DICT, BINARY_SUBSCR_GETITEM, BINARY_SUBSCR_LIST_INT, BINARY_SUBSCR_TUPLE_INT }; family(call) = { - CALL, CALL_ADAPTIVE, CALL_PY_EXACT_ARGS, + CALL, CALL_PY_EXACT_ARGS, CALL_PY_WITH_DEFAULTS, CALL_BOUND_METHOD_EXACT_ARGS, CALL_BUILTIN_CLASS, CALL_BUILTIN_FAST_WITH_KEYWORDS, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, CALL_NO_KW_BUILTIN_FAST, CALL_NO_KW_BUILTIN_O, CALL_NO_KW_ISINSTANCE, CALL_NO_KW_LEN, @@ -4020,14 +3972,14 @@ family(call) = { CALL_NO_KW_METHOD_DESCRIPTOR_O, CALL_NO_KW_STR_1, CALL_NO_KW_TUPLE_1, CALL_NO_KW_TYPE_1 }; family(compare_op) = { - COMPARE_OP, COMPARE_OP_ADAPTIVE, COMPARE_OP_FLOAT_JUMP, + COMPARE_OP, COMPARE_OP_FLOAT_JUMP, COMPARE_OP_INT_JUMP, COMPARE_OP_STR_JUMP }; family(extended_arg) = { EXTENDED_ARG, EXTENDED_ARG_QUICK }; family(for_iter) = { - FOR_ITER, FOR_ITER_ADAPTIVE, FOR_ITER_LIST, + FOR_ITER, FOR_ITER_LIST, FOR_ITER_RANGE }; family(load_attr) = { - LOAD_ATTR, LOAD_ATTR_ADAPTIVE, LOAD_ATTR_CLASS, + LOAD_ATTR, LOAD_ATTR_CLASS, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_MODULE, LOAD_ATTR_PROPERTY, LOAD_ATTR_SLOT, LOAD_ATTR_WITH_HINT, LOAD_ATTR_METHOD_LAZY_DICT, LOAD_ATTR_METHOD_NO_DICT, LOAD_ATTR_METHOD_WITH_DICT, @@ -4035,15 +3987,15 @@ family(load_attr) = { family(load_const) = { LOAD_CONST, LOAD_CONST__LOAD_FAST }; family(load_fast) = { LOAD_FAST, LOAD_FAST__LOAD_CONST, LOAD_FAST__LOAD_FAST }; family(load_global) = { - LOAD_GLOBAL, LOAD_GLOBAL_ADAPTIVE, LOAD_GLOBAL_BUILTIN, + LOAD_GLOBAL, LOAD_GLOBAL_BUILTIN, LOAD_GLOBAL_MODULE }; family(store_attr) = { - STORE_ATTR, STORE_ATTR_ADAPTIVE, STORE_ATTR_INSTANCE_VALUE, + STORE_ATTR, STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_SLOT, STORE_ATTR_WITH_HINT }; family(store_fast) = { STORE_FAST, STORE_FAST__LOAD_FAST, STORE_FAST__STORE_FAST }; family(store_subscr) = { - STORE_SUBSCR, STORE_SUBSCR_ADAPTIVE, STORE_SUBSCR_DICT, + STORE_SUBSCR, STORE_SUBSCR_DICT, STORE_SUBSCR_LIST_INT }; family(unpack_sequence) = { - UNPACK_SEQUENCE, UNPACK_SEQUENCE_ADAPTIVE, UNPACK_SEQUENCE_LIST, + UNPACK_SEQUENCE, UNPACK_SEQUENCE_LIST, UNPACK_SEQUENCE_TUPLE, UNPACK_SEQUENCE_TWO_TUPLE }; diff --git a/Python/ceval.c b/Python/ceval.c index 54f2282268242c..ad1561d8dcb391 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1255,9 +1255,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _Py_CODEUNIT *counter = (_Py_CODEUNIT *)next_instr; *counter -= 1; if (*counter == 0) { - int adaptive_opcode = _PyOpcode_Adaptive[opcode]; - assert(adaptive_opcode); - _Py_SET_OPCODE(next_instr[-1], adaptive_opcode); + _Py_SET_OPCODE(next_instr[-1], opcode); STAT_INC(opcode, deopt); *counter = adaptive_counter_start(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index bea51d7e5160dc..a6ea4139b668bd 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -361,20 +361,6 @@ DISPATCH(); } - TARGET(BINARY_SUBSCR) { - PREDICTED(BINARY_SUBSCR); - PyObject *sub = POP(); - PyObject *container = TOP(); - PyObject *res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - SET_TOP(res); - if (res == NULL) - goto error; - JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - DISPATCH(); - } - TARGET(BINARY_SLICE) { PyObject *stop = POP(); PyObject *start = POP(); @@ -415,9 +401,11 @@ DISPATCH(); } - TARGET(BINARY_SUBSCR_ADAPTIVE) { + TARGET(BINARY_SUBSCR) { _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (cframe.use_tracing || opcode != BINARY_SUBSCR) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { PyObject *sub = TOP(); PyObject *container = SECOND(); next_instr--; @@ -429,8 +417,16 @@ else { STAT_INC(BINARY_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(BINARY_SUBSCR); } + PyObject *sub = POP(); + PyObject *container = TOP(); + PyObject *res = PyObject_GetItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + SET_TOP(res); + if (res == NULL) + goto error; + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); DISPATCH(); } @@ -561,7 +557,22 @@ } TARGET(STORE_SUBSCR) { - PREDICTED(STORE_SUBSCR); + _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; + if (cframe.use_tracing || opcode != STORE_SUBSCR) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *sub = TOP(); + PyObject *container = SECOND(); + next_instr--; + if (_Py_Specialize_StoreSubscr(container, sub, next_instr) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(STORE_SUBSCR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + } PyObject *sub = TOP(); PyObject *container = SECOND(); PyObject *v = THIRD(); @@ -579,25 +590,6 @@ DISPATCH(); } - TARGET(STORE_SUBSCR_ADAPTIVE) { - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *sub = TOP(); - PyObject *container = SECOND(); - next_instr--; - if (_Py_Specialize_StoreSubscr(container, sub, next_instr) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - else { - STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(STORE_SUBSCR); - } - DISPATCH(); - } - TARGET(STORE_SUBSCR_LIST_INT) { assert(cframe.use_tracing == 0); PyObject *sub = TOP(); @@ -1128,23 +1120,10 @@ } TARGET(UNPACK_SEQUENCE) { - PREDICTED(UNPACK_SEQUENCE); - PyObject *seq = POP(); - PyObject **top = stack_pointer + oparg; - if (!unpack_iterable(tstate, seq, oparg, -1, top)) { - Py_DECREF(seq); - goto error; - } - STACK_GROW(oparg); - Py_DECREF(seq); - JUMPBY(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); - DISPATCH(); - } - - TARGET(UNPACK_SEQUENCE_ADAPTIVE) { - assert(cframe.use_tracing == 0); _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (cframe.use_tracing || opcode != UNPACK_SEQUENCE) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { PyObject *seq = TOP(); next_instr--; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); @@ -1153,8 +1132,16 @@ else { STAT_INC(UNPACK_SEQUENCE, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(UNPACK_SEQUENCE); } + PyObject *seq = POP(); + PyObject **top = stack_pointer + oparg; + if (!unpack_iterable(tstate, seq, oparg, -1, top)) { + Py_DECREF(seq); + goto error; + } + STACK_GROW(oparg); + Py_DECREF(seq); + JUMPBY(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); DISPATCH(); } @@ -1213,23 +1200,6 @@ DISPATCH(); } - TARGET(STORE_ATTR) { - PREDICTED(STORE_ATTR); - PyObject *name = GETITEM(names, oparg); - PyObject *owner = TOP(); - PyObject *v = SECOND(); - int err; - STACK_SHRINK(2); - err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); - if (err != 0) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); - DISPATCH(); - } - TARGET(DELETE_ATTR) { PyObject *name = GETITEM(names, oparg); PyObject *owner = POP(); @@ -1331,7 +1301,21 @@ } TARGET(LOAD_GLOBAL) { - PREDICTED(LOAD_GLOBAL); + _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; + if (cframe.use_tracing || opcode != LOAD_GLOBAL) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *name = GETITEM(names, oparg>>1); + next_instr--; + if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(LOAD_GLOBAL, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + } int push_null = oparg & 1; PEEK(0) = NULL; PyObject *name = GETITEM(names, oparg>>1); @@ -1383,25 +1367,6 @@ DISPATCH(); } - TARGET(LOAD_GLOBAL_ADAPTIVE) { - assert(cframe.use_tracing == 0); - _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *name = GETITEM(names, oparg>>1); - next_instr--; - if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - else { - STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(LOAD_GLOBAL); - } - DISPATCH(); - } - TARGET(LOAD_GLOBAL_MODULE) { assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); @@ -1793,7 +1758,22 @@ } TARGET(LOAD_ATTR) { - PREDICTED(LOAD_ATTR); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (cframe.use_tracing || opcode != LOAD_ATTR) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *owner = TOP(); + PyObject *name = GETITEM(names, oparg>>1); + next_instr--; + if (_Py_Specialize_LoadAttr(owner, next_instr, name) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(LOAD_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + } PyObject *name = GETITEM(names, oparg >> 1); PyObject *owner = TOP(); if (oparg & 1) { @@ -1841,26 +1821,6 @@ DISPATCH(); } - TARGET(LOAD_ATTR_ADAPTIVE) { - assert(cframe.use_tracing == 0); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *owner = TOP(); - PyObject *name = GETITEM(names, oparg>>1); - next_instr--; - if (_Py_Specialize_LoadAttr(owner, next_instr, name) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - else { - STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(LOAD_ATTR); - } - DISPATCH(); - } - TARGET(LOAD_ATTR_INSTANCE_VALUE) { assert(cframe.use_tracing == 0); PyObject *owner = TOP(); @@ -2074,10 +2034,11 @@ goto start_frame; } - TARGET(STORE_ATTR_ADAPTIVE) { - assert(cframe.use_tracing == 0); + TARGET(STORE_ATTR) { _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (cframe.use_tracing || opcode != STORE_ATTR) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg); next_instr--; @@ -2089,8 +2050,19 @@ else { STAT_INC(STORE_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(STORE_ATTR); } + PyObject *name = GETITEM(names, oparg); + PyObject *owner = TOP(); + PyObject *v = SECOND(); + int err; + STACK_SHRINK(2); + err = PyObject_SetAttr(owner, name, v); + Py_DECREF(v); + Py_DECREF(owner); + if (err != 0) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); DISPATCH(); } @@ -2196,25 +2168,10 @@ } TARGET(COMPARE_OP) { - PREDICTED(COMPARE_OP); - assert(oparg <= Py_GE); - PyObject *right = POP(); - PyObject *left = TOP(); - PyObject *res = PyObject_RichCompare(left, right, oparg); - SET_TOP(res); - Py_DECREF(left); - Py_DECREF(right); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); - DISPATCH(); - } - - TARGET(COMPARE_OP_ADAPTIVE) { - assert(cframe.use_tracing == 0); _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (cframe.use_tracing || opcode != COMPARE_OP) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { PyObject *right = TOP(); PyObject *left = SECOND(); next_instr--; @@ -2224,8 +2181,18 @@ else { STAT_INC(COMPARE_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(COMPARE_OP); } + assert(oparg <= Py_GE); + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyObject_RichCompare(left, right, oparg); + SET_TOP(res); + Py_DECREF(left); + Py_DECREF(right); + if (res == NULL) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); DISPATCH(); } @@ -2708,7 +2675,18 @@ } TARGET(FOR_ITER) { - PREDICTED(FOR_ITER); + _PyForIterCache *cache = (_PyForIterCache *)next_instr; + if (cframe.use_tracing || opcode != FOR_ITER) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + next_instr--; + _Py_Specialize_ForIter(TOP(), next_instr); + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(FOR_ITER, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + } /* before: [iter]; after: [iter, iter()] *or* [] */ PyObject *iter = TOP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); @@ -2735,22 +2713,6 @@ DISPATCH(); } - TARGET(FOR_ITER_ADAPTIVE) { - assert(cframe.use_tracing == 0); - _PyForIterCache *cache = (_PyForIterCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - next_instr--; - _Py_Specialize_ForIter(TOP(), next_instr); - DISPATCH_SAME_OPARG(); - } - else { - STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(FOR_ITER); - } - DISPATCH(); - } - TARGET(FOR_ITER_LIST) { assert(cframe.use_tracing == 0); _PyListIterObject *it = (_PyListIterObject *)TOP(); @@ -3036,7 +2998,25 @@ } TARGET(CALL) { - PREDICTED(CALL); + _PyCallCache *cache = (_PyCallCache *)next_instr; + if (cframe.use_tracing || opcode != CALL) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + next_instr--; + int is_meth = is_method(stack_pointer, oparg); + int nargs = oparg + is_meth; + PyObject *callable = PEEK(nargs + 1); + int err = _Py_Specialize_Call(callable, next_instr, nargs, + call_shape.kwnames); + if (err < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(CALL, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + } int total_args, is_meth; is_meth = is_method(stack_pointer, oparg); PyObject *function = PEEK(oparg + 1); @@ -3111,28 +3091,6 @@ DISPATCH(); } - TARGET(CALL_ADAPTIVE) { - _PyCallCache *cache = (_PyCallCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - next_instr--; - int is_meth = is_method(stack_pointer, oparg); - int nargs = oparg + is_meth; - PyObject *callable = PEEK(nargs + 1); - int err = _Py_Specialize_Call(callable, next_instr, nargs, - call_shape.kwnames); - if (err < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - else { - STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(CALL); - } - DISPATCH(); - } - TARGET(CALL_PY_EXACT_ARGS) { PREDICTED(CALL_PY_EXACT_ARGS); assert(call_shape.kwnames == NULL); @@ -3828,7 +3786,20 @@ } TARGET(BINARY_OP) { - PREDICTED(BINARY_OP); + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; + if (cframe.use_tracing || opcode != BINARY_OP) { + } + else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *lhs = SECOND(); + PyObject *rhs = TOP(); + next_instr--; + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(BINARY_OP, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + } PyObject *rhs = POP(); PyObject *lhs = TOP(); assert(0 <= oparg); @@ -3845,24 +3816,6 @@ DISPATCH(); } - TARGET(BINARY_OP_ADAPTIVE) { - assert(cframe.use_tracing == 0); - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *lhs = SECOND(); - PyObject *rhs = TOP(); - next_instr--; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); - DISPATCH_SAME_OPARG(); - } - else { - STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); - GO_TO_INSTRUCTION(BINARY_OP); - } - DISPATCH(); - } - TARGET(SWAP) { assert(oparg != 0); PyObject *top = TOP(); diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index a963a7a5679427..fb22110784d098 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -2,44 +2,41 @@ static void *opcode_targets[256] = { &&TARGET_CACHE, &&TARGET_POP_TOP, &&TARGET_PUSH_NULL, - &&TARGET_BINARY_OP_ADAPTIVE, - &&TARGET_END_FOR, &&TARGET_BINARY_OP_ADD_FLOAT, + &&TARGET_END_FOR, &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_UNICODE, &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, + &&TARGET_BINARY_OP_MULTIPLY_FLOAT, &&TARGET_NOP, &&TARGET_UNARY_POSITIVE, &&TARGET_UNARY_NEGATIVE, &&TARGET_UNARY_NOT, - &&TARGET_BINARY_OP_MULTIPLY_FLOAT, &&TARGET_BINARY_OP_MULTIPLY_INT, - &&TARGET_UNARY_INVERT, &&TARGET_BINARY_OP_SUBTRACT_FLOAT, + &&TARGET_UNARY_INVERT, &&TARGET_BINARY_OP_SUBTRACT_INT, - &&TARGET_BINARY_SUBSCR_ADAPTIVE, &&TARGET_BINARY_SUBSCR_DICT, &&TARGET_BINARY_SUBSCR_GETITEM, &&TARGET_BINARY_SUBSCR_LIST_INT, &&TARGET_BINARY_SUBSCR_TUPLE_INT, - &&TARGET_CALL_ADAPTIVE, &&TARGET_CALL_PY_EXACT_ARGS, + &&TARGET_CALL_PY_WITH_DEFAULTS, + &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_SLICE, &&TARGET_STORE_SLICE, - &&TARGET_CALL_PY_WITH_DEFAULTS, - &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, + &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, &&TARGET_GET_LEN, &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_MATCH_KEYS, - &&TARGET_CALL_BUILTIN_CLASS, + &&TARGET_CALL_NO_KW_BUILTIN_FAST, &&TARGET_PUSH_EXC_INFO, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CHECK_EG_MATCH, - &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, - &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, - &&TARGET_CALL_NO_KW_BUILTIN_FAST, &&TARGET_CALL_NO_KW_BUILTIN_O, &&TARGET_CALL_NO_KW_ISINSTANCE, &&TARGET_CALL_NO_KW_LEN, @@ -48,6 +45,9 @@ static void *opcode_targets[256] = { &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, &&TARGET_CALL_NO_KW_STR_1, + &&TARGET_CALL_NO_KW_TUPLE_1, + &&TARGET_CALL_NO_KW_TYPE_1, + &&TARGET_COMPARE_OP_FLOAT_JUMP, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -55,37 +55,37 @@ static void *opcode_targets[256] = { &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, - &&TARGET_CALL_NO_KW_TUPLE_1, - &&TARGET_CALL_NO_KW_TYPE_1, - &&TARGET_COMPARE_OP_ADAPTIVE, - &&TARGET_COMPARE_OP_FLOAT_JUMP, - &&TARGET_STORE_SUBSCR, - &&TARGET_DELETE_SUBSCR, &&TARGET_COMPARE_OP_INT_JUMP, - &&TARGET_STOPITERATION_ERROR, &&TARGET_COMPARE_OP_STR_JUMP, &&TARGET_EXTENDED_ARG_QUICK, - &&TARGET_FOR_ITER_ADAPTIVE, &&TARGET_FOR_ITER_LIST, - &&TARGET_GET_ITER, - &&TARGET_GET_YIELD_FROM_ITER, - &&TARGET_PRINT_EXPR, - &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_STORE_SUBSCR, + &&TARGET_DELETE_SUBSCR, &&TARGET_FOR_ITER_RANGE, - &&TARGET_LOAD_ATTR_ADAPTIVE, - &&TARGET_LOAD_ASSERTION_ERROR, - &&TARGET_RETURN_GENERATOR, + &&TARGET_STOPITERATION_ERROR, &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_ATTR_MODULE, + &&TARGET_GET_ITER, + &&TARGET_GET_YIELD_FROM_ITER, + &&TARGET_PRINT_EXPR, + &&TARGET_LOAD_BUILD_CLASS, &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ATTR_SLOT, + &&TARGET_LOAD_ASSERTION_ERROR, + &&TARGET_RETURN_GENERATOR, + &&TARGET_LOAD_ATTR_WITH_HINT, + &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, + &&TARGET_LOAD_ATTR_METHOD_NO_DICT, + &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_ATTR_WITH_HINT, + &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_LOAD_ATTR_METHOD_NO_DICT, + &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, + &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,36 +152,36 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_LOAD_CONST__LOAD_FAST, - &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_STORE_ATTR_SLOT, + &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, - &&TARGET_LOAD_FAST__LOAD_FAST, - &&TARGET_LOAD_GLOBAL_ADAPTIVE, - &&TARGET_LOAD_GLOBAL_BUILTIN, - &&TARGET_LOAD_GLOBAL_MODULE, - &&TARGET_LIST_EXTEND, - &&TARGET_SET_UPDATE, - &&TARGET_DICT_MERGE, - &&TARGET_DICT_UPDATE, - &&TARGET_STORE_ATTR_ADAPTIVE, - &&TARGET_STORE_ATTR_INSTANCE_VALUE, - &&TARGET_STORE_ATTR_SLOT, - &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_FAST__LOAD_FAST, - &&TARGET_CALL, - &&TARGET_KW_NAMES, &&TARGET_STORE_FAST__STORE_FAST, - &&TARGET_STORE_SUBSCR_ADAPTIVE, &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, - &&TARGET_UNPACK_SEQUENCE_ADAPTIVE, + &&TARGET_LIST_EXTEND, + &&TARGET_SET_UPDATE, + &&TARGET_DICT_MERGE, + &&TARGET_DICT_UPDATE, &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&_unknown_opcode, &&_unknown_opcode, + &&TARGET_CALL, + &&TARGET_KW_NAMES, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index 70a456cf0b54a0..ee8fab3173fab5 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -16,21 +16,6 @@ * ./adaptive.md */ -/* Map from opcode to adaptive opcode. - Values of zero are ignored. */ -uint8_t _PyOpcode_Adaptive[256] = { - [LOAD_ATTR] = LOAD_ATTR_ADAPTIVE, - [LOAD_GLOBAL] = LOAD_GLOBAL_ADAPTIVE, - [BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE, - [STORE_SUBSCR] = STORE_SUBSCR_ADAPTIVE, - [CALL] = CALL_ADAPTIVE, - [STORE_ATTR] = STORE_ATTR_ADAPTIVE, - [BINARY_OP] = BINARY_OP_ADAPTIVE, - [COMPARE_OP] = COMPARE_OP_ADAPTIVE, - [UNPACK_SEQUENCE] = UNPACK_SEQUENCE_ADAPTIVE, - [FOR_ITER] = FOR_ITER_ADAPTIVE, -}; - #ifdef Py_STATS PyStats _py_stats_struct = { 0 }; PyStats *_py_stats = &_py_stats_struct; @@ -143,7 +128,7 @@ print_spec_stats(FILE *out, OpcodeStats *stats) fprintf(out, "opcode[%d].specializable : 1\n", BINARY_SLICE); fprintf(out, "opcode[%d].specializable : 1\n", STORE_SLICE); for (int i = 0; i < 256; i++) { - if (_PyOpcode_Adaptive[i]) { + if (_PyOpcode_Caches[i]) { fprintf(out, "opcode[%d].specializable : 1\n", i); } PRINT_STAT(i, specialization.success); @@ -283,15 +268,13 @@ _PyCode_Quicken(PyCodeObject *code) _Py_CODEUNIT *instructions = _PyCode_CODE(code); for (int i = 0; i < Py_SIZE(code); i++) { int opcode = _PyOpcode_Deopt[_Py_OPCODE(instructions[i])]; - uint8_t adaptive_opcode = _PyOpcode_Adaptive[opcode]; - if (adaptive_opcode) { - _Py_SET_OPCODE(instructions[i], adaptive_opcode); + int caches = _PyOpcode_Caches[opcode]; + if (caches) { instructions[i + 1] = adaptive_counter_start(); previous_opcode = -1; - i += _PyOpcode_Caches[opcode]; + i += caches; } else { - assert(!_PyOpcode_Caches[opcode]); switch (opcode) { case EXTENDED_ARG: _Py_SET_OPCODE(instructions[i], EXTENDED_ARG_QUICK); @@ -1477,7 +1460,6 @@ static int specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames) { - assert(_Py_OPCODE(*instr) == CALL_ADAPTIVE); PyTypeObject *tp = _PyType_CAST(callable); if (tp->tp_new == PyBaseObject_Type.tp_new) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PYTHON_CLASS); @@ -1539,7 +1521,6 @@ static int specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames) { - assert(_Py_OPCODE(*instr) == CALL_ADAPTIVE); if (kwnames) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); return -1; @@ -1591,7 +1572,6 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames, bool bound_method) { _PyCallCache *cache = (_PyCallCache *)(instr + 1); - assert(_Py_OPCODE(*instr) == CALL_ADAPTIVE); PyCodeObject *code = (PyCodeObject *)func->func_code; int kind = function_kind(code); /* Don't specialize if PEP 523 is active */ @@ -1646,7 +1626,6 @@ static int specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames) { - assert(_Py_OPCODE(*instr) == CALL_ADAPTIVE); if (PyCFunction_GET_FUNCTION(callable) == NULL) { return 1; } @@ -1921,16 +1900,16 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, goto success; } break; -#ifndef Py_STATS - default: - // These operators don't have any available specializations. Rather - // than repeatedly attempting to specialize them, just convert them - // back to BINARY_OP (unless we're collecting stats, where it's more - // important to get accurate hit counts for the unadaptive version - // and each of the different failure types): - _Py_SET_OPCODE(*instr, BINARY_OP); - return; -#endif +// #ifndef Py_STATS +// default: +// // These operators don't have any available specializations. Rather +// // than repeatedly attempting to specialize them, just convert them +// // back to BINARY_OP (unless we're collecting stats, where it's more +// // important to get accurate hit counts for the unadaptive version +// // and each of the different failure types): +// _Py_SET_OPCODE(*instr, BINARY_OP); +// return; +// #endif } SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs)); STAT_INC(BINARY_OP, failure); @@ -2003,17 +1982,17 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, // we're collecting stats, where it's more important to get accurate hit // counts for the unadaptive version and each of the different failure // types): -#ifndef Py_STATS - _Py_SET_OPCODE(*instr, COMPARE_OP); - return; -#else +// #ifndef Py_STATS +// _Py_SET_OPCODE(*instr, COMPARE_OP); +// return; +// #else if (next_opcode == EXTENDED_ARG) { SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_EXTENDED_ARG); goto failure; } SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_NOT_FOLLOWED_BY_COND_JUMP); goto failure; -#endif +// #endif } assert(oparg <= Py_GE); int when_to_jump_mask = compare_masks[oparg]; diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index dbfb0e01e7f753..ea16ed2f5e941f 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -501,6 +501,5 @@ Python/pystate.c - initial - Python/specialize.c - adaptive_opcodes - Python/specialize.c - cache_requirements - Python/specialize.c - compare_masks - -Python/specialize.c - _PyOpcode_Adaptive - Python/stdlib_module_names.h - _Py_stdlib_module_names - Python/sysmodule.c - whatstrings - From 15c806fc223ed8ab6c6357497dcc7443f6f942e3 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 20 Oct 2022 02:30:46 -0700 Subject: [PATCH 02/21] Merge EXTENDED_ARG and EXTENDED_ARG_QUICK --- Include/internal/pycore_opcode.h | 28 ++++++++-------- Include/opcode.h | 57 ++++++++++++++++---------------- Lib/opcode.py | 3 -- Lib/test/test_embed.py | 4 --- Python/bytecodes.c | 22 ++++-------- Python/compile.c | 3 -- Python/generated_cases.c.h | 20 ++++------- Python/opcode_targets.h | 26 +++++++-------- Python/specialize.c | 57 +++++++++++--------------------- 9 files changed, 89 insertions(+), 131 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 3ec937d975e51a..40be6163fefb71 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -120,7 +120,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [END_ASYNC_FOR] = END_ASYNC_FOR, [END_FOR] = END_FOR, [EXTENDED_ARG] = EXTENDED_ARG, - [EXTENDED_ARG_QUICK] = EXTENDED_ARG, [FORMAT_VALUE] = FORMAT_VALUE, [FOR_ITER] = FOR_ITER, [FOR_ITER_LIST] = FOR_ITER, @@ -288,35 +287,35 @@ static const char *const _PyOpcode_OpName[263] = { [CLEANUP_THROW] = "CLEANUP_THROW", [COMPARE_OP_INT_JUMP] = "COMPARE_OP_INT_JUMP", [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP", - [EXTENDED_ARG_QUICK] = "EXTENDED_ARG_QUICK", [FOR_ITER_LIST] = "FOR_ITER_LIST", + [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", - [FOR_ITER_RANGE] = "FOR_ITER_RANGE", - [STOPITERATION_ERROR] = "STOPITERATION_ERROR", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", + [STOPITERATION_ERROR] = "STOPITERATION_ERROR", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [PRINT_EXPR] = "PRINT_EXPR", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", [LIST_TO_TUPLE] = "LIST_TO_TUPLE", [RETURN_VALUE] = "RETURN_VALUE", [IMPORT_STAR] = "IMPORT_STAR", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", @@ -343,7 +342,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -351,7 +350,7 @@ static const char *const _PyOpcode_OpName[263] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -371,9 +370,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", - [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", + [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -383,22 +382,22 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", + [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", - [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", + [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", - [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", + [168] = "<168>", [169] = "<169>", [170] = "<170>", [CALL] = "CALL", @@ -497,6 +496,7 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ + case 168: \ case 169: \ case 170: \ case 173: \ diff --git a/Include/opcode.h b/Include/opcode.h index 26e7c63d40e796..48d718a9615bd9 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -160,35 +160,34 @@ extern "C" { #define COMPARE_OP_FLOAT_JUMP 48 #define COMPARE_OP_INT_JUMP 56 #define COMPARE_OP_STR_JUMP 57 -#define EXTENDED_ARG_QUICK 58 -#define FOR_ITER_LIST 59 -#define FOR_ITER_RANGE 62 -#define LOAD_ATTR_CLASS 64 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 65 -#define LOAD_ATTR_INSTANCE_VALUE 66 -#define LOAD_ATTR_MODULE 67 -#define LOAD_ATTR_PROPERTY 72 -#define LOAD_ATTR_SLOT 73 -#define LOAD_ATTR_WITH_HINT 76 -#define LOAD_ATTR_METHOD_LAZY_DICT 77 -#define LOAD_ATTR_METHOD_NO_DICT 78 -#define LOAD_ATTR_METHOD_WITH_DICT 79 -#define LOAD_ATTR_METHOD_WITH_VALUES 80 -#define LOAD_CONST__LOAD_FAST 81 -#define LOAD_FAST__LOAD_CONST 86 -#define LOAD_FAST__LOAD_FAST 113 -#define LOAD_GLOBAL_BUILTIN 121 -#define LOAD_GLOBAL_MODULE 141 -#define STORE_ATTR_INSTANCE_VALUE 143 -#define STORE_ATTR_SLOT 153 -#define STORE_ATTR_WITH_HINT 154 -#define STORE_FAST__LOAD_FAST 158 -#define STORE_FAST__STORE_FAST 159 -#define STORE_SUBSCR_DICT 160 -#define STORE_SUBSCR_LIST_INT 161 -#define UNPACK_SEQUENCE_LIST 166 -#define UNPACK_SEQUENCE_TUPLE 167 -#define UNPACK_SEQUENCE_TWO_TUPLE 168 +#define FOR_ITER_LIST 58 +#define FOR_ITER_RANGE 59 +#define LOAD_ATTR_CLASS 62 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 64 +#define LOAD_ATTR_INSTANCE_VALUE 65 +#define LOAD_ATTR_MODULE 66 +#define LOAD_ATTR_PROPERTY 67 +#define LOAD_ATTR_SLOT 72 +#define LOAD_ATTR_WITH_HINT 73 +#define LOAD_ATTR_METHOD_LAZY_DICT 76 +#define LOAD_ATTR_METHOD_NO_DICT 77 +#define LOAD_ATTR_METHOD_WITH_DICT 78 +#define LOAD_ATTR_METHOD_WITH_VALUES 79 +#define LOAD_CONST__LOAD_FAST 80 +#define LOAD_FAST__LOAD_CONST 81 +#define LOAD_FAST__LOAD_FAST 86 +#define LOAD_GLOBAL_BUILTIN 113 +#define LOAD_GLOBAL_MODULE 121 +#define STORE_ATTR_INSTANCE_VALUE 141 +#define STORE_ATTR_SLOT 143 +#define STORE_ATTR_WITH_HINT 153 +#define STORE_FAST__LOAD_FAST 154 +#define STORE_FAST__STORE_FAST 158 +#define STORE_SUBSCR_DICT 159 +#define STORE_SUBSCR_LIST_INT 160 +#define UNPACK_SEQUENCE_LIST 161 +#define UNPACK_SEQUENCE_TUPLE 166 +#define UNPACK_SEQUENCE_TWO_TUPLE 167 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/opcode.py b/Lib/opcode.py index fc0fb13a270a59..8b2aea776da972 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -317,9 +317,6 @@ def pseudo_op(name, op, real_ops): "COMPARE_OP_INT_JUMP", "COMPARE_OP_STR_JUMP", ], - "EXTENDED_ARG": [ - "EXTENDED_ARG_QUICK", - ], "FOR_ITER": [ "FOR_ITER_LIST", "FOR_ITER_RANGE", diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 8c92f495d16704..2dda7ccf7bf80c 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -362,10 +362,6 @@ def is_specialized(f): opname in opcode._specialized_instructions # Exclude superinstructions: and "__" not in opname - # Exclude adaptive instructions: - and not opname.endswith("_ADAPTIVE") - # Exclude "quick" instructions: - and not opname.endswith("_QUICK") ): return True return False diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 974d2518ff6b6b..86527a2a6afc08 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3910,12 +3910,15 @@ dummy_func( // stack effect: ( -- ) inst(EXTENDED_ARG) { assert(oparg); + opcode = _Py_OPCODE(*next_instr); + if (cframe.use_tracing) { + // Deoptimize the next opcode to avoid breaking tracing + // guarantees in quickened instructions: + opcode = _PyOpcode_Deopt[opcode]; + PRE_DISPATCH_GOTO(); + } oparg <<= 8; oparg |= _Py_OPARG(*next_instr); - // We might be tracing. To avoid breaking tracing guarantees in - // quickened instructions, always deoptimize the next opcode: - opcode = _PyOpcode_Deopt[_Py_OPCODE(*next_instr)]; - PRE_DISPATCH_GOTO(); // CPython hasn't traced the following instruction historically // (DO_TRACING would clobber our extended oparg anyways), so just // skip our usual cframe.use_tracing check before dispatch. Also, @@ -3925,16 +3928,6 @@ dummy_func( DISPATCH_GOTO(); } - // stack effect: ( -- ) - inst(EXTENDED_ARG_QUICK) { - assert(cframe.use_tracing == 0); - assert(oparg); - int oldoparg = oparg; - NEXTOPARG(); - oparg |= oldoparg << 8; - DISPATCH_GOTO(); - } - // stack effect: ( -- ) inst(CACHE) { Py_UNREACHABLE(); @@ -3974,7 +3967,6 @@ family(call) = { family(compare_op) = { COMPARE_OP, COMPARE_OP_FLOAT_JUMP, COMPARE_OP_INT_JUMP, COMPARE_OP_STR_JUMP }; -family(extended_arg) = { EXTENDED_ARG, EXTENDED_ARG_QUICK }; family(for_iter) = { FOR_ITER, FOR_ITER_LIST, FOR_ITER_RANGE }; diff --git a/Python/compile.c b/Python/compile.c index 065d1b08d0642f..030378f19a3362 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -8062,7 +8062,6 @@ scan_block_for_locals(basicblock *b, basicblock ***sp) for (int i = 0; i < b->b_iused; i++) { struct instr *instr = &b->b_instr[i]; assert(instr->i_opcode != EXTENDED_ARG); - assert(instr->i_opcode != EXTENDED_ARG_QUICK); assert(!IS_SUPERINSTRUCTION_OPCODE(instr->i_opcode)); if (instr->i_except != NULL) { maybe_push(instr->i_except, unsafe_mask, sp); @@ -8119,7 +8118,6 @@ fast_scan_many_locals(basicblock *entryblock, int nlocals) for (int i = 0; i < b->b_iused; i++) { struct instr *instr = &b->b_instr[i]; assert(instr->i_opcode != EXTENDED_ARG); - assert(instr->i_opcode != EXTENDED_ARG_QUICK); assert(!IS_SUPERINSTRUCTION_OPCODE(instr->i_opcode)); int arg = instr->i_oparg; if (arg < 64) { @@ -8667,7 +8665,6 @@ fix_cell_offsets(struct compiler *c, basicblock *entryblock, int *fixedmap) struct instr *inst = &b->b_instr[i]; // This is called before extended args are generated. assert(inst->i_opcode != EXTENDED_ARG); - assert(inst->i_opcode != EXTENDED_ARG_QUICK); int oldoffset = inst->i_oparg; switch(inst->i_opcode) { case MAKE_CELL: diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a6ea4139b668bd..8486589ecb5614 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3826,12 +3826,15 @@ TARGET(EXTENDED_ARG) { assert(oparg); + opcode = _Py_OPCODE(*next_instr); + if (cframe.use_tracing) { + // Deoptimize the next opcode to avoid breaking tracing + // guarantees in quickened instructions: + opcode = _PyOpcode_Deopt[opcode]; + PRE_DISPATCH_GOTO(); + } oparg <<= 8; oparg |= _Py_OPARG(*next_instr); - // We might be tracing. To avoid breaking tracing guarantees in - // quickened instructions, always deoptimize the next opcode: - opcode = _PyOpcode_Deopt[_Py_OPCODE(*next_instr)]; - PRE_DISPATCH_GOTO(); // CPython hasn't traced the following instruction historically // (DO_TRACING would clobber our extended oparg anyways), so just // skip our usual cframe.use_tracing check before dispatch. Also, @@ -3841,15 +3844,6 @@ DISPATCH_GOTO(); } - TARGET(EXTENDED_ARG_QUICK) { - assert(cframe.use_tracing == 0); - assert(oparg); - int oldoparg = oparg; - NEXTOPARG(); - oparg |= oldoparg << 8; - DISPATCH_GOTO(); - } - TARGET(CACHE) { Py_UNREACHABLE(); } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index fb22110784d098..a60f8b324a3141 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -57,35 +57,35 @@ static void *opcode_targets[256] = { &&TARGET_CLEANUP_THROW, &&TARGET_COMPARE_OP_INT_JUMP, &&TARGET_COMPARE_OP_STR_JUMP, - &&TARGET_EXTENDED_ARG_QUICK, &&TARGET_FOR_ITER_LIST, + &&TARGET_FOR_ITER_RANGE, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, - &&TARGET_FOR_ITER_RANGE, - &&TARGET_STOPITERATION_ERROR, &&TARGET_LOAD_ATTR_CLASS, + &&TARGET_STOPITERATION_ERROR, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_ATTR_MODULE, + &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, - &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ATTR_SLOT, + &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, - &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_GLOBAL_MODULE, - &&TARGET_CALL_FUNCTION_EX, &&TARGET_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_CALL_FUNCTION_EX, + &&TARGET_STORE_ATTR_SLOT, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,24 +152,24 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, - &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, + &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, - &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_CALL, &&TARGET_KW_NAMES, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index ee8fab3173fab5..b5f744a6e6feca 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -264,52 +264,35 @@ do { \ void _PyCode_Quicken(PyCodeObject *code) { - int previous_opcode = -1; + int previous_opcode = 0; _Py_CODEUNIT *instructions = _PyCode_CODE(code); for (int i = 0; i < Py_SIZE(code); i++) { int opcode = _PyOpcode_Deopt[_Py_OPCODE(instructions[i])]; int caches = _PyOpcode_Caches[opcode]; if (caches) { instructions[i + 1] = adaptive_counter_start(); - previous_opcode = -1; + previous_opcode = 0; i += caches; + continue; } - else { - switch (opcode) { - case EXTENDED_ARG: - _Py_SET_OPCODE(instructions[i], EXTENDED_ARG_QUICK); - break; - case LOAD_FAST: - switch(previous_opcode) { - case LOAD_FAST: - _Py_SET_OPCODE(instructions[i - 1], - LOAD_FAST__LOAD_FAST); - break; - case STORE_FAST: - _Py_SET_OPCODE(instructions[i - 1], - STORE_FAST__LOAD_FAST); - break; - case LOAD_CONST: - _Py_SET_OPCODE(instructions[i - 1], - LOAD_CONST__LOAD_FAST); - break; - } - break; - case STORE_FAST: - if (previous_opcode == STORE_FAST) { - _Py_SET_OPCODE(instructions[i - 1], - STORE_FAST__STORE_FAST); - } - break; - case LOAD_CONST: - if (previous_opcode == LOAD_FAST) { - _Py_SET_OPCODE(instructions[i - 1], - LOAD_FAST__LOAD_CONST); - } - break; - } - previous_opcode = opcode; + switch (previous_opcode << 8 | opcode) { + case LOAD_FAST << 8 | LOAD_FAST: + _Py_SET_OPCODE(instructions[i - 1], LOAD_FAST__LOAD_FAST); + break; + case STORE_FAST << 8 | LOAD_FAST: + _Py_SET_OPCODE(instructions[i - 1], STORE_FAST__LOAD_FAST); + break; + case LOAD_CONST << 8 | LOAD_FAST: + _Py_SET_OPCODE(instructions[i - 1], LOAD_CONST__LOAD_FAST); + break; + case STORE_FAST << 8 | STORE_FAST: + _Py_SET_OPCODE(instructions[i - 1], STORE_FAST__STORE_FAST); + break; + case LOAD_FAST << 8 | LOAD_CONST: + _Py_SET_OPCODE(instructions[i - 1], LOAD_FAST__LOAD_CONST); + break; } + previous_opcode = opcode; } } From 7c164d272672a1e26dd9f4335a0821f112e3a341 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 20 Oct 2022 07:38:53 -0700 Subject: [PATCH 03/21] Add back BINARY_OP_GENERIC and COMPARE_OP_GENERIC --- Include/internal/pycore_opcode.h | 56 +++++++-------- Include/opcode.h | 116 ++++++++++++++++--------------- Lib/opcode.py | 2 + Python/bytecodes.c | 15 +++- Python/generated_cases.c.h | 10 +++ Python/opcode_targets.h | 52 +++++++------- Python/specialize.c | 30 ++++---- 7 files changed, 153 insertions(+), 128 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 40be6163fefb71..defd70ca251c3d 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -61,6 +61,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, [BINARY_OP_ADD_UNICODE] = BINARY_OP, + [BINARY_OP_GENERIC] = BINARY_OP, [BINARY_OP_INPLACE_ADD_UNICODE] = BINARY_OP, [BINARY_OP_MULTIPLY_FLOAT] = BINARY_OP, [BINARY_OP_MULTIPLY_INT] = BINARY_OP, @@ -104,6 +105,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [CLEANUP_THROW] = CLEANUP_THROW, [COMPARE_OP] = COMPARE_OP, [COMPARE_OP_FLOAT_JUMP] = COMPARE_OP, + [COMPARE_OP_GENERIC] = COMPARE_OP, [COMPARE_OP_INT_JUMP] = COMPARE_OP, [COMPARE_OP_STR_JUMP] = COMPARE_OP, [CONTAINS_OP] = CONTAINS_OP, @@ -236,15 +238,16 @@ static const char *const _PyOpcode_OpName[263] = { [END_FOR] = "END_FOR", [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", + [BINARY_OP_GENERIC] = "BINARY_OP_GENERIC", [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", - [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", [NOP] = "NOP", [UNARY_POSITIVE] = "UNARY_POSITIVE", [UNARY_NEGATIVE] = "UNARY_NEGATIVE", [UNARY_NOT] = "UNARY_NOT", + [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", - [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", [UNARY_INVERT] = "UNARY_INVERT", + [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", @@ -253,20 +256,20 @@ static const char *const _PyOpcode_OpName[263] = { [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", - [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [BINARY_SUBSCR] = "BINARY_SUBSCR", [BINARY_SLICE] = "BINARY_SLICE", [STORE_SLICE] = "STORE_SLICE", + [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [GET_LEN] = "GET_LEN", [MATCH_MAPPING] = "MATCH_MAPPING", [MATCH_SEQUENCE] = "MATCH_SEQUENCE", [MATCH_KEYS] = "MATCH_KEYS", - [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", [CHECK_EG_MATCH] = "CHECK_EG_MATCH", + [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", @@ -277,7 +280,6 @@ static const char *const _PyOpcode_OpName[263] = { [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", - [COMPARE_OP_FLOAT_JUMP] = "COMPARE_OP_FLOAT_JUMP", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", @@ -285,37 +287,37 @@ static const char *const _PyOpcode_OpName[263] = { [BEFORE_WITH] = "BEFORE_WITH", [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", + [COMPARE_OP_FLOAT_JUMP] = "COMPARE_OP_FLOAT_JUMP", + [COMPARE_OP_GENERIC] = "COMPARE_OP_GENERIC", [COMPARE_OP_INT_JUMP] = "COMPARE_OP_INT_JUMP", [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP", - [FOR_ITER_LIST] = "FOR_ITER_LIST", - [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", - [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", + [FOR_ITER_LIST] = "FOR_ITER_LIST", [STOPITERATION_ERROR] = "STOPITERATION_ERROR", + [FOR_ITER_RANGE] = "FOR_ITER_RANGE", + [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [PRINT_EXPR] = "PRINT_EXPR", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", - [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", + [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", [LIST_TO_TUPLE] = "LIST_TO_TUPLE", [RETURN_VALUE] = "RETURN_VALUE", [IMPORT_STAR] = "IMPORT_STAR", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", + [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", @@ -342,7 +344,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -350,7 +352,7 @@ static const char *const _PyOpcode_OpName[263] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -370,9 +372,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -382,23 +384,23 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", - [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", + [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", + [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", - [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", - [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", + [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [168] = "<168>", - [169] = "<169>", [170] = "<170>", [CALL] = "CALL", [KW_NAMES] = "KW_NAMES", @@ -496,8 +498,6 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ - case 168: \ - case 169: \ case 170: \ case 173: \ case 174: \ diff --git a/Include/opcode.h b/Include/opcode.h index 48d718a9615bd9..29ee69d465a7e8 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -131,63 +131,65 @@ extern "C" { #define BINARY_OP_ADD_FLOAT 3 #define BINARY_OP_ADD_INT 5 #define BINARY_OP_ADD_UNICODE 6 -#define BINARY_OP_INPLACE_ADD_UNICODE 7 -#define BINARY_OP_MULTIPLY_FLOAT 8 -#define BINARY_OP_MULTIPLY_INT 13 -#define BINARY_OP_SUBTRACT_FLOAT 14 -#define BINARY_OP_SUBTRACT_INT 16 -#define BINARY_SUBSCR_DICT 17 -#define BINARY_SUBSCR_GETITEM 18 -#define BINARY_SUBSCR_LIST_INT 19 -#define BINARY_SUBSCR_TUPLE_INT 20 -#define CALL_PY_EXACT_ARGS 21 -#define CALL_PY_WITH_DEFAULTS 22 -#define CALL_BOUND_METHOD_EXACT_ARGS 23 -#define CALL_BUILTIN_CLASS 24 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 28 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 29 -#define CALL_NO_KW_BUILTIN_FAST 34 -#define CALL_NO_KW_BUILTIN_O 38 -#define CALL_NO_KW_ISINSTANCE 39 -#define CALL_NO_KW_LEN 40 -#define CALL_NO_KW_LIST_APPEND 41 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 42 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 43 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 44 -#define CALL_NO_KW_STR_1 45 -#define CALL_NO_KW_TUPLE_1 46 -#define CALL_NO_KW_TYPE_1 47 -#define COMPARE_OP_FLOAT_JUMP 48 -#define COMPARE_OP_INT_JUMP 56 -#define COMPARE_OP_STR_JUMP 57 -#define FOR_ITER_LIST 58 -#define FOR_ITER_RANGE 59 -#define LOAD_ATTR_CLASS 62 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 64 -#define LOAD_ATTR_INSTANCE_VALUE 65 -#define LOAD_ATTR_MODULE 66 -#define LOAD_ATTR_PROPERTY 67 -#define LOAD_ATTR_SLOT 72 -#define LOAD_ATTR_WITH_HINT 73 -#define LOAD_ATTR_METHOD_LAZY_DICT 76 -#define LOAD_ATTR_METHOD_NO_DICT 77 -#define LOAD_ATTR_METHOD_WITH_DICT 78 -#define LOAD_ATTR_METHOD_WITH_VALUES 79 -#define LOAD_CONST__LOAD_FAST 80 -#define LOAD_FAST__LOAD_CONST 81 -#define LOAD_FAST__LOAD_FAST 86 -#define LOAD_GLOBAL_BUILTIN 113 -#define LOAD_GLOBAL_MODULE 121 -#define STORE_ATTR_INSTANCE_VALUE 141 -#define STORE_ATTR_SLOT 143 -#define STORE_ATTR_WITH_HINT 153 -#define STORE_FAST__LOAD_FAST 154 -#define STORE_FAST__STORE_FAST 158 -#define STORE_SUBSCR_DICT 159 -#define STORE_SUBSCR_LIST_INT 160 -#define UNPACK_SEQUENCE_LIST 161 -#define UNPACK_SEQUENCE_TUPLE 166 -#define UNPACK_SEQUENCE_TWO_TUPLE 167 +#define BINARY_OP_GENERIC 7 +#define BINARY_OP_INPLACE_ADD_UNICODE 8 +#define BINARY_OP_MULTIPLY_FLOAT 13 +#define BINARY_OP_MULTIPLY_INT 14 +#define BINARY_OP_SUBTRACT_FLOAT 16 +#define BINARY_OP_SUBTRACT_INT 17 +#define BINARY_SUBSCR_DICT 18 +#define BINARY_SUBSCR_GETITEM 19 +#define BINARY_SUBSCR_LIST_INT 20 +#define BINARY_SUBSCR_TUPLE_INT 21 +#define CALL_PY_EXACT_ARGS 22 +#define CALL_PY_WITH_DEFAULTS 23 +#define CALL_BOUND_METHOD_EXACT_ARGS 24 +#define CALL_BUILTIN_CLASS 28 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 29 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 34 +#define CALL_NO_KW_BUILTIN_FAST 38 +#define CALL_NO_KW_BUILTIN_O 39 +#define CALL_NO_KW_ISINSTANCE 40 +#define CALL_NO_KW_LEN 41 +#define CALL_NO_KW_LIST_APPEND 42 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 43 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 44 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 45 +#define CALL_NO_KW_STR_1 46 +#define CALL_NO_KW_TUPLE_1 47 +#define CALL_NO_KW_TYPE_1 48 +#define COMPARE_OP_FLOAT_JUMP 56 +#define COMPARE_OP_GENERIC 57 +#define COMPARE_OP_INT_JUMP 58 +#define COMPARE_OP_STR_JUMP 59 +#define FOR_ITER_LIST 62 +#define FOR_ITER_RANGE 64 +#define LOAD_ATTR_CLASS 65 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 66 +#define LOAD_ATTR_INSTANCE_VALUE 67 +#define LOAD_ATTR_MODULE 72 +#define LOAD_ATTR_PROPERTY 73 +#define LOAD_ATTR_SLOT 76 +#define LOAD_ATTR_WITH_HINT 77 +#define LOAD_ATTR_METHOD_LAZY_DICT 78 +#define LOAD_ATTR_METHOD_NO_DICT 79 +#define LOAD_ATTR_METHOD_WITH_DICT 80 +#define LOAD_ATTR_METHOD_WITH_VALUES 81 +#define LOAD_CONST__LOAD_FAST 86 +#define LOAD_FAST__LOAD_CONST 113 +#define LOAD_FAST__LOAD_FAST 121 +#define LOAD_GLOBAL_BUILTIN 141 +#define LOAD_GLOBAL_MODULE 143 +#define STORE_ATTR_INSTANCE_VALUE 153 +#define STORE_ATTR_SLOT 154 +#define STORE_ATTR_WITH_HINT 158 +#define STORE_FAST__LOAD_FAST 159 +#define STORE_FAST__STORE_FAST 160 +#define STORE_SUBSCR_DICT 161 +#define STORE_SUBSCR_LIST_INT 166 +#define UNPACK_SEQUENCE_LIST 167 +#define UNPACK_SEQUENCE_TUPLE 168 +#define UNPACK_SEQUENCE_TWO_TUPLE 169 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/opcode.py b/Lib/opcode.py index 8b2aea776da972..0de42fe9989f43 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -281,6 +281,7 @@ def pseudo_op(name, op, real_ops): "BINARY_OP_ADD_FLOAT", "BINARY_OP_ADD_INT", "BINARY_OP_ADD_UNICODE", + "BINARY_OP_GENERIC", "BINARY_OP_INPLACE_ADD_UNICODE", "BINARY_OP_MULTIPLY_FLOAT", "BINARY_OP_MULTIPLY_INT", @@ -314,6 +315,7 @@ def pseudo_op(name, op, real_ops): ], "COMPARE_OP": [ "COMPARE_OP_FLOAT_JUMP", + "COMPARE_OP_GENERIC", "COMPARE_OP_INT_JUMP", "COMPARE_OP_STR_JUMP", ], diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 86527a2a6afc08..21d1c5a225b46c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2246,8 +2246,13 @@ dummy_func( JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); } + inst(COMPARE_OP_GENERIC) { + GO_TO_INSTRUCTION(COMPARE_OP); + } + // stack effect: (__0 -- ) inst(COMPARE_OP) { + PREDICTED(COMPARE_OP); _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (cframe.use_tracing || opcode != COMPARE_OP) { } @@ -3868,8 +3873,14 @@ dummy_func( Py_INCREF(peek); PUSH(peek); } + + inst(BINARY_OP_GENERIC) { + GO_TO_INSTRUCTION(BINARY_OP); + } + // stack effect: (__0 -- ) inst(BINARY_OP) { + PREDICTED(BINARY_OP); _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (cframe.use_tracing || opcode != BINARY_OP) { } @@ -3950,7 +3961,7 @@ dummy_func( family(binary_op) = { BINARY_OP, BINARY_OP_ADD_FLOAT, - BINARY_OP_ADD_INT, BINARY_OP_ADD_UNICODE, BINARY_OP_INPLACE_ADD_UNICODE, + BINARY_OP_ADD_INT, BINARY_OP_ADD_UNICODE, BINARY_OP_GENERIC, BINARY_OP_INPLACE_ADD_UNICODE, BINARY_OP_MULTIPLY_FLOAT, BINARY_OP_MULTIPLY_INT, BINARY_OP_SUBTRACT_FLOAT, BINARY_OP_SUBTRACT_INT }; family(binary_subscr) = { @@ -3965,7 +3976,7 @@ family(call) = { CALL_NO_KW_METHOD_DESCRIPTOR_O, CALL_NO_KW_STR_1, CALL_NO_KW_TUPLE_1, CALL_NO_KW_TYPE_1 }; family(compare_op) = { - COMPARE_OP, COMPARE_OP_FLOAT_JUMP, + COMPARE_OP, COMPARE_OP_FLOAT_JUMP, COMPARE_OP_GENERIC, COMPARE_OP_INT_JUMP, COMPARE_OP_STR_JUMP }; family(for_iter) = { FOR_ITER, FOR_ITER_LIST, diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 8486589ecb5614..6d050193977b2e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2167,7 +2167,12 @@ DISPATCH(); } + TARGET(COMPARE_OP_GENERIC) { + GO_TO_INSTRUCTION(COMPARE_OP); + } + TARGET(COMPARE_OP) { + PREDICTED(COMPARE_OP); _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (cframe.use_tracing || opcode != COMPARE_OP) { } @@ -3785,7 +3790,12 @@ DISPATCH(); } + TARGET(BINARY_OP_GENERIC) { + GO_TO_INSTRUCTION(BINARY_OP); + } + TARGET(BINARY_OP) { + PREDICTED(BINARY_OP); _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (cframe.use_tracing || opcode != BINARY_OP) { } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index a60f8b324a3141..e99a2c09a1e00c 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -6,15 +6,16 @@ static void *opcode_targets[256] = { &&TARGET_END_FOR, &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_UNICODE, + &&TARGET_BINARY_OP_GENERIC, &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, - &&TARGET_BINARY_OP_MULTIPLY_FLOAT, &&TARGET_NOP, &&TARGET_UNARY_POSITIVE, &&TARGET_UNARY_NEGATIVE, &&TARGET_UNARY_NOT, + &&TARGET_BINARY_OP_MULTIPLY_FLOAT, &&TARGET_BINARY_OP_MULTIPLY_INT, - &&TARGET_BINARY_OP_SUBTRACT_FLOAT, &&TARGET_UNARY_INVERT, + &&TARGET_BINARY_OP_SUBTRACT_FLOAT, &&TARGET_BINARY_OP_SUBTRACT_INT, &&TARGET_BINARY_SUBSCR_DICT, &&TARGET_BINARY_SUBSCR_GETITEM, @@ -23,20 +24,20 @@ static void *opcode_targets[256] = { &&TARGET_CALL_PY_EXACT_ARGS, &&TARGET_CALL_PY_WITH_DEFAULTS, &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, - &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_SLICE, &&TARGET_STORE_SLICE, + &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, - &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, &&TARGET_GET_LEN, &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_MATCH_KEYS, - &&TARGET_CALL_NO_KW_BUILTIN_FAST, + &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, &&TARGET_PUSH_EXC_INFO, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CHECK_EG_MATCH, + &&TARGET_CALL_NO_KW_BUILTIN_FAST, &&TARGET_CALL_NO_KW_BUILTIN_O, &&TARGET_CALL_NO_KW_ISINSTANCE, &&TARGET_CALL_NO_KW_LEN, @@ -47,7 +48,6 @@ static void *opcode_targets[256] = { &&TARGET_CALL_NO_KW_STR_1, &&TARGET_CALL_NO_KW_TUPLE_1, &&TARGET_CALL_NO_KW_TYPE_1, - &&TARGET_COMPARE_OP_FLOAT_JUMP, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -55,37 +55,37 @@ static void *opcode_targets[256] = { &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, + &&TARGET_COMPARE_OP_FLOAT_JUMP, + &&TARGET_COMPARE_OP_GENERIC, &&TARGET_COMPARE_OP_INT_JUMP, &&TARGET_COMPARE_OP_STR_JUMP, - &&TARGET_FOR_ITER_LIST, - &&TARGET_FOR_ITER_RANGE, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, - &&TARGET_LOAD_ATTR_CLASS, + &&TARGET_FOR_ITER_LIST, &&TARGET_STOPITERATION_ERROR, + &&TARGET_FOR_ITER_RANGE, + &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, - &&TARGET_LOAD_ATTR_MODULE, - &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, - &&TARGET_LOAD_ATTR_SLOT, - &&TARGET_LOAD_ATTR_WITH_HINT, + &&TARGET_LOAD_ATTR_MODULE, + &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_LOAD_ATTR_SLOT, + &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, - &&TARGET_LOAD_CONST__LOAD_FAST, - &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_STORE_ATTR_SLOT, + &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,24 +152,24 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_STORE_ATTR_WITH_HINT, - &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_STORE_ATTR_SLOT, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_DICT, - &&TARGET_STORE_SUBSCR_LIST_INT, - &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, + &&TARGET_STORE_SUBSCR_LIST_INT, + &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_CALL, &&TARGET_KW_NAMES, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index b5f744a6e6feca..204067ecd5b275 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1883,16 +1883,16 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, goto success; } break; -// #ifndef Py_STATS -// default: -// // These operators don't have any available specializations. Rather -// // than repeatedly attempting to specialize them, just convert them -// // back to BINARY_OP (unless we're collecting stats, where it's more -// // important to get accurate hit counts for the unadaptive version -// // and each of the different failure types): -// _Py_SET_OPCODE(*instr, BINARY_OP); -// return; -// #endif +#ifndef Py_STATS + default: + // These operators don't have any available specializations. Rather + // than repeatedly attempting to specialize them, just convert them + // back to BINARY_OP (unless we're collecting stats, where it's more + // important to get accurate hit counts for the unadaptive version + // and each of the different failure types): + _Py_SET_OPCODE(*instr, BINARY_OP_GENERIC); + return; +#endif } SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs)); STAT_INC(BINARY_OP, failure); @@ -1965,17 +1965,17 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, // we're collecting stats, where it's more important to get accurate hit // counts for the unadaptive version and each of the different failure // types): -// #ifndef Py_STATS -// _Py_SET_OPCODE(*instr, COMPARE_OP); -// return; -// #else +#ifndef Py_STATS + _Py_SET_OPCODE(*instr, COMPARE_OP_GENERIC); + return; +#else if (next_opcode == EXTENDED_ARG) { SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_EXTENDED_ARG); goto failure; } SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_NOT_FOLLOWED_BY_COND_JUMP); goto failure; -// #endif +#endif } assert(oparg <= Py_GE); int when_to_jump_mask = compare_masks[oparg]; From 2eda5205b0c7784736090adc6dfcc0c8e761523f Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 20 Oct 2022 10:30:31 -0700 Subject: [PATCH 04/21] Merge the miss and backoff counters --- Include/internal/pycore_code.h | 18 ++- Python/bytecodes.c | 220 +++++++++++++++------------------ Python/ceval.c | 8 -- Python/specialize.c | 43 +++---- 4 files changed, 134 insertions(+), 155 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 17a2f05072a5e9..2779d108fab8eb 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -375,8 +375,12 @@ write_location_entry_start(uint8_t *ptr, int code, int length) /* With a 16-bit counter, we have 12 bits for the counter value, and 4 bits for the backoff */ #define ADAPTIVE_BACKOFF_BITS 4 -/* The initial counter value is 1 == 2**ADAPTIVE_BACKOFF_START - 1 */ -#define ADAPTIVE_BACKOFF_START 1 + +#define ADAPTIVE_BACKOFF_START_VALUE 1 +#define ADAPTIVE_BACKOFF_START_BACKOFF 1 // 2**0 <= 1 < 2**1 + +#define ADAPTIVE_BACKOFF_RESTART_VALUE 52 +#define ADAPTIVE_BACKOFF_RESTART_BACKOFF 6 // 2**5 <= 53 < 2**6 #define MAX_BACKOFF_VALUE (16 - ADAPTIVE_BACKOFF_BITS) @@ -389,8 +393,14 @@ adaptive_counter_bits(int value, int backoff) { static inline uint16_t adaptive_counter_start(void) { - unsigned int value = (1 << ADAPTIVE_BACKOFF_START) - 1; - return adaptive_counter_bits(value, ADAPTIVE_BACKOFF_START); + return adaptive_counter_bits(ADAPTIVE_BACKOFF_START_VALUE, + ADAPTIVE_BACKOFF_START_BACKOFF); +} + +static inline uint16_t +adaptive_counter_restart(void) { + return adaptive_counter_bits(ADAPTIVE_BACKOFF_RESTART_VALUE, + ADAPTIVE_BACKOFF_RESTART_BACKOFF); } static inline uint16_t diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 21d1c5a225b46c..2dd7646342e885 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -471,19 +471,17 @@ dummy_func( // stack effect: (__0 -- ) inst(BINARY_SUBSCR) { - _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (cframe.use_tracing || opcode != BINARY_SUBSCR) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *sub = TOP(); - PyObject *container = SECOND(); - next_instr--; - if (_Py_Specialize_BinarySubscr(container, sub, next_instr) < 0) { - goto error; + if (!cframe.use_tracing) { + _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *sub = TOP(); + PyObject *container = SECOND(); + next_instr--; + if (_Py_Specialize_BinarySubscr(container, sub, next_instr) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); } - DISPATCH_SAME_OPARG(); - } - else { STAT_INC(BINARY_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } @@ -627,19 +625,17 @@ dummy_func( // stack effect: (__0, __1, __2 -- ) inst(STORE_SUBSCR) { - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (cframe.use_tracing || opcode != STORE_SUBSCR) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *sub = TOP(); - PyObject *container = SECOND(); - next_instr--; - if (_Py_Specialize_StoreSubscr(container, sub, next_instr) < 0) { - goto error; + if (!cframe.use_tracing) { + _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *sub = TOP(); + PyObject *container = SECOND(); + next_instr--; + if (_Py_Specialize_StoreSubscr(container, sub, next_instr) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); } - DISPATCH_SAME_OPARG(); - } - else { STAT_INC(STORE_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } @@ -1196,16 +1192,14 @@ dummy_func( // stack effect: (__0 -- __array[oparg]) inst(UNPACK_SEQUENCE) { - _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (cframe.use_tracing || opcode != UNPACK_SEQUENCE) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *seq = TOP(); - next_instr--; - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - else { + if (!cframe.use_tracing) { + _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *seq = TOP(); + next_instr--; + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } STAT_INC(UNPACK_SEQUENCE, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } @@ -1377,18 +1371,16 @@ dummy_func( // error: LOAD_GLOBAL has irregular stack effect inst(LOAD_GLOBAL) { - _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (cframe.use_tracing || opcode != LOAD_GLOBAL) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *name = GETITEM(names, oparg>>1); - next_instr--; - if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) { - goto error; + if (!cframe.use_tracing) { + _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *name = GETITEM(names, oparg>>1); + next_instr--; + if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); } - DISPATCH_SAME_OPARG(); - } - else { STAT_INC(LOAD_GLOBAL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } @@ -1836,19 +1828,17 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR) { - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (cframe.use_tracing || opcode != LOAD_ATTR) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *owner = TOP(); - PyObject *name = GETITEM(names, oparg>>1); - next_instr--; - if (_Py_Specialize_LoadAttr(owner, next_instr, name) < 0) { - goto error; + if (!cframe.use_tracing) { + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *owner = TOP(); + PyObject *name = GETITEM(names, oparg>>1); + next_instr--; + if (_Py_Specialize_LoadAttr(owner, next_instr, name) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); } - DISPATCH_SAME_OPARG(); - } - else { STAT_INC(LOAD_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } @@ -2115,19 +2105,17 @@ dummy_func( // stack effect: (__0, __1 -- ) inst(STORE_ATTR) { - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (cframe.use_tracing || opcode != STORE_ATTR) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *owner = TOP(); - PyObject *name = GETITEM(names, oparg); - next_instr--; - if (_Py_Specialize_StoreAttr(owner, next_instr, name) < 0) { - goto error; + if (!cframe.use_tracing) { + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *owner = TOP(); + PyObject *name = GETITEM(names, oparg); + next_instr--; + if (_Py_Specialize_StoreAttr(owner, next_instr, name) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); } - DISPATCH_SAME_OPARG(); - } - else { STAT_INC(STORE_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } @@ -2247,26 +2235,24 @@ dummy_func( } inst(COMPARE_OP_GENERIC) { - GO_TO_INSTRUCTION(COMPARE_OP); + goto compare_op; } // stack effect: (__0 -- ) inst(COMPARE_OP) { - PREDICTED(COMPARE_OP); - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (cframe.use_tracing || opcode != COMPARE_OP) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *right = TOP(); - PyObject *left = SECOND(); - next_instr--; - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - else { + if (!cframe.use_tracing) { + _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *right = TOP(); + PyObject *left = SECOND(); + next_instr--; + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } STAT_INC(COMPARE_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } + compare_op: assert(oparg <= Py_GE); PyObject *right = POP(); PyObject *left = TOP(); @@ -2760,15 +2746,13 @@ dummy_func( // stack effect: ( -- __0) inst(FOR_ITER) { - _PyForIterCache *cache = (_PyForIterCache *)next_instr; - if (cframe.use_tracing || opcode != FOR_ITER) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - next_instr--; - _Py_Specialize_ForIter(TOP(), next_instr); - DISPATCH_SAME_OPARG(); - } - else { + if (!cframe.use_tracing) { + _PyForIterCache *cache = (_PyForIterCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + next_instr--; + _Py_Specialize_ForIter(TOP(), next_instr); + DISPATCH_SAME_OPARG(); + } STAT_INC(FOR_ITER, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } @@ -3085,22 +3069,20 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL) { - _PyCallCache *cache = (_PyCallCache *)next_instr; - if (cframe.use_tracing || opcode != CALL) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - next_instr--; - int is_meth = is_method(stack_pointer, oparg); - int nargs = oparg + is_meth; - PyObject *callable = PEEK(nargs + 1); - int err = _Py_Specialize_Call(callable, next_instr, nargs, - call_shape.kwnames); - if (err < 0) { - goto error; + if (!cframe.use_tracing) { + _PyCallCache *cache = (_PyCallCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + next_instr--; + int is_meth = is_method(stack_pointer, oparg); + int nargs = oparg + is_meth; + PyObject *callable = PEEK(nargs + 1); + int err = _Py_Specialize_Call(callable, next_instr, nargs, + call_shape.kwnames); + if (err < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); } - DISPATCH_SAME_OPARG(); - } - else { STAT_INC(CALL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } @@ -3875,26 +3857,24 @@ dummy_func( } inst(BINARY_OP_GENERIC) { - GO_TO_INSTRUCTION(BINARY_OP); + goto binary_op; } // stack effect: (__0 -- ) inst(BINARY_OP) { - PREDICTED(BINARY_OP); - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (cframe.use_tracing || opcode != BINARY_OP) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *lhs = SECOND(); - PyObject *rhs = TOP(); - next_instr--; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); - DISPATCH_SAME_OPARG(); - } - else { + if (!cframe.use_tracing) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *lhs = SECOND(); + PyObject *rhs = TOP(); + next_instr--; + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); + DISPATCH_SAME_OPARG(); + } STAT_INC(BINARY_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } + binary_op: PyObject *rhs = POP(); PyObject *lhs = TOP(); assert(0 <= oparg); @@ -3922,7 +3902,7 @@ dummy_func( inst(EXTENDED_ARG) { assert(oparg); opcode = _Py_OPCODE(*next_instr); - if (cframe.use_tracing) { + if (!cframe.use_tracing) { // Deoptimize the next opcode to avoid breaking tracing // guarantees in quickened instructions: opcode = _PyOpcode_Deopt[opcode]; diff --git a/Python/ceval.c b/Python/ceval.c index ad1561d8dcb391..64b5c15d72529d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1251,14 +1251,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int STAT_INC(opcode, miss); opcode = _PyOpcode_Deopt[opcode]; STAT_INC(opcode, miss); - /* The counter is always the first cache entry: */ - _Py_CODEUNIT *counter = (_Py_CODEUNIT *)next_instr; - *counter -= 1; - if (*counter == 0) { - _Py_SET_OPCODE(next_instr[-1], opcode); - STAT_INC(opcode, deopt); - *counter = adaptive_counter_start(); - } next_instr--; DISPATCH_GOTO(); } diff --git a/Python/specialize.c b/Python/specialize.c index 204067ecd5b275..952dca12c177c5 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -296,19 +296,6 @@ _PyCode_Quicken(PyCodeObject *code) } } -static inline int -miss_counter_start(void) { - /* Starting value for the counter. - * This value needs to be not too low, otherwise - * it would cause excessive de-optimization. - * Neither should it be too high, or that would delay - * de-optimization excessively when it is needed. - * A value around 50 seems to work, and we choose a - * prime number to avoid artifacts. - */ - return 53; -} - #define SIMPLE_FUNCTION 0 /* Common */ @@ -825,12 +812,13 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) fail: STAT_INC(LOAD_ATTR, failure); assert(!PyErr_Occurred()); + _Py_SET_OPCODE(*instr, LOAD_ATTR); cache->counter = adaptive_counter_backoff(cache->counter); return 0; success: STAT_INC(LOAD_ATTR, success); assert(!PyErr_Occurred()); - cache->counter = miss_counter_start(); + cache->counter = adaptive_counter_restart(); return 0; } @@ -908,12 +896,13 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) fail: STAT_INC(STORE_ATTR, failure); assert(!PyErr_Occurred()); + _Py_SET_OPCODE(*instr, STORE_ATTR); cache->counter = adaptive_counter_backoff(cache->counter); return 0; success: STAT_INC(STORE_ATTR, success); assert(!PyErr_Occurred()); - cache->counter = miss_counter_start(); + cache->counter = adaptive_counter_restart(); return 0; } @@ -1166,12 +1155,13 @@ _Py_Specialize_LoadGlobal( fail: STAT_INC(LOAD_GLOBAL, failure); assert(!PyErr_Occurred()); + _Py_SET_OPCODE(*instr, LOAD_GLOBAL); cache->counter = adaptive_counter_backoff(cache->counter); return 0; success: STAT_INC(LOAD_GLOBAL, success); assert(!PyErr_Occurred()); - cache->counter = miss_counter_start(); + cache->counter = adaptive_counter_restart(); return 0; } @@ -1326,12 +1316,13 @@ _Py_Specialize_BinarySubscr( fail: STAT_INC(BINARY_SUBSCR, failure); assert(!PyErr_Occurred()); + _Py_SET_OPCODE(*instr, BINARY_SUBSCR); cache->counter = adaptive_counter_backoff(cache->counter); return 0; success: STAT_INC(BINARY_SUBSCR, success); assert(!PyErr_Occurred()); - cache->counter = miss_counter_start(); + cache->counter = adaptive_counter_restart(); return 0; } @@ -1430,12 +1421,13 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins fail: STAT_INC(STORE_SUBSCR, failure); assert(!PyErr_Occurred()); + _Py_SET_OPCODE(*instr, STORE_SUBSCR); cache->counter = adaptive_counter_backoff(cache->counter); return 0; success: STAT_INC(STORE_SUBSCR, success); assert(!PyErr_Occurred()); - cache->counter = miss_counter_start(); + cache->counter = adaptive_counter_restart(); return 0; } @@ -1743,12 +1735,13 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, if (fail) { STAT_INC(CALL, failure); assert(!PyErr_Occurred()); + _Py_SET_OPCODE(*instr, CALL); cache->counter = adaptive_counter_backoff(cache->counter); } else { STAT_INC(CALL, success); assert(!PyErr_Occurred()); - cache->counter = miss_counter_start(); + cache->counter = adaptive_counter_restart(); } return 0; } @@ -1896,11 +1889,12 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, } SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs)); STAT_INC(BINARY_OP, failure); + _Py_SET_OPCODE(*instr, BINARY_OP); cache->counter = adaptive_counter_backoff(cache->counter); return; success: STAT_INC(BINARY_OP, success); - cache->counter = miss_counter_start(); + cache->counter = adaptive_counter_restart(); } @@ -2016,11 +2010,12 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); failure: STAT_INC(COMPARE_OP, failure); + _Py_SET_OPCODE(*instr, COMPARE_OP); cache->counter = adaptive_counter_backoff(cache->counter); return; success: STAT_INC(COMPARE_OP, success); - cache->counter = miss_counter_start(); + cache->counter = adaptive_counter_restart(); } #ifdef Py_STATS @@ -2066,11 +2061,12 @@ _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) SPECIALIZATION_FAIL(UNPACK_SEQUENCE, unpack_sequence_fail_kind(seq)); failure: STAT_INC(UNPACK_SEQUENCE, failure); + _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE); cache->counter = adaptive_counter_backoff(cache->counter); return; success: STAT_INC(UNPACK_SEQUENCE, success); - cache->counter = miss_counter_start(); + cache->counter = adaptive_counter_restart(); } #ifdef Py_STATS @@ -2164,9 +2160,10 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr) SPECIALIZATION_FAIL(FOR_ITER, _PySpecialization_ClassifyIterator(iter)); STAT_INC(FOR_ITER, failure); + _Py_SET_OPCODE(*instr, FOR_ITER); cache->counter = adaptive_counter_backoff(cache->counter); return; success: STAT_INC(FOR_ITER, success); - cache->counter = miss_counter_start(); + cache->counter = adaptive_counter_restart(); } From e36c5d718faf04cb7e40c36c84d919c50d76fa66 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 20 Oct 2022 12:03:51 -0700 Subject: [PATCH 05/21] make regen-cases --- Python/bytecodes.c | 3 +- Python/generated_cases.c.h | 219 +++++++++++++++++-------------------- 2 files changed, 102 insertions(+), 120 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2dd7646342e885..e31c0ad67f3b3a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3875,6 +3875,7 @@ dummy_func( DECREMENT_ADAPTIVE_COUNTER(cache); } binary_op: + ; // Why is this still a thing in 2022? PyObject *rhs = POP(); PyObject *lhs = TOP(); assert(0 <= oparg); @@ -3902,7 +3903,7 @@ dummy_func( inst(EXTENDED_ARG) { assert(oparg); opcode = _Py_OPCODE(*next_instr); - if (!cframe.use_tracing) { + if (cframe.use_tracing) { // Deoptimize the next opcode to avoid breaking tracing // guarantees in quickened instructions: opcode = _PyOpcode_Deopt[opcode]; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6d050193977b2e..d7058bad7ce7b7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -402,19 +402,17 @@ } TARGET(BINARY_SUBSCR) { - _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (cframe.use_tracing || opcode != BINARY_SUBSCR) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *sub = TOP(); - PyObject *container = SECOND(); - next_instr--; - if (_Py_Specialize_BinarySubscr(container, sub, next_instr) < 0) { - goto error; + if (!cframe.use_tracing) { + _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *sub = TOP(); + PyObject *container = SECOND(); + next_instr--; + if (_Py_Specialize_BinarySubscr(container, sub, next_instr) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); } - DISPATCH_SAME_OPARG(); - } - else { STAT_INC(BINARY_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } @@ -557,19 +555,17 @@ } TARGET(STORE_SUBSCR) { - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (cframe.use_tracing || opcode != STORE_SUBSCR) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *sub = TOP(); - PyObject *container = SECOND(); - next_instr--; - if (_Py_Specialize_StoreSubscr(container, sub, next_instr) < 0) { - goto error; + if (!cframe.use_tracing) { + _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *sub = TOP(); + PyObject *container = SECOND(); + next_instr--; + if (_Py_Specialize_StoreSubscr(container, sub, next_instr) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); } - DISPATCH_SAME_OPARG(); - } - else { STAT_INC(STORE_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } @@ -1120,16 +1116,14 @@ } TARGET(UNPACK_SEQUENCE) { - _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (cframe.use_tracing || opcode != UNPACK_SEQUENCE) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *seq = TOP(); - next_instr--; - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - else { + if (!cframe.use_tracing) { + _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *seq = TOP(); + next_instr--; + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } STAT_INC(UNPACK_SEQUENCE, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } @@ -1301,18 +1295,16 @@ } TARGET(LOAD_GLOBAL) { - _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (cframe.use_tracing || opcode != LOAD_GLOBAL) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *name = GETITEM(names, oparg>>1); - next_instr--; - if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) { - goto error; + if (!cframe.use_tracing) { + _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *name = GETITEM(names, oparg>>1); + next_instr--; + if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); } - DISPATCH_SAME_OPARG(); - } - else { STAT_INC(LOAD_GLOBAL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } @@ -1758,19 +1750,17 @@ } TARGET(LOAD_ATTR) { - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (cframe.use_tracing || opcode != LOAD_ATTR) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *owner = TOP(); - PyObject *name = GETITEM(names, oparg>>1); - next_instr--; - if (_Py_Specialize_LoadAttr(owner, next_instr, name) < 0) { - goto error; + if (!cframe.use_tracing) { + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *owner = TOP(); + PyObject *name = GETITEM(names, oparg>>1); + next_instr--; + if (_Py_Specialize_LoadAttr(owner, next_instr, name) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); } - DISPATCH_SAME_OPARG(); - } - else { STAT_INC(LOAD_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } @@ -2035,19 +2025,17 @@ } TARGET(STORE_ATTR) { - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (cframe.use_tracing || opcode != STORE_ATTR) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *owner = TOP(); - PyObject *name = GETITEM(names, oparg); - next_instr--; - if (_Py_Specialize_StoreAttr(owner, next_instr, name) < 0) { - goto error; + if (!cframe.use_tracing) { + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *owner = TOP(); + PyObject *name = GETITEM(names, oparg); + next_instr--; + if (_Py_Specialize_StoreAttr(owner, next_instr, name) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); } - DISPATCH_SAME_OPARG(); - } - else { STAT_INC(STORE_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } @@ -2168,25 +2156,23 @@ } TARGET(COMPARE_OP_GENERIC) { - GO_TO_INSTRUCTION(COMPARE_OP); + goto compare_op; } TARGET(COMPARE_OP) { - PREDICTED(COMPARE_OP); - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (cframe.use_tracing || opcode != COMPARE_OP) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *right = TOP(); - PyObject *left = SECOND(); - next_instr--; - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - else { + if (!cframe.use_tracing) { + _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *right = TOP(); + PyObject *left = SECOND(); + next_instr--; + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } STAT_INC(COMPARE_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } + compare_op: assert(oparg <= Py_GE); PyObject *right = POP(); PyObject *left = TOP(); @@ -2680,15 +2666,13 @@ } TARGET(FOR_ITER) { - _PyForIterCache *cache = (_PyForIterCache *)next_instr; - if (cframe.use_tracing || opcode != FOR_ITER) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - next_instr--; - _Py_Specialize_ForIter(TOP(), next_instr); - DISPATCH_SAME_OPARG(); - } - else { + if (!cframe.use_tracing) { + _PyForIterCache *cache = (_PyForIterCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + next_instr--; + _Py_Specialize_ForIter(TOP(), next_instr); + DISPATCH_SAME_OPARG(); + } STAT_INC(FOR_ITER, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } @@ -3003,22 +2987,20 @@ } TARGET(CALL) { - _PyCallCache *cache = (_PyCallCache *)next_instr; - if (cframe.use_tracing || opcode != CALL) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - next_instr--; - int is_meth = is_method(stack_pointer, oparg); - int nargs = oparg + is_meth; - PyObject *callable = PEEK(nargs + 1); - int err = _Py_Specialize_Call(callable, next_instr, nargs, - call_shape.kwnames); - if (err < 0) { - goto error; + if (!cframe.use_tracing) { + _PyCallCache *cache = (_PyCallCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + next_instr--; + int is_meth = is_method(stack_pointer, oparg); + int nargs = oparg + is_meth; + PyObject *callable = PEEK(nargs + 1); + int err = _Py_Specialize_Call(callable, next_instr, nargs, + call_shape.kwnames); + if (err < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); } - DISPATCH_SAME_OPARG(); - } - else { STAT_INC(CALL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } @@ -3791,25 +3773,24 @@ } TARGET(BINARY_OP_GENERIC) { - GO_TO_INSTRUCTION(BINARY_OP); + goto binary_op; } TARGET(BINARY_OP) { - PREDICTED(BINARY_OP); - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (cframe.use_tracing || opcode != BINARY_OP) { - } - else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *lhs = SECOND(); - PyObject *rhs = TOP(); - next_instr--; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); - DISPATCH_SAME_OPARG(); - } - else { + if (!cframe.use_tracing) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *lhs = SECOND(); + PyObject *rhs = TOP(); + next_instr--; + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); + DISPATCH_SAME_OPARG(); + } STAT_INC(BINARY_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } + binary_op: + ; // Why is this still a thing in 2022? PyObject *rhs = POP(); PyObject *lhs = TOP(); assert(0 <= oparg); From 3caa5d48eb620232c280f34aa12c87632ec8ade7 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 20 Oct 2022 15:34:36 -0700 Subject: [PATCH 06/21] Try inlining the miss label everywhere --- Python/bytecodes.c | 4 ++-- Python/ceval.c | 21 +++++++++------------ Python/generated_cases.c.h | 4 ++-- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e31c0ad67f3b3a..6c0ae96f091584 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2079,7 +2079,7 @@ dummy_func( DEOPT_IF(f->func_version != func_version, LOAD_ATTR); PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(names, oparg >> 1); @@ -2173,7 +2173,7 @@ dummy_func( DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(dict == NULL, STORE_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); diff --git a/Python/ceval.c b/Python/ceval.c index 64b5c15d72529d..24e0e463b637e1 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -852,7 +852,15 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define GO_TO_INSTRUCTION(op) goto PREDICT_ID(op) -#define DEOPT_IF(cond, instname) if (cond) { goto miss; } +#define DEOPT_IF(cond, instname) \ + if (cond) { \ + STAT_INC(opcode, miss); \ + STAT_INC(instname, miss); \ + next_instr--; \ + assert(instname == _PyOpcode_Deopt[opcode]); \ + opcode = instname; \ + DISPATCH_GOTO(); \ + } #define GLOBALS() frame->f_globals @@ -1244,17 +1252,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int or goto error. */ Py_UNREACHABLE(); -/* Specialization misses */ - -miss: - { - STAT_INC(opcode, miss); - opcode = _PyOpcode_Deopt[opcode]; - STAT_INC(opcode, miss); - next_instr--; - DISPATCH_GOTO(); - } - unbound_local_error: { format_exc_check_arg(tstate, PyExc_UnboundLocalError, diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d7058bad7ce7b7..a378d8a252c355 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2000,7 +2000,7 @@ DEOPT_IF(f->func_version != func_version, LOAD_ATTR); PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(names, oparg >> 1); @@ -2093,7 +2093,7 @@ DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(dict == NULL, STORE_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); From ea175fc6b74706dc227866903966e94f7e3cf59d Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 4 Nov 2022 19:43:20 -0700 Subject: [PATCH 07/21] Use a single direct jump for misses --- Python/bytecodes.c | 10 ++++++++++ Python/ceval.c | 12 +++++------- Python/generated_cases.c.h | 10 ++++++++++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6c0ae96f091584..ea3f0c2ba7cdeb 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -471,6 +471,7 @@ dummy_func( // stack effect: (__0 -- ) inst(BINARY_SUBSCR) { + PREDICTED(BINARY_SUBSCR); if (!cframe.use_tracing) { _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { @@ -625,6 +626,7 @@ dummy_func( // stack effect: (__0, __1, __2 -- ) inst(STORE_SUBSCR) { + PREDICTED(STORE_SUBSCR); if (!cframe.use_tracing) { _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { @@ -1192,6 +1194,7 @@ dummy_func( // stack effect: (__0 -- __array[oparg]) inst(UNPACK_SEQUENCE) { + PREDICTED(UNPACK_SEQUENCE); if (!cframe.use_tracing) { _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { @@ -1371,6 +1374,7 @@ dummy_func( // error: LOAD_GLOBAL has irregular stack effect inst(LOAD_GLOBAL) { + PREDICTED(LOAD_GLOBAL); if (!cframe.use_tracing) { _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { @@ -1828,6 +1832,7 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR) { + PREDICTED(LOAD_ATTR); if (!cframe.use_tracing) { _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { @@ -2105,6 +2110,7 @@ dummy_func( // stack effect: (__0, __1 -- ) inst(STORE_ATTR) { + PREDICTED(STORE_ATTR); if (!cframe.use_tracing) { _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { @@ -2240,6 +2246,7 @@ dummy_func( // stack effect: (__0 -- ) inst(COMPARE_OP) { + PREDICTED(COMPARE_OP); if (!cframe.use_tracing) { _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { @@ -2746,6 +2753,7 @@ dummy_func( // stack effect: ( -- __0) inst(FOR_ITER) { + PREDICTED(FOR_ITER); if (!cframe.use_tracing) { _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { @@ -3069,6 +3077,7 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL) { + PREDICTED(CALL); if (!cframe.use_tracing) { _PyCallCache *cache = (_PyCallCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { @@ -3862,6 +3871,7 @@ dummy_func( // stack effect: (__0 -- ) inst(BINARY_OP) { + PREDICTED(BINARY_OP); if (!cframe.use_tracing) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { diff --git a/Python/ceval.c b/Python/ceval.c index 24e0e463b637e1..15e7fcc27306d7 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -852,14 +852,12 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define GO_TO_INSTRUCTION(op) goto PREDICT_ID(op) -#define DEOPT_IF(cond, instname) \ - if (cond) { \ +#define DEOPT_IF(COND, INSTNAME) \ + if (COND) { \ STAT_INC(opcode, miss); \ - STAT_INC(instname, miss); \ - next_instr--; \ - assert(instname == _PyOpcode_Deopt[opcode]); \ - opcode = instname; \ - DISPATCH_GOTO(); \ + STAT_INC(INSTNAME, miss); \ + assert(_PyOpcode_Deopt[opcode] == INSTNAME); \ + GO_TO_INSTRUCTION(INSTNAME); \ } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a378d8a252c355..e0520787eecba3 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -402,6 +402,7 @@ } TARGET(BINARY_SUBSCR) { + PREDICTED(BINARY_SUBSCR); if (!cframe.use_tracing) { _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { @@ -555,6 +556,7 @@ } TARGET(STORE_SUBSCR) { + PREDICTED(STORE_SUBSCR); if (!cframe.use_tracing) { _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { @@ -1116,6 +1118,7 @@ } TARGET(UNPACK_SEQUENCE) { + PREDICTED(UNPACK_SEQUENCE); if (!cframe.use_tracing) { _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { @@ -1295,6 +1298,7 @@ } TARGET(LOAD_GLOBAL) { + PREDICTED(LOAD_GLOBAL); if (!cframe.use_tracing) { _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { @@ -1750,6 +1754,7 @@ } TARGET(LOAD_ATTR) { + PREDICTED(LOAD_ATTR); if (!cframe.use_tracing) { _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { @@ -2025,6 +2030,7 @@ } TARGET(STORE_ATTR) { + PREDICTED(STORE_ATTR); if (!cframe.use_tracing) { _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { @@ -2160,6 +2166,7 @@ } TARGET(COMPARE_OP) { + PREDICTED(COMPARE_OP); if (!cframe.use_tracing) { _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { @@ -2666,6 +2673,7 @@ } TARGET(FOR_ITER) { + PREDICTED(FOR_ITER); if (!cframe.use_tracing) { _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { @@ -2987,6 +2995,7 @@ } TARGET(CALL) { + PREDICTED(CALL); if (!cframe.use_tracing) { _PyCallCache *cache = (_PyCallCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { @@ -3777,6 +3786,7 @@ } TARGET(BINARY_OP) { + PREDICTED(BINARY_OP); if (!cframe.use_tracing) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { From 553ebabc98ff7c7bb88392d6af7541f76d40e331 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sat, 5 Nov 2022 06:24:59 -0700 Subject: [PATCH 08/21] Revert some unrelated changes --- Include/internal/pycore_code.h | 4 +- Include/internal/pycore_opcode.h | 26 ++++---- Include/opcode.h | 57 ++++++++--------- Lib/opcode.py | 3 + Lib/test/test_embed.py | 2 + Python/bytecodes.c | 77 ++++++++++++----------- Python/compile.c | 3 + Python/generated_cases.c.h | 81 +++++++++++++------------ Python/opcode_targets.h | 24 ++++---- Python/specialize.c | 57 +++++++++++------ Tools/cases_generator/generate_cases.py | 1 - 11 files changed, 185 insertions(+), 150 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 2779d108fab8eb..66f42f62acd15b 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -377,10 +377,10 @@ write_location_entry_start(uint8_t *ptr, int code, int length) #define ADAPTIVE_BACKOFF_BITS 4 #define ADAPTIVE_BACKOFF_START_VALUE 1 -#define ADAPTIVE_BACKOFF_START_BACKOFF 1 // 2**0 <= 1 < 2**1 +#define ADAPTIVE_BACKOFF_START_BACKOFF 1 #define ADAPTIVE_BACKOFF_RESTART_VALUE 52 -#define ADAPTIVE_BACKOFF_RESTART_BACKOFF 6 // 2**5 <= 53 < 2**6 +#define ADAPTIVE_BACKOFF_RESTART_BACKOFF 0 #define MAX_BACKOFF_VALUE (16 - ADAPTIVE_BACKOFF_BITS) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index defd70ca251c3d..cbeaeddc9da6a2 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -122,6 +122,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [END_ASYNC_FOR] = END_ASYNC_FOR, [END_FOR] = END_FOR, [EXTENDED_ARG] = EXTENDED_ARG, + [EXTENDED_ARG_QUICK] = EXTENDED_ARG, [FORMAT_VALUE] = FORMAT_VALUE, [FOR_ITER] = FOR_ITER, [FOR_ITER_LIST] = FOR_ITER, @@ -293,31 +294,31 @@ static const char *const _PyOpcode_OpName[263] = { [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", - [FOR_ITER_LIST] = "FOR_ITER_LIST", + [EXTENDED_ARG_QUICK] = "EXTENDED_ARG_QUICK", [STOPITERATION_ERROR] = "STOPITERATION_ERROR", + [FOR_ITER_LIST] = "FOR_ITER_LIST", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [PRINT_EXPR] = "PRINT_EXPR", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [LIST_TO_TUPLE] = "LIST_TO_TUPLE", [RETURN_VALUE] = "RETURN_VALUE", [IMPORT_STAR] = "IMPORT_STAR", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", @@ -344,7 +345,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -352,7 +353,7 @@ static const char *const _PyOpcode_OpName[263] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -372,9 +373,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -384,24 +385,24 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", - [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [170] = "<170>", [CALL] = "CALL", [KW_NAMES] = "KW_NAMES", [173] = "<173>", @@ -498,7 +499,6 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ - case 170: \ case 173: \ case 174: \ case 175: \ diff --git a/Include/opcode.h b/Include/opcode.h index 29ee69d465a7e8..745fad7feaae6c 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -162,34 +162,35 @@ extern "C" { #define COMPARE_OP_GENERIC 57 #define COMPARE_OP_INT_JUMP 58 #define COMPARE_OP_STR_JUMP 59 -#define FOR_ITER_LIST 62 -#define FOR_ITER_RANGE 64 -#define LOAD_ATTR_CLASS 65 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 66 -#define LOAD_ATTR_INSTANCE_VALUE 67 -#define LOAD_ATTR_MODULE 72 -#define LOAD_ATTR_PROPERTY 73 -#define LOAD_ATTR_SLOT 76 -#define LOAD_ATTR_WITH_HINT 77 -#define LOAD_ATTR_METHOD_LAZY_DICT 78 -#define LOAD_ATTR_METHOD_NO_DICT 79 -#define LOAD_ATTR_METHOD_WITH_DICT 80 -#define LOAD_ATTR_METHOD_WITH_VALUES 81 -#define LOAD_CONST__LOAD_FAST 86 -#define LOAD_FAST__LOAD_CONST 113 -#define LOAD_FAST__LOAD_FAST 121 -#define LOAD_GLOBAL_BUILTIN 141 -#define LOAD_GLOBAL_MODULE 143 -#define STORE_ATTR_INSTANCE_VALUE 153 -#define STORE_ATTR_SLOT 154 -#define STORE_ATTR_WITH_HINT 158 -#define STORE_FAST__LOAD_FAST 159 -#define STORE_FAST__STORE_FAST 160 -#define STORE_SUBSCR_DICT 161 -#define STORE_SUBSCR_LIST_INT 166 -#define UNPACK_SEQUENCE_LIST 167 -#define UNPACK_SEQUENCE_TUPLE 168 -#define UNPACK_SEQUENCE_TWO_TUPLE 169 +#define EXTENDED_ARG_QUICK 62 +#define FOR_ITER_LIST 64 +#define FOR_ITER_RANGE 65 +#define LOAD_ATTR_CLASS 66 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 67 +#define LOAD_ATTR_INSTANCE_VALUE 72 +#define LOAD_ATTR_MODULE 73 +#define LOAD_ATTR_PROPERTY 76 +#define LOAD_ATTR_SLOT 77 +#define LOAD_ATTR_WITH_HINT 78 +#define LOAD_ATTR_METHOD_LAZY_DICT 79 +#define LOAD_ATTR_METHOD_NO_DICT 80 +#define LOAD_ATTR_METHOD_WITH_DICT 81 +#define LOAD_ATTR_METHOD_WITH_VALUES 86 +#define LOAD_CONST__LOAD_FAST 113 +#define LOAD_FAST__LOAD_CONST 121 +#define LOAD_FAST__LOAD_FAST 141 +#define LOAD_GLOBAL_BUILTIN 143 +#define LOAD_GLOBAL_MODULE 153 +#define STORE_ATTR_INSTANCE_VALUE 154 +#define STORE_ATTR_SLOT 158 +#define STORE_ATTR_WITH_HINT 159 +#define STORE_FAST__LOAD_FAST 160 +#define STORE_FAST__STORE_FAST 161 +#define STORE_SUBSCR_DICT 166 +#define STORE_SUBSCR_LIST_INT 167 +#define UNPACK_SEQUENCE_LIST 168 +#define UNPACK_SEQUENCE_TUPLE 169 +#define UNPACK_SEQUENCE_TWO_TUPLE 170 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/opcode.py b/Lib/opcode.py index 0de42fe9989f43..7c5628e0ebe952 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -319,6 +319,9 @@ def pseudo_op(name, op, real_ops): "COMPARE_OP_INT_JUMP", "COMPARE_OP_STR_JUMP", ], + "EXTENDED_ARG": [ + "EXTENDED_ARG_QUICK", + ], "FOR_ITER": [ "FOR_ITER_LIST", "FOR_ITER_RANGE", diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 2dda7ccf7bf80c..d56bea22d15093 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -362,6 +362,8 @@ def is_specialized(f): opname in opcode._specialized_instructions # Exclude superinstructions: and "__" not in opname + # Exclude "quick" instructions: + and not opname.endswith("_QUICK") ): return True return False diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9ac47a9cf60ace..b7bb2fb14180b5 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2268,7 +2268,17 @@ dummy_func( } inst(COMPARE_OP_GENERIC) { - goto compare_op; + assert(oparg <= Py_GE); + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyObject_RichCompare(left, right, oparg); + SET_TOP(res); + Py_DECREF(left); + Py_DECREF(right); + if (res == NULL) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); } // stack effect: (__0 -- ) @@ -2285,18 +2295,7 @@ dummy_func( STAT_INC(COMPARE_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } - compare_op: - assert(oparg <= Py_GE); - PyObject *right = POP(); - PyObject *left = TOP(); - PyObject *res = PyObject_RichCompare(left, right, oparg); - SET_TOP(res); - Py_DECREF(left); - Py_DECREF(right); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); + GO_TO_INSTRUCTION(COMPARE_OP_GENERIC); } // stack effect: (__0 -- ) @@ -3886,7 +3885,19 @@ dummy_func( } inst(BINARY_OP_GENERIC) { - goto binary_op; + PyObject *rhs = POP(); + PyObject *lhs = TOP(); + assert(0 <= oparg); + assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); + assert(binary_ops[oparg]); + PyObject *res = binary_ops[oparg](lhs, rhs); + Py_DECREF(lhs); + Py_DECREF(rhs); + SET_TOP(res); + if (res == NULL) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); } // stack effect: (__0 -- ) @@ -3903,21 +3914,7 @@ dummy_func( STAT_INC(BINARY_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } - binary_op: - ; // Why is this still a thing in 2022? - PyObject *rhs = POP(); - PyObject *lhs = TOP(); - assert(0 <= oparg); - assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); - assert(binary_ops[oparg]); - PyObject *res = binary_ops[oparg](lhs, rhs); - Py_DECREF(lhs); - Py_DECREF(rhs); - SET_TOP(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); + GO_TO_INSTRUCTION(BINARY_OP_GENERIC); } // stack effect: ( -- ) @@ -3931,15 +3928,12 @@ dummy_func( // stack effect: ( -- ) inst(EXTENDED_ARG) { assert(oparg); - opcode = _Py_OPCODE(*next_instr); - if (cframe.use_tracing) { - // Deoptimize the next opcode to avoid breaking tracing - // guarantees in quickened instructions: - opcode = _PyOpcode_Deopt[opcode]; - PRE_DISPATCH_GOTO(); - } oparg <<= 8; oparg |= _Py_OPARG(*next_instr); + // We might be tracing. To avoid breaking tracing guarantees in + // quickened instructions, always deoptimize the next opcode: + opcode = _PyOpcode_Deopt[_Py_OPCODE(*next_instr)]; + PRE_DISPATCH_GOTO(); // CPython hasn't traced the following instruction historically // (DO_TRACING would clobber our extended oparg anyways), so just // skip our usual cframe.use_tracing check before dispatch. Also, @@ -3949,6 +3943,16 @@ dummy_func( DISPATCH_GOTO(); } + // stack effect: ( -- ) + inst(EXTENDED_ARG_QUICK) { + assert(cframe.use_tracing == 0); + assert(oparg); + int oldoparg = oparg; + NEXTOPARG(); + oparg |= oldoparg << 8; + DISPATCH_GOTO(); + } + // stack effect: ( -- ) inst(CACHE) { Py_UNREACHABLE(); @@ -3989,6 +3993,7 @@ family(call) = { family(compare_op) = { COMPARE_OP, COMPARE_OP_FLOAT_JUMP, COMPARE_OP_GENERIC, COMPARE_OP_INT_JUMP, COMPARE_OP_STR_JUMP }; +family(extended_arg) = { EXTENDED_ARG, EXTENDED_ARG_QUICK }; family(for_iter) = { FOR_ITER, FOR_ITER_LIST, FOR_ITER_RANGE }; diff --git a/Python/compile.c b/Python/compile.c index 030378f19a3362..065d1b08d0642f 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -8062,6 +8062,7 @@ scan_block_for_locals(basicblock *b, basicblock ***sp) for (int i = 0; i < b->b_iused; i++) { struct instr *instr = &b->b_instr[i]; assert(instr->i_opcode != EXTENDED_ARG); + assert(instr->i_opcode != EXTENDED_ARG_QUICK); assert(!IS_SUPERINSTRUCTION_OPCODE(instr->i_opcode)); if (instr->i_except != NULL) { maybe_push(instr->i_except, unsafe_mask, sp); @@ -8118,6 +8119,7 @@ fast_scan_many_locals(basicblock *entryblock, int nlocals) for (int i = 0; i < b->b_iused; i++) { struct instr *instr = &b->b_instr[i]; assert(instr->i_opcode != EXTENDED_ARG); + assert(instr->i_opcode != EXTENDED_ARG_QUICK); assert(!IS_SUPERINSTRUCTION_OPCODE(instr->i_opcode)); int arg = instr->i_oparg; if (arg < 64) { @@ -8665,6 +8667,7 @@ fix_cell_offsets(struct compiler *c, basicblock *entryblock, int *fixedmap) struct instr *inst = &b->b_instr[i]; // This is called before extended args are generated. assert(inst->i_opcode != EXTENDED_ARG); + assert(inst->i_opcode != EXTENDED_ARG_QUICK); int oldoffset = inst->i_oparg; switch(inst->i_opcode) { case MAKE_CELL: diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e8e35f553553ac..055accd9d60aed 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2162,7 +2162,19 @@ } TARGET(COMPARE_OP_GENERIC) { - goto compare_op; + PREDICTED(COMPARE_OP_GENERIC); + assert(oparg <= Py_GE); + PyObject *right = POP(); + PyObject *left = TOP(); + PyObject *res = PyObject_RichCompare(left, right, oparg); + SET_TOP(res); + Py_DECREF(left); + Py_DECREF(right); + if (res == NULL) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); + DISPATCH(); } TARGET(COMPARE_OP) { @@ -2179,19 +2191,7 @@ STAT_INC(COMPARE_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } - compare_op: - assert(oparg <= Py_GE); - PyObject *right = POP(); - PyObject *left = TOP(); - PyObject *res = PyObject_RichCompare(left, right, oparg); - SET_TOP(res); - Py_DECREF(left); - Py_DECREF(right); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); - DISPATCH(); + GO_TO_INSTRUCTION(COMPARE_OP_GENERIC); } TARGET(COMPARE_OP_FLOAT_JUMP) { @@ -3782,7 +3782,21 @@ } TARGET(BINARY_OP_GENERIC) { - goto binary_op; + PREDICTED(BINARY_OP_GENERIC); + PyObject *rhs = POP(); + PyObject *lhs = TOP(); + assert(0 <= oparg); + assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); + assert(binary_ops[oparg]); + PyObject *res = binary_ops[oparg](lhs, rhs); + Py_DECREF(lhs); + Py_DECREF(rhs); + SET_TOP(res); + if (res == NULL) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); + DISPATCH(); } TARGET(BINARY_OP) { @@ -3799,22 +3813,7 @@ STAT_INC(BINARY_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache); } - binary_op: - ; // Why is this still a thing in 2022? - PyObject *rhs = POP(); - PyObject *lhs = TOP(); - assert(0 <= oparg); - assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); - assert(binary_ops[oparg]); - PyObject *res = binary_ops[oparg](lhs, rhs); - Py_DECREF(lhs); - Py_DECREF(rhs); - SET_TOP(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); - DISPATCH(); + GO_TO_INSTRUCTION(BINARY_OP_GENERIC); } TARGET(SWAP) { @@ -3827,15 +3826,12 @@ TARGET(EXTENDED_ARG) { assert(oparg); - opcode = _Py_OPCODE(*next_instr); - if (cframe.use_tracing) { - // Deoptimize the next opcode to avoid breaking tracing - // guarantees in quickened instructions: - opcode = _PyOpcode_Deopt[opcode]; - PRE_DISPATCH_GOTO(); - } oparg <<= 8; oparg |= _Py_OPARG(*next_instr); + // We might be tracing. To avoid breaking tracing guarantees in + // quickened instructions, always deoptimize the next opcode: + opcode = _PyOpcode_Deopt[_Py_OPCODE(*next_instr)]; + PRE_DISPATCH_GOTO(); // CPython hasn't traced the following instruction historically // (DO_TRACING would clobber our extended oparg anyways), so just // skip our usual cframe.use_tracing check before dispatch. Also, @@ -3845,6 +3841,15 @@ DISPATCH_GOTO(); } + TARGET(EXTENDED_ARG_QUICK) { + assert(cframe.use_tracing == 0); + assert(oparg); + int oldoparg = oparg; + NEXTOPARG(); + oparg |= oldoparg << 8; + DISPATCH_GOTO(); + } + TARGET(CACHE) { Py_UNREACHABLE(); } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index e99a2c09a1e00c..42ae6042efb519 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -61,31 +61,31 @@ static void *opcode_targets[256] = { &&TARGET_COMPARE_OP_STR_JUMP, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, - &&TARGET_FOR_ITER_LIST, + &&TARGET_EXTENDED_ARG_QUICK, &&TARGET_STOPITERATION_ERROR, + &&TARGET_FOR_ITER_LIST, &&TARGET_FOR_ITER_RANGE, &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, - &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_ATTR_MODULE, - &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,24 +152,24 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, + &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_STORE_ATTR_INSTANCE_VALUE, - &&TARGET_STORE_ATTR_SLOT, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_STORE_FAST__STORE_FAST, - &&TARGET_STORE_SUBSCR_DICT, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, + &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, - &&_unknown_opcode, &&TARGET_CALL, &&TARGET_KW_NAMES, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index 952dca12c177c5..1228828bc855af 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -264,35 +264,52 @@ do { \ void _PyCode_Quicken(PyCodeObject *code) { - int previous_opcode = 0; + int previous_opcode = -1; _Py_CODEUNIT *instructions = _PyCode_CODE(code); for (int i = 0; i < Py_SIZE(code); i++) { int opcode = _PyOpcode_Deopt[_Py_OPCODE(instructions[i])]; int caches = _PyOpcode_Caches[opcode]; if (caches) { instructions[i + 1] = adaptive_counter_start(); - previous_opcode = 0; + previous_opcode = -1; i += caches; - continue; } - switch (previous_opcode << 8 | opcode) { - case LOAD_FAST << 8 | LOAD_FAST: - _Py_SET_OPCODE(instructions[i - 1], LOAD_FAST__LOAD_FAST); - break; - case STORE_FAST << 8 | LOAD_FAST: - _Py_SET_OPCODE(instructions[i - 1], STORE_FAST__LOAD_FAST); - break; - case LOAD_CONST << 8 | LOAD_FAST: - _Py_SET_OPCODE(instructions[i - 1], LOAD_CONST__LOAD_FAST); - break; - case STORE_FAST << 8 | STORE_FAST: - _Py_SET_OPCODE(instructions[i - 1], STORE_FAST__STORE_FAST); - break; - case LOAD_FAST << 8 | LOAD_CONST: - _Py_SET_OPCODE(instructions[i - 1], LOAD_FAST__LOAD_CONST); - break; + else { + switch (opcode) { + case EXTENDED_ARG: + _Py_SET_OPCODE(instructions[i], EXTENDED_ARG_QUICK); + break; + case LOAD_FAST: + switch(previous_opcode) { + case LOAD_FAST: + _Py_SET_OPCODE(instructions[i - 1], + LOAD_FAST__LOAD_FAST); + break; + case STORE_FAST: + _Py_SET_OPCODE(instructions[i - 1], + STORE_FAST__LOAD_FAST); + break; + case LOAD_CONST: + _Py_SET_OPCODE(instructions[i - 1], + LOAD_CONST__LOAD_FAST); + break; + } + break; + case STORE_FAST: + if (previous_opcode == STORE_FAST) { + _Py_SET_OPCODE(instructions[i - 1], + STORE_FAST__STORE_FAST); + } + break; + case LOAD_CONST: + if (previous_opcode == LOAD_FAST) { + _Py_SET_OPCODE(instructions[i - 1], + LOAD_FAST__LOAD_CONST); + } + break; + } + previous_opcode = opcode; } - previous_opcode = opcode; } } diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 46811c6fa66094..902ebc71de2fdf 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -22,7 +22,6 @@ arg_parser.add_argument("-q", "--quiet", action="store_true") - def eopen(filename: str, mode: str = "r"): if filename == "-": if "r" in mode: From dc545bda87a9f37c5b7c61f66590b71cf10a0423 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sat, 5 Nov 2022 07:03:36 -0700 Subject: [PATCH 09/21] Clean up the diff --- Python/bytecodes.c | 116 ++++++++++++++++++------------------ Python/generated_cases.c.h | 118 ++++++++++++++++++------------------- Python/specialize.c | 22 +++---- 3 files changed, 129 insertions(+), 127 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b7bb2fb14180b5..381cfac302d633 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -463,6 +463,33 @@ dummy_func( JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP); } + // stack effect: (__0 -- ) + inst(BINARY_SUBSCR) { + if (!cframe.use_tracing) { + _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *sub = TOP(); + PyObject *container = SECOND(); + next_instr--; + if (_Py_Specialize_BinarySubscr(container, sub, next_instr) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + STAT_INC(BINARY_SUBSCR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + } + PyObject *sub = POP(); + PyObject *container = TOP(); + PyObject *res = PyObject_GetItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + SET_TOP(res); + if (res == NULL) + goto error; + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + } + // stack effect: (__0, __1 -- ) inst(BINARY_SLICE) { PyObject *stop = POP(); @@ -503,33 +530,6 @@ dummy_func( Py_DECREF(container); } - // stack effect: (__0 -- ) - inst(BINARY_SUBSCR) { - if (!cframe.use_tracing) { - _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *sub = TOP(); - PyObject *container = SECOND(); - next_instr--; - if (_Py_Specialize_BinarySubscr(container, sub, next_instr) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); - } - PyObject *sub = POP(); - PyObject *container = TOP(); - PyObject *res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - SET_TOP(res); - if (res == NULL) - goto error; - JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - } - // stack effect: (__0 -- ) inst(BINARY_SUBSCR_LIST_INT) { assert(cframe.use_tracing == 0); @@ -1302,6 +1302,36 @@ dummy_func( Py_DECREF(seq); } + // stack effect: (__0, __1 -- ) + inst(STORE_ATTR) { + if (!cframe.use_tracing) { + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *owner = TOP(); + PyObject *name = GETITEM(names, oparg); + next_instr--; + if (_Py_Specialize_StoreAttr(owner, next_instr, name) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + STAT_INC(STORE_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + } + PyObject *name = GETITEM(names, oparg); + PyObject *owner = TOP(); + PyObject *v = SECOND(); + int err; + STACK_SHRINK(2); + err = PyObject_SetAttr(owner, name, v); + Py_DECREF(v); + Py_DECREF(owner); + if (err != 0) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); + } + // stack effect: (__0 -- ) inst(DELETE_ATTR) { PyObject *name = GETITEM(names, oparg); @@ -2136,36 +2166,6 @@ dummy_func( goto start_frame; } - // stack effect: (__0, __1 -- ) - inst(STORE_ATTR) { - if (!cframe.use_tracing) { - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *owner = TOP(); - PyObject *name = GETITEM(names, oparg); - next_instr--; - if (_Py_Specialize_StoreAttr(owner, next_instr, name) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); - } - PyObject *name = GETITEM(names, oparg); - PyObject *owner = TOP(); - PyObject *v = SECOND(); - int err; - STACK_SHRINK(2); - err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); - if (err != 0) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); - } - // stack effect: (__0, __1 -- ) inst(STORE_ATTR_INSTANCE_VALUE) { assert(cframe.use_tracing == 0); @@ -2267,6 +2267,7 @@ dummy_func( JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); } + // stack effect: (__0 -- ) inst(COMPARE_OP_GENERIC) { assert(oparg <= Py_GE); PyObject *right = POP(); @@ -3884,6 +3885,7 @@ dummy_func( PUSH(peek); } + // stack effect: (__0 -- ) inst(BINARY_OP_GENERIC) { PyObject *rhs = POP(); PyObject *lhs = TOP(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 055accd9d60aed..83c0f8028e2f6c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -361,6 +361,34 @@ DISPATCH(); } + TARGET(BINARY_SUBSCR) { + PREDICTED(BINARY_SUBSCR); + if (!cframe.use_tracing) { + _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *sub = TOP(); + PyObject *container = SECOND(); + next_instr--; + if (_Py_Specialize_BinarySubscr(container, sub, next_instr) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + STAT_INC(BINARY_SUBSCR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + } + PyObject *sub = POP(); + PyObject *container = TOP(); + PyObject *res = PyObject_GetItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + SET_TOP(res); + if (res == NULL) + goto error; + JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); + DISPATCH(); + } + TARGET(BINARY_SLICE) { PyObject *stop = POP(); PyObject *start = POP(); @@ -401,34 +429,6 @@ DISPATCH(); } - TARGET(BINARY_SUBSCR) { - PREDICTED(BINARY_SUBSCR); - if (!cframe.use_tracing) { - _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *sub = TOP(); - PyObject *container = SECOND(); - next_instr--; - if (_Py_Specialize_BinarySubscr(container, sub, next_instr) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); - } - PyObject *sub = POP(); - PyObject *container = TOP(); - PyObject *res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - SET_TOP(res); - if (res == NULL) - goto error; - JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - DISPATCH(); - } - TARGET(BINARY_SUBSCR_LIST_INT) { assert(cframe.use_tracing == 0); PyObject *sub = TOP(); @@ -1197,6 +1197,37 @@ DISPATCH(); } + TARGET(STORE_ATTR) { + PREDICTED(STORE_ATTR); + if (!cframe.use_tracing) { + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + PyObject *owner = TOP(); + PyObject *name = GETITEM(names, oparg); + next_instr--; + if (_Py_Specialize_StoreAttr(owner, next_instr, name) < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + STAT_INC(STORE_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + } + PyObject *name = GETITEM(names, oparg); + PyObject *owner = TOP(); + PyObject *v = SECOND(); + int err; + STACK_SHRINK(2); + err = PyObject_SetAttr(owner, name, v); + Py_DECREF(v); + Py_DECREF(owner); + if (err != 0) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); + DISPATCH(); + } + TARGET(DELETE_ATTR) { PyObject *name = GETITEM(names, oparg); PyObject *owner = POP(); @@ -2029,37 +2060,6 @@ goto start_frame; } - TARGET(STORE_ATTR) { - PREDICTED(STORE_ATTR); - if (!cframe.use_tracing) { - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - PyObject *owner = TOP(); - PyObject *name = GETITEM(names, oparg); - next_instr--; - if (_Py_Specialize_StoreAttr(owner, next_instr, name) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); - } - PyObject *name = GETITEM(names, oparg); - PyObject *owner = TOP(); - PyObject *v = SECOND(); - int err; - STACK_SHRINK(2); - err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); - if (err != 0) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); - DISPATCH(); - } - TARGET(STORE_ATTR_INSTANCE_VALUE) { assert(cframe.use_tracing == 0); PyObject *owner = TOP(); diff --git a/Python/specialize.c b/Python/specialize.c index 1228828bc855af..0c20803e2a2ec6 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -270,7 +270,7 @@ _PyCode_Quicken(PyCodeObject *code) int opcode = _PyOpcode_Deopt[_Py_OPCODE(instructions[i])]; int caches = _PyOpcode_Caches[opcode]; if (caches) { - instructions[i + 1] = adaptive_counter_start(); + instructions[i + 1] = adaptive_counter_warmup(); previous_opcode = -1; i += caches; } @@ -835,7 +835,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) success: STAT_INC(LOAD_ATTR, success); assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_restart(); + cache->counter = adaptive_counter_cooldown(); return 0; } @@ -919,7 +919,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) success: STAT_INC(STORE_ATTR, success); assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_restart(); + cache->counter = adaptive_counter_cooldown(); return 0; } @@ -1178,7 +1178,7 @@ _Py_Specialize_LoadGlobal( success: STAT_INC(LOAD_GLOBAL, success); assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_restart(); + cache->counter = adaptive_counter_cooldown(); return 0; } @@ -1339,7 +1339,7 @@ _Py_Specialize_BinarySubscr( success: STAT_INC(BINARY_SUBSCR, success); assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_restart(); + cache->counter = adaptive_counter_cooldown(); return 0; } @@ -1444,7 +1444,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins success: STAT_INC(STORE_SUBSCR, success); assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_restart(); + cache->counter = adaptive_counter_cooldown(); return 0; } @@ -1758,7 +1758,7 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, else { STAT_INC(CALL, success); assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_restart(); + cache->counter = adaptive_counter_cooldown(); } return 0; } @@ -1911,7 +1911,7 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, return; success: STAT_INC(BINARY_OP, success); - cache->counter = adaptive_counter_restart(); + cache->counter = adaptive_counter_cooldown(); } @@ -2032,7 +2032,7 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, return; success: STAT_INC(COMPARE_OP, success); - cache->counter = adaptive_counter_restart(); + cache->counter = adaptive_counter_cooldown(); } #ifdef Py_STATS @@ -2083,7 +2083,7 @@ _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) return; success: STAT_INC(UNPACK_SEQUENCE, success); - cache->counter = adaptive_counter_restart(); + cache->counter = adaptive_counter_cooldown(); } #ifdef Py_STATS @@ -2182,5 +2182,5 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr) return; success: STAT_INC(FOR_ITER, success); - cache->counter = adaptive_counter_restart(); + cache->counter = adaptive_counter_cooldown(); } From 3639b66a409b6328cc0215de402b9b8ccd44aaac Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sat, 5 Nov 2022 07:04:13 -0700 Subject: [PATCH 10/21] Clarify the reasoning behind each counter value --- Include/internal/pycore_code.h | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 66f42f62acd15b..2e5e73cd4942a2 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -376,11 +376,21 @@ write_location_entry_start(uint8_t *ptr, int code, int length) /* With a 16-bit counter, we have 12 bits for the counter value, and 4 bits for the backoff */ #define ADAPTIVE_BACKOFF_BITS 4 -#define ADAPTIVE_BACKOFF_START_VALUE 1 -#define ADAPTIVE_BACKOFF_START_BACKOFF 1 - -#define ADAPTIVE_BACKOFF_RESTART_VALUE 52 -#define ADAPTIVE_BACKOFF_RESTART_BACKOFF 0 +// A value of 1 means that we attempt to specialize the *second* time each +// instruction is executed. Executing twice is a much better indicator of +// "hotness" than executing once, but additional warmup delays only prevent +// specialization. Most types stabilize by the second execution, too: +#define ADAPTIVE_WARMUP_VALUE 1 +#define ADAPTIVE_WARMUP_BACKOFF 1 + +// A value of 52 means that we attempt to re-specialize after 53 misses (a prime +// number, useful for avoiding artifacts if every nth value is a different type +// or something). Setting the backoff to 0 means that the counter is reset to +// the same state as a warming-up instruction (value == 1, backoff == 1) after +// deoptimization. This isn't strictly necessary, but it is bit easier to reason +// about when thinking about the opcode transitions as a state machine: +#define ADAPTIVE_COOLDOWN_VALUE 52 +#define ADAPTIVE_COOLDOWN_BACKOFF 0 #define MAX_BACKOFF_VALUE (16 - ADAPTIVE_BACKOFF_BITS) @@ -392,15 +402,15 @@ adaptive_counter_bits(int value, int backoff) { } static inline uint16_t -adaptive_counter_start(void) { - return adaptive_counter_bits(ADAPTIVE_BACKOFF_START_VALUE, - ADAPTIVE_BACKOFF_START_BACKOFF); +adaptive_counter_warmup(void) { + return adaptive_counter_bits(ADAPTIVE_WARMUP_VALUE, + ADAPTIVE_WARMUP_BACKOFF); } static inline uint16_t -adaptive_counter_restart(void) { - return adaptive_counter_bits(ADAPTIVE_BACKOFF_RESTART_VALUE, - ADAPTIVE_BACKOFF_RESTART_BACKOFF); +adaptive_counter_cooldown(void) { + return adaptive_counter_bits(ADAPTIVE_COOLDOWN_VALUE, + ADAPTIVE_COOLDOWN_BACKOFF); } static inline uint16_t From 84bc48114d768b8dfd9883efa6bf9bf20c52af8e Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sat, 5 Nov 2022 12:03:01 -0700 Subject: [PATCH 11/21] Merge EXTENDED_ARG and EXTENDED_ARG_QUICK --- Include/internal/pycore_opcode.h | 26 +++++++------- Include/opcode.h | 57 +++++++++++++++--------------- Lib/opcode.py | 3 -- Lib/test/test_embed.py | 2 -- Python/bytecodes.c | 20 ++++------- Python/compile.c | 3 -- Python/generated_cases.c.h | 18 ++++------ Python/opcode_targets.h | 24 ++++++------- Python/specialize.c | 59 ++++++++++++-------------------- 9 files changed, 86 insertions(+), 126 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index cbeaeddc9da6a2..defd70ca251c3d 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -122,7 +122,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [END_ASYNC_FOR] = END_ASYNC_FOR, [END_FOR] = END_FOR, [EXTENDED_ARG] = EXTENDED_ARG, - [EXTENDED_ARG_QUICK] = EXTENDED_ARG, [FORMAT_VALUE] = FORMAT_VALUE, [FOR_ITER] = FOR_ITER, [FOR_ITER_LIST] = FOR_ITER, @@ -294,31 +293,31 @@ static const char *const _PyOpcode_OpName[263] = { [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", - [EXTENDED_ARG_QUICK] = "EXTENDED_ARG_QUICK", - [STOPITERATION_ERROR] = "STOPITERATION_ERROR", [FOR_ITER_LIST] = "FOR_ITER_LIST", + [STOPITERATION_ERROR] = "STOPITERATION_ERROR", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [PRINT_EXPR] = "PRINT_EXPR", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", - [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [LIST_TO_TUPLE] = "LIST_TO_TUPLE", [RETURN_VALUE] = "RETURN_VALUE", [IMPORT_STAR] = "IMPORT_STAR", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", @@ -345,7 +344,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -353,7 +352,7 @@ static const char *const _PyOpcode_OpName[263] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -373,9 +372,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", - [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -385,24 +384,24 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", + [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", - [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", + [170] = "<170>", [CALL] = "CALL", [KW_NAMES] = "KW_NAMES", [173] = "<173>", @@ -499,6 +498,7 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ + case 170: \ case 173: \ case 174: \ case 175: \ diff --git a/Include/opcode.h b/Include/opcode.h index 745fad7feaae6c..29ee69d465a7e8 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -162,35 +162,34 @@ extern "C" { #define COMPARE_OP_GENERIC 57 #define COMPARE_OP_INT_JUMP 58 #define COMPARE_OP_STR_JUMP 59 -#define EXTENDED_ARG_QUICK 62 -#define FOR_ITER_LIST 64 -#define FOR_ITER_RANGE 65 -#define LOAD_ATTR_CLASS 66 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 67 -#define LOAD_ATTR_INSTANCE_VALUE 72 -#define LOAD_ATTR_MODULE 73 -#define LOAD_ATTR_PROPERTY 76 -#define LOAD_ATTR_SLOT 77 -#define LOAD_ATTR_WITH_HINT 78 -#define LOAD_ATTR_METHOD_LAZY_DICT 79 -#define LOAD_ATTR_METHOD_NO_DICT 80 -#define LOAD_ATTR_METHOD_WITH_DICT 81 -#define LOAD_ATTR_METHOD_WITH_VALUES 86 -#define LOAD_CONST__LOAD_FAST 113 -#define LOAD_FAST__LOAD_CONST 121 -#define LOAD_FAST__LOAD_FAST 141 -#define LOAD_GLOBAL_BUILTIN 143 -#define LOAD_GLOBAL_MODULE 153 -#define STORE_ATTR_INSTANCE_VALUE 154 -#define STORE_ATTR_SLOT 158 -#define STORE_ATTR_WITH_HINT 159 -#define STORE_FAST__LOAD_FAST 160 -#define STORE_FAST__STORE_FAST 161 -#define STORE_SUBSCR_DICT 166 -#define STORE_SUBSCR_LIST_INT 167 -#define UNPACK_SEQUENCE_LIST 168 -#define UNPACK_SEQUENCE_TUPLE 169 -#define UNPACK_SEQUENCE_TWO_TUPLE 170 +#define FOR_ITER_LIST 62 +#define FOR_ITER_RANGE 64 +#define LOAD_ATTR_CLASS 65 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 66 +#define LOAD_ATTR_INSTANCE_VALUE 67 +#define LOAD_ATTR_MODULE 72 +#define LOAD_ATTR_PROPERTY 73 +#define LOAD_ATTR_SLOT 76 +#define LOAD_ATTR_WITH_HINT 77 +#define LOAD_ATTR_METHOD_LAZY_DICT 78 +#define LOAD_ATTR_METHOD_NO_DICT 79 +#define LOAD_ATTR_METHOD_WITH_DICT 80 +#define LOAD_ATTR_METHOD_WITH_VALUES 81 +#define LOAD_CONST__LOAD_FAST 86 +#define LOAD_FAST__LOAD_CONST 113 +#define LOAD_FAST__LOAD_FAST 121 +#define LOAD_GLOBAL_BUILTIN 141 +#define LOAD_GLOBAL_MODULE 143 +#define STORE_ATTR_INSTANCE_VALUE 153 +#define STORE_ATTR_SLOT 154 +#define STORE_ATTR_WITH_HINT 158 +#define STORE_FAST__LOAD_FAST 159 +#define STORE_FAST__STORE_FAST 160 +#define STORE_SUBSCR_DICT 161 +#define STORE_SUBSCR_LIST_INT 166 +#define UNPACK_SEQUENCE_LIST 167 +#define UNPACK_SEQUENCE_TUPLE 168 +#define UNPACK_SEQUENCE_TWO_TUPLE 169 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/opcode.py b/Lib/opcode.py index 7c5628e0ebe952..0de42fe9989f43 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -319,9 +319,6 @@ def pseudo_op(name, op, real_ops): "COMPARE_OP_INT_JUMP", "COMPARE_OP_STR_JUMP", ], - "EXTENDED_ARG": [ - "EXTENDED_ARG_QUICK", - ], "FOR_ITER": [ "FOR_ITER_LIST", "FOR_ITER_RANGE", diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index d56bea22d15093..2dda7ccf7bf80c 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -362,8 +362,6 @@ def is_specialized(f): opname in opcode._specialized_instructions # Exclude superinstructions: and "__" not in opname - # Exclude "quick" instructions: - and not opname.endswith("_QUICK") ): return True return False diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 381cfac302d633..c369d957702f50 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3930,11 +3930,14 @@ dummy_func( // stack effect: ( -- ) inst(EXTENDED_ARG) { assert(oparg); + opcode = _Py_OPCODE(*next_instr); + if (cframe.use_tracing) { + // Deoptimize the next opcode to avoid breaking tracing + // guarantees in quickened instructions: + opcode = _PyOpcode_Deopt[opcode]; + } oparg <<= 8; oparg |= _Py_OPARG(*next_instr); - // We might be tracing. To avoid breaking tracing guarantees in - // quickened instructions, always deoptimize the next opcode: - opcode = _PyOpcode_Deopt[_Py_OPCODE(*next_instr)]; PRE_DISPATCH_GOTO(); // CPython hasn't traced the following instruction historically // (DO_TRACING would clobber our extended oparg anyways), so just @@ -3945,16 +3948,6 @@ dummy_func( DISPATCH_GOTO(); } - // stack effect: ( -- ) - inst(EXTENDED_ARG_QUICK) { - assert(cframe.use_tracing == 0); - assert(oparg); - int oldoparg = oparg; - NEXTOPARG(); - oparg |= oldoparg << 8; - DISPATCH_GOTO(); - } - // stack effect: ( -- ) inst(CACHE) { Py_UNREACHABLE(); @@ -3995,7 +3988,6 @@ family(call) = { family(compare_op) = { COMPARE_OP, COMPARE_OP_FLOAT_JUMP, COMPARE_OP_GENERIC, COMPARE_OP_INT_JUMP, COMPARE_OP_STR_JUMP }; -family(extended_arg) = { EXTENDED_ARG, EXTENDED_ARG_QUICK }; family(for_iter) = { FOR_ITER, FOR_ITER_LIST, FOR_ITER_RANGE }; diff --git a/Python/compile.c b/Python/compile.c index 065d1b08d0642f..030378f19a3362 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -8062,7 +8062,6 @@ scan_block_for_locals(basicblock *b, basicblock ***sp) for (int i = 0; i < b->b_iused; i++) { struct instr *instr = &b->b_instr[i]; assert(instr->i_opcode != EXTENDED_ARG); - assert(instr->i_opcode != EXTENDED_ARG_QUICK); assert(!IS_SUPERINSTRUCTION_OPCODE(instr->i_opcode)); if (instr->i_except != NULL) { maybe_push(instr->i_except, unsafe_mask, sp); @@ -8119,7 +8118,6 @@ fast_scan_many_locals(basicblock *entryblock, int nlocals) for (int i = 0; i < b->b_iused; i++) { struct instr *instr = &b->b_instr[i]; assert(instr->i_opcode != EXTENDED_ARG); - assert(instr->i_opcode != EXTENDED_ARG_QUICK); assert(!IS_SUPERINSTRUCTION_OPCODE(instr->i_opcode)); int arg = instr->i_oparg; if (arg < 64) { @@ -8667,7 +8665,6 @@ fix_cell_offsets(struct compiler *c, basicblock *entryblock, int *fixedmap) struct instr *inst = &b->b_instr[i]; // This is called before extended args are generated. assert(inst->i_opcode != EXTENDED_ARG); - assert(inst->i_opcode != EXTENDED_ARG_QUICK); int oldoffset = inst->i_oparg; switch(inst->i_opcode) { case MAKE_CELL: diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 83c0f8028e2f6c..215d8129f745da 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3826,11 +3826,14 @@ TARGET(EXTENDED_ARG) { assert(oparg); + opcode = _Py_OPCODE(*next_instr); + if (cframe.use_tracing) { + // Deoptimize the next opcode to avoid breaking tracing + // guarantees in quickened instructions: + opcode = _PyOpcode_Deopt[opcode]; + } oparg <<= 8; oparg |= _Py_OPARG(*next_instr); - // We might be tracing. To avoid breaking tracing guarantees in - // quickened instructions, always deoptimize the next opcode: - opcode = _PyOpcode_Deopt[_Py_OPCODE(*next_instr)]; PRE_DISPATCH_GOTO(); // CPython hasn't traced the following instruction historically // (DO_TRACING would clobber our extended oparg anyways), so just @@ -3841,15 +3844,6 @@ DISPATCH_GOTO(); } - TARGET(EXTENDED_ARG_QUICK) { - assert(cframe.use_tracing == 0); - assert(oparg); - int oldoparg = oparg; - NEXTOPARG(); - oparg |= oldoparg << 8; - DISPATCH_GOTO(); - } - TARGET(CACHE) { Py_UNREACHABLE(); } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 42ae6042efb519..e99a2c09a1e00c 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -61,31 +61,31 @@ static void *opcode_targets[256] = { &&TARGET_COMPARE_OP_STR_JUMP, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, - &&TARGET_EXTENDED_ARG_QUICK, - &&TARGET_STOPITERATION_ERROR, &&TARGET_FOR_ITER_LIST, + &&TARGET_STOPITERATION_ERROR, &&TARGET_FOR_ITER_RANGE, &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, - &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_ATTR_MODULE, + &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, - &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_FAST__LOAD_FAST, - &&TARGET_CALL_FUNCTION_EX, &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_CALL_FUNCTION_EX, + &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,24 +152,24 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_STORE_ATTR_SLOT, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, - &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_STORE_FAST__STORE_FAST, + &&TARGET_STORE_SUBSCR_DICT, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, - &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, + &&_unknown_opcode, &&TARGET_CALL, &&TARGET_KW_NAMES, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index 0c20803e2a2ec6..723424189c97d2 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -260,56 +260,39 @@ do { \ #define SPECIALIZATION_FAIL(opcode, kind) ((void)0) #endif -// Insert adaptive instructions and superinstructions. This cannot fail. +// Initialize warmup counters and insert superinstructions. This cannot fail. void _PyCode_Quicken(PyCodeObject *code) { - int previous_opcode = -1; + int previous_opcode = 0; _Py_CODEUNIT *instructions = _PyCode_CODE(code); for (int i = 0; i < Py_SIZE(code); i++) { int opcode = _PyOpcode_Deopt[_Py_OPCODE(instructions[i])]; int caches = _PyOpcode_Caches[opcode]; if (caches) { instructions[i + 1] = adaptive_counter_warmup(); - previous_opcode = -1; + previous_opcode = 0; i += caches; + continue; } - else { - switch (opcode) { - case EXTENDED_ARG: - _Py_SET_OPCODE(instructions[i], EXTENDED_ARG_QUICK); - break; - case LOAD_FAST: - switch(previous_opcode) { - case LOAD_FAST: - _Py_SET_OPCODE(instructions[i - 1], - LOAD_FAST__LOAD_FAST); - break; - case STORE_FAST: - _Py_SET_OPCODE(instructions[i - 1], - STORE_FAST__LOAD_FAST); - break; - case LOAD_CONST: - _Py_SET_OPCODE(instructions[i - 1], - LOAD_CONST__LOAD_FAST); - break; - } - break; - case STORE_FAST: - if (previous_opcode == STORE_FAST) { - _Py_SET_OPCODE(instructions[i - 1], - STORE_FAST__STORE_FAST); - } - break; - case LOAD_CONST: - if (previous_opcode == LOAD_FAST) { - _Py_SET_OPCODE(instructions[i - 1], - LOAD_FAST__LOAD_CONST); - } - break; - } - previous_opcode = opcode; + switch (previous_opcode << 8 | opcode) { + case LOAD_CONST << 8 | LOAD_FAST: + _Py_SET_OPCODE(instructions[i - 1], LOAD_CONST__LOAD_FAST); + break; + case LOAD_FAST << 8 | LOAD_CONST: + _Py_SET_OPCODE(instructions[i - 1], LOAD_FAST__LOAD_CONST); + break; + case LOAD_FAST << 8 | LOAD_FAST: + _Py_SET_OPCODE(instructions[i - 1], LOAD_FAST__LOAD_FAST); + break; + case STORE_FAST << 8 | LOAD_FAST: + _Py_SET_OPCODE(instructions[i - 1], STORE_FAST__LOAD_FAST); + break; + case STORE_FAST << 8 | STORE_FAST: + _Py_SET_OPCODE(instructions[i - 1], STORE_FAST__STORE_FAST); + break; } + previous_opcode = opcode; } } From f885e6c6e2e9db4787be9b47f18634fb345835dc Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sat, 5 Nov 2022 12:53:35 -0700 Subject: [PATCH 12/21] Fix stats --- Python/bytecodes.c | 50 +++++++++++++++++++------------------- Python/ceval.c | 29 ++++++++++++++-------- Python/generated_cases.c.h | 44 ++++++++++++++++----------------- 3 files changed, 66 insertions(+), 57 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c369d957702f50..1336b900398440 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -467,7 +467,7 @@ dummy_func( inst(BINARY_SUBSCR) { if (!cframe.use_tracing) { _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *sub = TOP(); PyObject *container = SECOND(); next_instr--; @@ -477,7 +477,7 @@ dummy_func( DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } PyObject *sub = POP(); PyObject *container = TOP(); @@ -661,7 +661,7 @@ dummy_func( inst(STORE_SUBSCR) { if (!cframe.use_tracing) { _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *sub = TOP(); PyObject *container = SECOND(); next_instr--; @@ -671,7 +671,7 @@ dummy_func( DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } PyObject *sub = TOP(); PyObject *container = SECOND(); @@ -1227,14 +1227,14 @@ dummy_func( inst(UNPACK_SEQUENCE) { if (!cframe.use_tracing) { _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *seq = TOP(); next_instr--; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } PyObject *seq = POP(); PyObject **top = stack_pointer + oparg; @@ -1306,7 +1306,7 @@ dummy_func( inst(STORE_ATTR) { if (!cframe.use_tracing) { _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg); next_instr--; @@ -1316,7 +1316,7 @@ dummy_func( DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } PyObject *name = GETITEM(names, oparg); PyObject *owner = TOP(); @@ -1436,7 +1436,7 @@ dummy_func( inst(LOAD_GLOBAL) { if (!cframe.use_tracing) { _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *name = GETITEM(names, oparg>>1); next_instr--; if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) { @@ -1445,7 +1445,7 @@ dummy_func( DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } int push_null = oparg & 1; PEEK(0) = NULL; @@ -1893,7 +1893,7 @@ dummy_func( inst(LOAD_ATTR) { if (!cframe.use_tracing) { _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg>>1); next_instr--; @@ -1903,7 +1903,7 @@ dummy_func( DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } PyObject *name = GETITEM(names, oparg >> 1); PyObject *owner = TOP(); @@ -2286,7 +2286,7 @@ dummy_func( inst(COMPARE_OP) { if (!cframe.use_tracing) { _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *right = TOP(); PyObject *left = SECOND(); next_instr--; @@ -2294,7 +2294,7 @@ dummy_func( DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } GO_TO_INSTRUCTION(COMPARE_OP_GENERIC); } @@ -2779,13 +2779,13 @@ dummy_func( inst(FOR_ITER) { if (!cframe.use_tracing) { _PyForIterCache *cache = (_PyForIterCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { next_instr--; _Py_Specialize_ForIter(TOP(), next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } /* before: [iter]; after: [iter, iter()] *or* [] */ PyObject *iter = TOP(); @@ -3102,7 +3102,7 @@ dummy_func( inst(CALL) { if (!cframe.use_tracing) { _PyCallCache *cache = (_PyCallCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { next_instr--; int is_meth = is_method(stack_pointer, oparg); int nargs = oparg + is_meth; @@ -3115,7 +3115,7 @@ dummy_func( DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } int total_args, is_meth; is_meth = is_method(stack_pointer, oparg); @@ -3906,7 +3906,7 @@ dummy_func( inst(BINARY_OP) { if (!cframe.use_tracing) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *lhs = SECOND(); PyObject *rhs = TOP(); next_instr--; @@ -3914,7 +3914,7 @@ dummy_func( DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } GO_TO_INSTRUCTION(BINARY_OP_GENERIC); } @@ -3930,11 +3930,11 @@ dummy_func( // stack effect: ( -- ) inst(EXTENDED_ARG) { assert(oparg); - opcode = _Py_OPCODE(*next_instr); - if (cframe.use_tracing) { - // Deoptimize the next opcode to avoid breaking tracing - // guarantees in quickened instructions: - opcode = _PyOpcode_Deopt[opcode]; + opcode = _Py_OPCODE(*next_instr); + if (cframe.use_tracing) { + // Deoptimize the next opcode to avoid breaking tracing + // guarantees in quickened instructions: + opcode = _PyOpcode_Deopt[opcode]; } oparg <<= 8; oparg |= _Py_OPARG(*next_instr); diff --git a/Python/ceval.c b/Python/ceval.c index 524e7d17aa5c74..ebb0e8088f48d3 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -852,12 +852,21 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define GO_TO_INSTRUCTION(op) goto PREDICT_ID(op) -#define DEOPT_IF(COND, INSTNAME) \ - if (COND) { \ - STAT_INC(opcode, miss); \ - STAT_INC(INSTNAME, miss); \ - assert(_PyOpcode_Deopt[opcode] == INSTNAME); \ - GO_TO_INSTRUCTION(INSTNAME); \ +#define DEOPT_IF(COND, INSTNAME) \ + if (COND) { \ + /* This is only a single jump on release builds! */ \ + STAT_INC(opcode, miss); \ + STAT_INC(INSTNAME, miss); \ + /* The counter is always the first cache entry: */ \ + if (ADAPTIVE_COUNTER_IS_ZERO(*next_instr)) { \ + STAT_INC(INSTNAME, deopt); \ + } \ + else { \ + /* This is about to be (incorrectly) incremented: */ \ + STAT_DEC(INSTNAME, deferred); \ + } \ + assert(_PyOpcode_Deopt[opcode] == INSTNAME); \ + GO_TO_INSTRUCTION(INSTNAME); \ } @@ -910,11 +919,11 @@ GETITEM(PyObject *v, Py_ssize_t i) { dtrace_function_entry(frame); \ } -#define ADAPTIVE_COUNTER_IS_ZERO(cache) \ - (cache)->counter < (1<counter -= (1<counter)) { PyObject *sub = TOP(); PyObject *container = SECOND(); next_instr--; @@ -375,7 +375,7 @@ DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } PyObject *sub = POP(); PyObject *container = TOP(); @@ -559,7 +559,7 @@ PREDICTED(STORE_SUBSCR); if (!cframe.use_tracing) { _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *sub = TOP(); PyObject *container = SECOND(); next_instr--; @@ -569,7 +569,7 @@ DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } PyObject *sub = TOP(); PyObject *container = SECOND(); @@ -1121,14 +1121,14 @@ PREDICTED(UNPACK_SEQUENCE); if (!cframe.use_tracing) { _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *seq = TOP(); next_instr--; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } PyObject *seq = POP(); PyObject **top = stack_pointer + oparg; @@ -1201,7 +1201,7 @@ PREDICTED(STORE_ATTR); if (!cframe.use_tracing) { _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg); next_instr--; @@ -1211,7 +1211,7 @@ DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } PyObject *name = GETITEM(names, oparg); PyObject *owner = TOP(); @@ -1332,7 +1332,7 @@ PREDICTED(LOAD_GLOBAL); if (!cframe.use_tracing) { _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *name = GETITEM(names, oparg>>1); next_instr--; if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) { @@ -1341,7 +1341,7 @@ DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } int push_null = oparg & 1; PEEK(0) = NULL; @@ -1788,7 +1788,7 @@ PREDICTED(LOAD_ATTR); if (!cframe.use_tracing) { _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg>>1); next_instr--; @@ -1798,7 +1798,7 @@ DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } PyObject *name = GETITEM(names, oparg >> 1); PyObject *owner = TOP(); @@ -2181,7 +2181,7 @@ PREDICTED(COMPARE_OP); if (!cframe.use_tracing) { _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *right = TOP(); PyObject *left = SECOND(); next_instr--; @@ -2189,7 +2189,7 @@ DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } GO_TO_INSTRUCTION(COMPARE_OP_GENERIC); } @@ -2676,13 +2676,13 @@ PREDICTED(FOR_ITER); if (!cframe.use_tracing) { _PyForIterCache *cache = (_PyForIterCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { next_instr--; _Py_Specialize_ForIter(TOP(), next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } /* before: [iter]; after: [iter, iter()] *or* [] */ PyObject *iter = TOP(); @@ -2998,7 +2998,7 @@ PREDICTED(CALL); if (!cframe.use_tracing) { _PyCallCache *cache = (_PyCallCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { next_instr--; int is_meth = is_method(stack_pointer, oparg); int nargs = oparg + is_meth; @@ -3011,7 +3011,7 @@ DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } int total_args, is_meth; is_meth = is_method(stack_pointer, oparg); @@ -3803,7 +3803,7 @@ PREDICTED(BINARY_OP); if (!cframe.use_tracing) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { PyObject *lhs = SECOND(); PyObject *rhs = TOP(); next_instr--; @@ -3811,7 +3811,7 @@ DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); } GO_TO_INSTRUCTION(BINARY_OP_GENERIC); } @@ -3828,8 +3828,8 @@ assert(oparg); opcode = _Py_OPCODE(*next_instr); if (cframe.use_tracing) { - // Deoptimize the next opcode to avoid breaking tracing - // guarantees in quickened instructions: + // Deoptimize the next opcode to avoid breaking tracing + // guarantees in quickened instructions: opcode = _PyOpcode_Deopt[opcode]; } oparg <<= 8; From f33d882b2e7ea77eb763a9f8cd92e8823f886cc3 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sat, 5 Nov 2022 13:05:36 -0700 Subject: [PATCH 13/21] fixup --- Python/generated_cases.c.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1d915e11c765b2..5a244783e5ae99 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3828,8 +3828,8 @@ assert(oparg); opcode = _Py_OPCODE(*next_instr); if (cframe.use_tracing) { - // Deoptimize the next opcode to avoid breaking tracing - // guarantees in quickened instructions: + // Deoptimize the next opcode to avoid breaking tracing + // guarantees in quickened instructions: opcode = _PyOpcode_Deopt[opcode]; } oparg <<= 8; From cf98d4aba6a2aa3e03b027bced7a0ead010f70a5 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sun, 6 Nov 2022 00:48:00 -0700 Subject: [PATCH 14/21] blurb add --- .../2022-11-06-00-47-11.gh-issue-98686.DBDy6U.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-11-06-00-47-11.gh-issue-98686.DBDy6U.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-06-00-47-11.gh-issue-98686.DBDy6U.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-06-00-47-11.gh-issue-98686.DBDy6U.rst new file mode 100644 index 00000000000000..f259b389e32a1b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-06-00-47-11.gh-issue-98686.DBDy6U.rst @@ -0,0 +1,5 @@ +Merge the adaptive opcode logic into each instruction's unquickened variant, +and merge the logic in ``EXTENDED_ARG_QUICK`` into :opcode:`EXTENDED_ARG`. +With these changes, the quickening that happens at code object creation is +now only responsible for initializing warmup counters and inserting +superinstructions. From db558f12392ae66580b0273a8706efac06c8e310 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 21 Oct 2022 02:02:17 -0700 Subject: [PATCH 15/21] Update adaptive.md --- Python/adaptive.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Python/adaptive.md b/Python/adaptive.md index e8161bcdd5b9c6..d978c089b237e0 100644 --- a/Python/adaptive.md +++ b/Python/adaptive.md @@ -11,7 +11,7 @@ A family of instructions has the following fundamental properties: generated by the bytecode compiler. * It has a single adaptive instruction that records an execution count and, at regular intervals, attempts to specialize itself. If not specializing, - it executes the non-adaptive instruction. + it executes the base implementation. * It has at least one specialized form of the instruction that is tailored for a particular value or set of values at runtime. * All members of the family must have the same number of inline cache entries, @@ -22,19 +22,18 @@ A family of instructions has the following fundamental properties: The current implementation also requires the following, although these are not fundamental and may change: -* All families uses one or more inline cache entries, +* All families use one or more inline cache entries, the first entry is always the counter. -* All instruction names should start with the name of the non-adaptive +* All instruction names should start with the name of the adaptive instruction. -* The adaptive instruction should end in `_ADAPTIVE`. * Specialized forms should have names describing their specialization. ## Example family -The `LOAD_GLOBAL` instruction (in Python/ceval.c) already has an adaptive +The `LOAD_GLOBAL` instruction (in Python/bytecodes.c) already has an adaptive family that serves as a relatively simple example. -The `LOAD_GLOBAL_ADAPTIVE` instruction performs adaptive specialization, +The `LOAD_GLOBAL` instruction performs adaptive specialization, calling `_Py_Specialize_LoadGlobal()` when the counter reaches zero. There are two specialized instructions in the family, `LOAD_GLOBAL_MODULE` @@ -138,5 +137,5 @@ to eliminate the branches. Finally, take care that stats are gather correctly. After the last `DEOPT_IF` has passed, a hit should be recorded with `STAT_INC(BASE_INSTRUCTION, hit)`. -After a optimization has been deferred in the `ADAPTIVE` form, +After an optimization has been deferred in the adaptive instruction, that should be recorded with `STAT_INC(BASE_INSTRUCTION, deferred)`. From b8796e68063551b1e8c72721ed96ac4675c49ed6 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 21 Oct 2022 02:08:40 -0700 Subject: [PATCH 16/21] Remove branching from EXTENDED_ARG --- Python/bytecodes.c | 15 ++------------- Python/ceval.c | 24 ++++++++++++++++-------- Python/generated_cases.c.h | 15 ++------------- 3 files changed, 20 insertions(+), 34 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 717ee5c8cabbdd..76e24025c1e82d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3911,21 +3911,10 @@ dummy_func( // stack effect: ( -- ) inst(EXTENDED_ARG) { assert(oparg); + assert(cframe.use_tracing == 0); opcode = _Py_OPCODE(*next_instr); - if (cframe.use_tracing) { - // Deoptimize the next opcode to avoid breaking tracing - // guarantees in quickened instructions: - opcode = _PyOpcode_Deopt[opcode]; - } - oparg <<= 8; - oparg |= _Py_OPARG(*next_instr); + oparg = oparg << 8 | _Py_OPARG(*next_instr); PRE_DISPATCH_GOTO(); - // CPython hasn't traced the following instruction historically - // (DO_TRACING would clobber our extended oparg anyways), so just - // skip our usual cframe.use_tracing check before dispatch. Also, - // make sure the next instruction isn't a RESUME, since that needs - // to trace properly (and shouldn't have an extended arg anyways): - assert(opcode != RESUME); DISPATCH_GOTO(); } diff --git a/Python/ceval.c b/Python/ceval.c index 61fdde666dd963..1a54a7e5363afc 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -743,12 +743,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define JUMPTO(x) (next_instr = first_instr + (x)) #define JUMPBY(x) (next_instr += (x)) -/* Get opcode and oparg from original instructions, not quickened form. */ -#define TRACING_NEXTOPARG() do { \ - NEXTOPARG(); \ - opcode = _PyOpcode_Deopt[opcode]; \ - } while (0) - /* OpCode prediction macros Some opcodes tend to come in pairs thus making it possible to predict the second code when the first is run. For example, @@ -1186,7 +1180,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int if (INSTR_OFFSET() >= frame->f_code->_co_firsttraceable) { int instr_prev = _PyInterpreterFrame_LASTI(frame); frame->prev_instr = next_instr; - TRACING_NEXTOPARG(); + NEXTOPARG(); + // No _PyOpcode_Deopt here, since RESUME has no optimized forms: if (opcode == RESUME) { if (oparg < 2) { CHECK_EVAL_BREAKER(); @@ -1233,8 +1228,21 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } } } - TRACING_NEXTOPARG(); + NEXTOPARG(); PRE_DISPATCH_GOTO(); + // No _PyOpcode_Deopt here, since EXTENDED_ARG has no optimized forms: + while (opcode == EXTENDED_ARG) { + // CPython hasn't ever traced the instruction after an EXTENDED_ARG. + // Inline the instruction here, so we can avoid branching there: + INSTRUCTION_START(); + opcode = _Py_OPCODE(*next_instr); + oparg = oparg << 8 | _Py_OPARG(*next_instr); + // Make sure the next instruction isn't a RESUME, since that needs + // to trace properly (and shouldn't have an EXTENDED_ARG, anyways): + assert(opcode != RESUME); + PRE_DISPATCH_GOTO(); + } + opcode = _PyOpcode_Deopt[opcode]; DISPATCH_GOTO(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e4c7de00269228..23166085e60519 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3799,21 +3799,10 @@ TARGET(EXTENDED_ARG) { assert(oparg); + assert(cframe.use_tracing == 0); opcode = _Py_OPCODE(*next_instr); - if (cframe.use_tracing) { - // Deoptimize the next opcode to avoid breaking tracing - // guarantees in quickened instructions: - opcode = _PyOpcode_Deopt[opcode]; - } - oparg <<= 8; - oparg |= _Py_OPARG(*next_instr); + oparg = oparg << 8 | _Py_OPARG(*next_instr); PRE_DISPATCH_GOTO(); - // CPython hasn't traced the following instruction historically - // (DO_TRACING would clobber our extended oparg anyways), so just - // skip our usual cframe.use_tracing check before dispatch. Also, - // make sure the next instruction isn't a RESUME, since that needs - // to trace properly (and shouldn't have an extended arg anyways): - assert(opcode != RESUME); DISPATCH_GOTO(); } From a7a451bdad25af74fc676e9ffc89d80e2d5d3c1a Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 21 Oct 2022 02:08:56 -0700 Subject: [PATCH 17/21] fixup --- Python/ceval.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ceval.c b/Python/ceval.c index 1a54a7e5363afc..8971631a72dd54 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1233,7 +1233,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int // No _PyOpcode_Deopt here, since EXTENDED_ARG has no optimized forms: while (opcode == EXTENDED_ARG) { // CPython hasn't ever traced the instruction after an EXTENDED_ARG. - // Inline the instruction here, so we can avoid branching there: + // Inline the EXTENDED_ARG here, so we can avoid branching there: INSTRUCTION_START(); opcode = _Py_OPCODE(*next_instr); oparg = oparg << 8 | _Py_OPARG(*next_instr); From 6fdf5adf74f828b8466e4e018cd6c35512ed8fd0 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 21 Oct 2022 03:39:34 -0700 Subject: [PATCH 18/21] Remove tracing branches in adaptive instructions --- Python/bytecodes.c | 234 +++++++++++++++++-------------------- Python/ceval.c | 26 ++++- Python/generated_cases.c.h | 234 +++++++++++++++++-------------------- 3 files changed, 237 insertions(+), 257 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 76e24025c1e82d..d8e03d242681a1 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -410,20 +410,17 @@ dummy_func( // stack effect: (__0 -- ) inst(BINARY_SUBSCR) { - if (!cframe.use_tracing) { - _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *sub = TOP(); - PyObject *container = SECOND(); - next_instr--; - if (_Py_Specialize_BinarySubscr(container, sub, next_instr) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - } + _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + PyObject *sub = TOP(); + PyObject *container = SECOND(); + next_instr--; + _Py_Specialize_BinarySubscr(container, sub, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(BINARY_SUBSCR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); PyObject *sub = POP(); PyObject *container = TOP(); PyObject *res = PyObject_GetItem(container, sub); @@ -604,20 +601,17 @@ dummy_func( // stack effect: (__0, __1, __2 -- ) inst(STORE_SUBSCR) { - if (!cframe.use_tracing) { - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *sub = TOP(); - PyObject *container = SECOND(); - next_instr--; - if (_Py_Specialize_StoreSubscr(container, sub, next_instr) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - } + _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + PyObject *sub = TOP(); + PyObject *container = SECOND(); + next_instr--; + _Py_Specialize_StoreSubscr(container, sub, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(STORE_SUBSCR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); PyObject *sub = TOP(); PyObject *container = SECOND(); PyObject *v = THIRD(); @@ -1182,17 +1176,16 @@ dummy_func( // stack effect: (__0 -- __array[oparg]) inst(UNPACK_SEQUENCE) { - if (!cframe.use_tracing) { - _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *seq = TOP(); - next_instr--; - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - } + _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + PyObject *seq = TOP(); + next_instr--; + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(UNPACK_SEQUENCE, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); PyObject *seq = POP(); PyObject **top = stack_pointer + oparg; if (!unpack_iterable(tstate, seq, oparg, -1, top)) { @@ -1261,20 +1254,19 @@ dummy_func( // stack effect: (__0, __1 -- ) inst(STORE_ATTR) { - if (!cframe.use_tracing) { - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *owner = TOP(); - PyObject *name = GETITEM(names, oparg); - next_instr--; - if (_Py_Specialize_StoreAttr(owner, next_instr, name) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + PyObject *owner = TOP(); + PyObject *name = GETITEM(names, oparg); + next_instr--; + if (_Py_Specialize_StoreAttr(owner, next_instr, name)) { + goto error; } - STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DISPATCH_SAME_OPARG(); } + STAT_INC(STORE_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); PyObject *name = GETITEM(names, oparg); PyObject *owner = TOP(); PyObject *v = SECOND(); @@ -1391,19 +1383,16 @@ dummy_func( // error: LOAD_GLOBAL has irregular stack effect inst(LOAD_GLOBAL) { - if (!cframe.use_tracing) { - _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *name = GETITEM(names, oparg>>1); - next_instr--; - if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - } + _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + PyObject *name = GETITEM(names, oparg>>1); + next_instr--; + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(LOAD_GLOBAL, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); int push_null = oparg & 1; PEEK(0) = NULL; PyObject *name = GETITEM(names, oparg>>1); @@ -1848,20 +1837,19 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR) { - if (!cframe.use_tracing) { - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *owner = TOP(); - PyObject *name = GETITEM(names, oparg>>1); - next_instr--; - if (_Py_Specialize_LoadAttr(owner, next_instr, name) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + PyObject *owner = TOP(); + PyObject *name = GETITEM(names, oparg>>1); + next_instr--; + if (_Py_Specialize_LoadAttr(owner, next_instr, name)) { + goto error; } - STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DISPATCH_SAME_OPARG(); } + STAT_INC(LOAD_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); PyObject *name = GETITEM(names, oparg >> 1); PyObject *owner = TOP(); if (oparg & 1) { @@ -2241,18 +2229,17 @@ dummy_func( // stack effect: (__0 -- ) inst(COMPARE_OP) { - if (!cframe.use_tracing) { - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *right = TOP(); - PyObject *left = SECOND(); - next_instr--; - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - } + _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + PyObject *right = TOP(); + PyObject *left = SECOND(); + next_instr--; + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(COMPARE_OP, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(COMPARE_OP_GENERIC); } @@ -2734,16 +2721,15 @@ dummy_func( // stack effect: ( -- __0) inst(FOR_ITER) { - if (!cframe.use_tracing) { - _PyForIterCache *cache = (_PyForIterCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_ForIter(TOP(), next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - } + _PyForIterCache *cache = (_PyForIterCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + next_instr--; + _Py_Specialize_ForIter(TOP(), next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(FOR_ITER, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); /* before: [iter]; after: [iter, iter()] *or* [] */ PyObject *iter = TOP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); @@ -3081,23 +3067,18 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL) { - if (!cframe.use_tracing) { - _PyCallCache *cache = (_PyCallCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - int is_meth = is_method(stack_pointer, oparg); - int nargs = oparg + is_meth; - PyObject *callable = PEEK(nargs + 1); - int err = _Py_Specialize_Call(callable, next_instr, nargs, - call_shape.kwnames); - if (err < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - } + _PyCallCache *cache = (_PyCallCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + int is_meth = is_method(stack_pointer, oparg); + int nargs = oparg + is_meth; + PyObject *callable = PEEK(nargs + 1); + next_instr--; + _Py_Specialize_Call(callable, next_instr, nargs, call_shape.kwnames); + DISPATCH_SAME_OPARG(); + } + STAT_INC(CALL, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); int total_args, is_meth; is_meth = is_method(stack_pointer, oparg); PyObject *function = PEEK(oparg + 1); @@ -3885,18 +3866,17 @@ dummy_func( // stack effect: (__0 -- ) inst(BINARY_OP) { - if (!cframe.use_tracing) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *lhs = SECOND(); - PyObject *rhs = TOP(); - next_instr--; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - } + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + PyObject *lhs = SECOND(); + PyObject *rhs = TOP(); + next_instr--; + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); + DISPATCH_SAME_OPARG(); + } + STAT_INC(BINARY_OP, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(BINARY_OP_GENERIC); } diff --git a/Python/ceval.c b/Python/ceval.c index 8971631a72dd54..ff491e47222a69 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -914,10 +914,22 @@ GETITEM(PyObject *v, Py_ssize_t i) { } #define ADAPTIVE_COUNTER_IS_ZERO(COUNTER) \ - ((COUNTER) < (1<> ADAPTIVE_BACKOFF_BITS) == 0) -#define DECREMENT_ADAPTIVE_COUNTER(COUNTER) \ - ((COUNTER) -= (1<> ADAPTIVE_BACKOFF_BITS) == ((1 << MAX_BACKOFF_VALUE) - 1)) + +#define DECREMENT_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + assert(!ADAPTIVE_COUNTER_IS_ZERO((COUNTER))); \ + (COUNTER) -= (1 << ADAPTIVE_BACKOFF_BITS); \ + } while (0); + +#define INCREMENT_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + assert(!ADAPTIVE_COUNTER_IS_MAX((COUNTER))); \ + (COUNTER) += (1 << ADAPTIVE_BACKOFF_BITS); \ + } while (0); static int trace_function_entry(PyThreadState *tstate, _PyInterpreterFrame *frame) @@ -1243,6 +1255,14 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PRE_DISPATCH_GOTO(); } opcode = _PyOpcode_Deopt[opcode]; + if (_PyOpcode_Caches[opcode]) { + _Py_CODEUNIT *counter = &next_instr[1]; + // The instruction is going to decrement the counter, so we need to + // increment it here to make sure it doesn't try to specialize: + if (!ADAPTIVE_COUNTER_IS_MAX(*counter)) { + INCREMENT_ADAPTIVE_COUNTER(*counter); + } + } DISPATCH_GOTO(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 23166085e60519..f3fab4e757f39f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -301,20 +301,17 @@ TARGET(BINARY_SUBSCR) { PREDICTED(BINARY_SUBSCR); - if (!cframe.use_tracing) { - _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *sub = TOP(); - PyObject *container = SECOND(); - next_instr--; - if (_Py_Specialize_BinarySubscr(container, sub, next_instr) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - } + _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + PyObject *sub = TOP(); + PyObject *container = SECOND(); + next_instr--; + _Py_Specialize_BinarySubscr(container, sub, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(BINARY_SUBSCR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); PyObject *sub = POP(); PyObject *container = TOP(); PyObject *res = PyObject_GetItem(container, sub); @@ -495,20 +492,17 @@ TARGET(STORE_SUBSCR) { PREDICTED(STORE_SUBSCR); - if (!cframe.use_tracing) { - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *sub = TOP(); - PyObject *container = SECOND(); - next_instr--; - if (_Py_Specialize_StoreSubscr(container, sub, next_instr) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - } + _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + PyObject *sub = TOP(); + PyObject *container = SECOND(); + next_instr--; + _Py_Specialize_StoreSubscr(container, sub, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(STORE_SUBSCR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); PyObject *sub = TOP(); PyObject *container = SECOND(); PyObject *v = THIRD(); @@ -1069,17 +1063,16 @@ TARGET(UNPACK_SEQUENCE) { PREDICTED(UNPACK_SEQUENCE); - if (!cframe.use_tracing) { - _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *seq = TOP(); - next_instr--; - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - } + _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + PyObject *seq = TOP(); + next_instr--; + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(UNPACK_SEQUENCE, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); PyObject *seq = POP(); PyObject **top = stack_pointer + oparg; if (!unpack_iterable(tstate, seq, oparg, -1, top)) { @@ -1149,20 +1142,19 @@ TARGET(STORE_ATTR) { PREDICTED(STORE_ATTR); - if (!cframe.use_tracing) { - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *owner = TOP(); - PyObject *name = GETITEM(names, oparg); - next_instr--; - if (_Py_Specialize_StoreAttr(owner, next_instr, name) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + PyObject *owner = TOP(); + PyObject *name = GETITEM(names, oparg); + next_instr--; + if (_Py_Specialize_StoreAttr(owner, next_instr, name)) { + goto error; } - STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DISPATCH_SAME_OPARG(); } + STAT_INC(STORE_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); PyObject *name = GETITEM(names, oparg); PyObject *owner = TOP(); PyObject *v = SECOND(); @@ -1280,19 +1272,16 @@ TARGET(LOAD_GLOBAL) { PREDICTED(LOAD_GLOBAL); - if (!cframe.use_tracing) { - _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *name = GETITEM(names, oparg>>1); - next_instr--; - if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - } + _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + PyObject *name = GETITEM(names, oparg>>1); + next_instr--; + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(LOAD_GLOBAL, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); int push_null = oparg & 1; PEEK(0) = NULL; PyObject *name = GETITEM(names, oparg>>1); @@ -1736,20 +1725,19 @@ TARGET(LOAD_ATTR) { PREDICTED(LOAD_ATTR); - if (!cframe.use_tracing) { - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *owner = TOP(); - PyObject *name = GETITEM(names, oparg>>1); - next_instr--; - if (_Py_Specialize_LoadAttr(owner, next_instr, name) < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + PyObject *owner = TOP(); + PyObject *name = GETITEM(names, oparg>>1); + next_instr--; + if (_Py_Specialize_LoadAttr(owner, next_instr, name)) { + goto error; } - STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DISPATCH_SAME_OPARG(); } + STAT_INC(LOAD_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); PyObject *name = GETITEM(names, oparg >> 1); PyObject *owner = TOP(); if (oparg & 1) { @@ -2129,18 +2117,17 @@ TARGET(COMPARE_OP) { PREDICTED(COMPARE_OP); - if (!cframe.use_tracing) { - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *right = TOP(); - PyObject *left = SECOND(); - next_instr--; - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - } + _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + PyObject *right = TOP(); + PyObject *left = SECOND(); + next_instr--; + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(COMPARE_OP, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(COMPARE_OP_GENERIC); } @@ -2624,16 +2611,15 @@ TARGET(FOR_ITER) { PREDICTED(FOR_ITER); - if (!cframe.use_tracing) { - _PyForIterCache *cache = (_PyForIterCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - _Py_Specialize_ForIter(TOP(), next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - } + _PyForIterCache *cache = (_PyForIterCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + next_instr--; + _Py_Specialize_ForIter(TOP(), next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(FOR_ITER, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); /* before: [iter]; after: [iter, iter()] *or* [] */ PyObject *iter = TOP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); @@ -2969,23 +2955,18 @@ TARGET(CALL) { PREDICTED(CALL); - if (!cframe.use_tracing) { - _PyCallCache *cache = (_PyCallCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; - int is_meth = is_method(stack_pointer, oparg); - int nargs = oparg + is_meth; - PyObject *callable = PEEK(nargs + 1); - int err = _Py_Specialize_Call(callable, next_instr, nargs, - call_shape.kwnames); - if (err < 0) { - goto error; - } - DISPATCH_SAME_OPARG(); - } - STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - } + _PyCallCache *cache = (_PyCallCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + int is_meth = is_method(stack_pointer, oparg); + int nargs = oparg + is_meth; + PyObject *callable = PEEK(nargs + 1); + next_instr--; + _Py_Specialize_Call(callable, next_instr, nargs, call_shape.kwnames); + DISPATCH_SAME_OPARG(); + } + STAT_INC(CALL, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); int total_args, is_meth; is_meth = is_method(stack_pointer, oparg); PyObject *function = PEEK(oparg + 1); @@ -3774,18 +3755,17 @@ TARGET(BINARY_OP) { PREDICTED(BINARY_OP); - if (!cframe.use_tracing) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - PyObject *lhs = SECOND(); - PyObject *rhs = TOP(); - next_instr--; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); - } + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + PyObject *lhs = SECOND(); + PyObject *rhs = TOP(); + next_instr--; + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); + DISPATCH_SAME_OPARG(); + } + STAT_INC(BINARY_OP, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(BINARY_OP_GENERIC); } From e69e254e15215460d728b1e5b076e7305f868df0 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 21 Oct 2022 03:40:24 -0700 Subject: [PATCH 19/21] Remove error checking from many specializations --- Include/internal/pycore_code.h | 13 ++++++++----- Python/specialize.c | 18 +++++++----------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 0fe9ab8c74b061..b0703073f1d14b 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -217,11 +217,14 @@ extern int _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name); extern int _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name); -extern int _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name); -extern int _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr); -extern int _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr); -extern int _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, - int nargs, PyObject *kwnames); +extern void _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, + _Py_CODEUNIT *instr, PyObject *name); +extern void _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, + _Py_CODEUNIT *instr); +extern void _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, + _Py_CODEUNIT *instr); +extern void _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, + int nargs, PyObject *kwnames); extern void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, int oparg, PyObject **locals); extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, diff --git a/Python/specialize.c b/Python/specialize.c index f9fe2b2babe125..61d7a5de0a7ebf 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1082,7 +1082,7 @@ PyObject *descr, DescriptorClassification kind) return 0; } -int +void _Py_Specialize_LoadGlobal( PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name) @@ -1157,12 +1157,11 @@ _Py_Specialize_LoadGlobal( assert(!PyErr_Occurred()); _Py_SET_OPCODE(*instr, LOAD_GLOBAL); cache->counter = adaptive_counter_backoff(cache->counter); - return 0; + return; success: STAT_INC(LOAD_GLOBAL, success); assert(!PyErr_Occurred()); cache->counter = adaptive_counter_cooldown(); - return 0; } #ifdef Py_STATS @@ -1250,7 +1249,7 @@ function_get_version(PyObject *o, int opcode) return version; } -int +void _Py_Specialize_BinarySubscr( PyObject *container, PyObject *sub, _Py_CODEUNIT *instr) { @@ -1318,15 +1317,14 @@ _Py_Specialize_BinarySubscr( assert(!PyErr_Occurred()); _Py_SET_OPCODE(*instr, BINARY_SUBSCR); cache->counter = adaptive_counter_backoff(cache->counter); - return 0; + return; success: STAT_INC(BINARY_SUBSCR, success); assert(!PyErr_Occurred()); cache->counter = adaptive_counter_cooldown(); - return 0; } -int +void _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr) { _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)(instr + 1); @@ -1423,12 +1421,11 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins assert(!PyErr_Occurred()); _Py_SET_OPCODE(*instr, STORE_SUBSCR); cache->counter = adaptive_counter_backoff(cache->counter); - return 0; + return; success: STAT_INC(STORE_SUBSCR, success); assert(!PyErr_Occurred()); cache->counter = adaptive_counter_cooldown(); - return 0; } static int @@ -1697,7 +1694,7 @@ call_fail_kind(PyObject *callable) /* TODO: - Specialize calling classes. */ -int +void _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames) { @@ -1743,7 +1740,6 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, assert(!PyErr_Occurred()); cache->counter = adaptive_counter_cooldown(); } - return 0; } #ifdef Py_STATS From 513aaab954e1a09b01d353d70632599303334463 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 21 Oct 2022 04:19:08 -0700 Subject: [PATCH 20/21] Fix macro --- Python/ceval.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ceval.c b/Python/ceval.c index ff491e47222a69..fbb025c0a355c3 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1246,7 +1246,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int while (opcode == EXTENDED_ARG) { // CPython hasn't ever traced the instruction after an EXTENDED_ARG. // Inline the EXTENDED_ARG here, so we can avoid branching there: - INSTRUCTION_START(); + INSTRUCTION_START(EXTENDED_ARG); opcode = _Py_OPCODE(*next_instr); oparg = oparg << 8 | _Py_OPARG(*next_instr); // Make sure the next instruction isn't a RESUME, since that needs From 1cd6d66d1f2754920b58f1f4cae2de906c19926e Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 21 Oct 2022 14:36:37 -0700 Subject: [PATCH 21/21] Make sure stats overhead is disabled be default --- Python/ceval.c | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 460f5690aab462..af706e1a02140c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -846,22 +846,30 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define GO_TO_INSTRUCTION(op) goto PREDICT_ID(op) - -#define DEOPT_IF(COND, INSTNAME) \ - if (COND) { \ - /* This is only a single jump on release builds! */ \ +#ifdef Py_STATS +#define UPDATE_MISS_STATS(INSTNAME) \ + do { \ STAT_INC(opcode, miss); \ - STAT_INC(INSTNAME, miss); \ + STAT_INC((INSTNAME), miss); \ /* The counter is always the first cache entry: */ \ if (ADAPTIVE_COUNTER_IS_ZERO(*next_instr)) { \ - STAT_INC(INSTNAME, deopt); \ + STAT_INC((INSTNAME), deopt); \ } \ else { \ /* This is about to be (incorrectly) incremented: */ \ - STAT_DEC(INSTNAME, deferred); \ + STAT_DEC((INSTNAME), deferred); \ } \ - assert(_PyOpcode_Deopt[opcode] == INSTNAME); \ - GO_TO_INSTRUCTION(INSTNAME); \ + } while (0) +#else +#define UPDATE_MISS_STATS(INSTNAME) ((void)0) +#endif + +#define DEOPT_IF(COND, INSTNAME) \ + if ((COND)) { \ + /* This is only a single jump on release builds! */ \ + UPDATE_MISS_STATS((INSTNAME)); \ + assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ + GO_TO_INSTRUCTION(INSTNAME); \ }