diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 46c9986b500c80..e096504b7a5d48 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -104,7 +104,8 @@ const uint8_t _PyOpcode_Deopt[256] = { [DICT_MERGE] = DICT_MERGE, [DICT_UPDATE] = DICT_UPDATE, [END_ASYNC_FOR] = END_ASYNC_FOR, - [EXTENDED_ARG] = EXTENDED_ARG, + [EXTENDED_ARG] = EXTENDED_ARG_TRACE, + [EXTENDED_ARG_TRACE] = EXTENDED_ARG_TRACE, [FORMAT_VALUE] = FORMAT_VALUE, [FOR_ITER] = FOR_ITER, [GET_AITER] = GET_AITER, @@ -381,44 +382,44 @@ static const char *const _PyOpcode_OpName[256] = { [JUMP_BACKWARD] = "JUMP_BACKWARD", [PRECALL_PYFUNC] = "PRECALL_PYFUNC", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [RESUME_QUICK] = "RESUME_QUICK", + [EXTENDED_ARG_TRACE] = "EXTENDED_ARG_TRACE", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", [MAP_ADD] = "MAP_ADD", [LOAD_CLASSDEREF] = "LOAD_CLASSDEREF", [COPY_FREE_VARS] = "COPY_FREE_VARS", - [STORE_ATTR_ADAPTIVE] = "STORE_ATTR_ADAPTIVE", + [RESUME_QUICK] = "RESUME_QUICK", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", + [STORE_ATTR_ADAPTIVE] = "STORE_ATTR_ADAPTIVE", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", - [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [LOAD_METHOD] = "LOAD_METHOD", - [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", + [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", [PRECALL] = "PRECALL", + [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [STORE_SUBSCR_ADAPTIVE] = "STORE_SUBSCR_ADAPTIVE", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", - [UNPACK_SEQUENCE_ADAPTIVE] = "UNPACK_SEQUENCE_ADAPTIVE", [CALL] = "CALL", [KW_NAMES] = "KW_NAMES", [POP_JUMP_BACKWARD_IF_NOT_NONE] = "POP_JUMP_BACKWARD_IF_NOT_NONE", [POP_JUMP_BACKWARD_IF_NONE] = "POP_JUMP_BACKWARD_IF_NONE", [POP_JUMP_BACKWARD_IF_FALSE] = "POP_JUMP_BACKWARD_IF_FALSE", [POP_JUMP_BACKWARD_IF_TRUE] = "POP_JUMP_BACKWARD_IF_TRUE", + [UNPACK_SEQUENCE_ADAPTIVE] = "UNPACK_SEQUENCE_ADAPTIVE", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [180] = "<180>", [181] = "<181>", [182] = "<182>", [183] = "<183>", @@ -498,7 +499,6 @@ static const char *const _PyOpcode_OpName[256] = { #endif #define EXTRA_CASES \ - case 180: \ case 181: \ case 182: \ case 183: \ diff --git a/Include/opcode.h b/Include/opcode.h index 399847d4ba050d..be7a16a6586402 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -96,6 +96,7 @@ extern "C" { #define DELETE_DEREF 139 #define JUMP_BACKWARD 140 #define CALL_FUNCTION_EX 142 +#define EXTENDED_ARG_TRACE 143 #define EXTENDED_ARG 144 #define LIST_APPEND 145 #define SET_ADD 146 @@ -175,20 +176,20 @@ extern "C" { #define PRECALL_NO_KW_TUPLE_1 121 #define PRECALL_NO_KW_TYPE_1 127 #define PRECALL_PYFUNC 141 -#define RESUME_QUICK 143 -#define STORE_ATTR_ADAPTIVE 150 -#define STORE_ATTR_INSTANCE_VALUE 153 -#define STORE_ATTR_SLOT 154 -#define STORE_ATTR_WITH_HINT 158 -#define STORE_FAST__LOAD_FAST 159 -#define STORE_FAST__STORE_FAST 161 -#define STORE_SUBSCR_ADAPTIVE 167 -#define STORE_SUBSCR_DICT 168 -#define STORE_SUBSCR_LIST_INT 169 -#define UNPACK_SEQUENCE_ADAPTIVE 170 -#define UNPACK_SEQUENCE_LIST 177 -#define UNPACK_SEQUENCE_TUPLE 178 -#define UNPACK_SEQUENCE_TWO_TUPLE 179 +#define RESUME_QUICK 150 +#define STORE_ATTR_ADAPTIVE 153 +#define STORE_ATTR_INSTANCE_VALUE 154 +#define STORE_ATTR_SLOT 158 +#define STORE_ATTR_WITH_HINT 159 +#define STORE_FAST__LOAD_FAST 161 +#define STORE_FAST__STORE_FAST 167 +#define STORE_SUBSCR_ADAPTIVE 168 +#define STORE_SUBSCR_DICT 169 +#define STORE_SUBSCR_LIST_INT 170 +#define UNPACK_SEQUENCE_ADAPTIVE 177 +#define UNPACK_SEQUENCE_LIST 178 +#define UNPACK_SEQUENCE_TUPLE 179 +#define UNPACK_SEQUENCE_TWO_TUPLE 180 #define DO_TRACING 255 #define HAS_CONST(op) (false\ diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 71e1e24b51e22d..5f67226adfc522 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -416,7 +416,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3493).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3494).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 9ee06831c37687..6030324ff1d569 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -169,8 +169,10 @@ def jabs_op(name, op, entries=0): def_op('CALL_FUNCTION_EX', 142) # Flags +def_op('EXTENDED_ARG_TRACE', 143) def_op('EXTENDED_ARG', 144) EXTENDED_ARG = 144 + def_op('LIST_APPEND', 145) def_op('SET_ADD', 146) def_op('MAP_ADD', 147) diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 2a4c0d2eeb656a..6f136292fd5239 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -18,7 +18,10 @@ def test_stack_effect(self): self.assertRaises(ValueError, stack_effect, dis.opmap['BUILD_SLICE']) self.assertRaises(ValueError, stack_effect, dis.opmap['POP_TOP'], 0) # All defined opcodes - for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): + opnames = {name: op for name, op in dis.opmap.items() + if name not in dis.deoptmap + and name != 'EXTENDED_ARG_TRACE'} + for name, code in opnames.items(): with self.subTest(opname=name): if code < dis.HAVE_ARGUMENT: stack_effect(code) @@ -31,6 +34,8 @@ def test_stack_effect(self): with self.subTest(opcode=code): self.assertRaises(ValueError, stack_effect, code) self.assertRaises(ValueError, stack_effect, code, 0) + self.assertRaises(ValueError, stack_effect, + dis.opmap["EXTENDED_ARG_TRACE"]) def test_stack_effect_jump(self): JUMP_IF_TRUE_OR_POP = dis.opmap['JUMP_IF_TRUE_OR_POP'] @@ -47,7 +52,10 @@ def test_stack_effect_jump(self): self.assertEqual(stack_effect(JUMP_FORWARD, 0, jump=False), 0) # All defined opcodes has_jump = dis.hasjabs + dis.hasjrel - for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): + opnames = {name: op for name, op in dis.opmap.items() + if name not in dis.deoptmap + and name != 'EXTENDED_ARG_TRACE'} + for name, code in opnames.items(): with self.subTest(opname=name): if code < dis.HAVE_ARGUMENT: common = stack_effect(code) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 85d6bdf7221e39..b1c8f6f80af835 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2432,5 +2432,47 @@ def gen(): output.append(5) +class TestExtendedArgs(unittest.TestCase): + + def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) + sys.settrace(None) + + def count_traces(self, func): + # warmup + for _ in range(20): + func() + + counts = {"call": 0, "line": 0, "return": 0} + def trace(frame, event, arg): + counts[event] += 1 + return trace + + sys.settrace(trace) + func() + sys.settrace(None) + + return counts + + def test_trace_unpack_long_sequence(self): + ns = {} + code = "def f():\n (" + "y,\n "*300 + ") = range(300)" + exec(code, ns) + counts = self.count_traces(ns["f"]) + self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1}) + + def test_trace_lots_of_globals(self): + code = """if 1: + def f(): + return ( + {} + ) + """.format("\n+\n".join(f"var{i}\n" for i in range(1000))) + ns = {f"var{i}": i for i in range(1000)} + exec(code, ns) + counts = self.count_traces(ns["f"]) + self.assertEqual(counts, {'call': 1, 'line': 2000, 'return': 1}) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-04-26-05-05-32.gh-issue-91869.ELbTXl.rst b/Misc/NEWS.d/next/Core and Builtins/2022-04-26-05-05-32.gh-issue-91869.ELbTXl.rst new file mode 100644 index 00000000000000..05b84be23483be --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-04-26-05-05-32.gh-issue-91869.ELbTXl.rst @@ -0,0 +1 @@ +Fix an issue where specialized opcodes with extended arguments could produce incorrect tracing output or lead to assertion failures. diff --git a/Python/ceval.c b/Python/ceval.c index 6e7a2483112d7b..20a32440a83fc8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -5630,6 +5630,15 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int NOTRACE_DISPATCH_SAME_OPARG(); } + TARGET(EXTENDED_ARG_TRACE) { + assert(oparg); + oparg <<= 8; + oparg |= _Py_OPARG(*next_instr); + opcode = _PyOpcode_Deopt[_Py_OPCODE(*next_instr)]; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); + } + TARGET(CACHE) { Py_UNREACHABLE(); } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 5268c4f2a27750..317fc3b09805e0 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -142,40 +142,41 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_BACKWARD, &&TARGET_PRECALL_PYFUNC, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_RESUME_QUICK, + &&TARGET_EXTENDED_ARG_TRACE, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, &&TARGET_MAP_ADD, &&TARGET_LOAD_CLASSDEREF, &&TARGET_COPY_FREE_VARS, - &&TARGET_STORE_ATTR_ADAPTIVE, + &&TARGET_RESUME_QUICK, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, + &&TARGET_STORE_ATTR_ADAPTIVE, &&TARGET_STORE_ATTR_INSTANCE_VALUE, - &&TARGET_STORE_ATTR_SLOT, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, - &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_LOAD_METHOD, - &&TARGET_STORE_FAST__STORE_FAST, + &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, &&TARGET_PRECALL, + &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_ADAPTIVE, &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, - &&TARGET_UNPACK_SEQUENCE_ADAPTIVE, &&TARGET_CALL, &&TARGET_KW_NAMES, &&TARGET_POP_JUMP_BACKWARD_IF_NOT_NONE, &&TARGET_POP_JUMP_BACKWARD_IF_NONE, &&TARGET_POP_JUMP_BACKWARD_IF_FALSE, &&TARGET_POP_JUMP_BACKWARD_IF_TRUE, + &&TARGET_UNPACK_SEQUENCE_ADAPTIVE, &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, @@ -253,6 +254,5 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_DO_TRACING }; diff --git a/Tools/scripts/generate_opcode_h.py b/Tools/scripts/generate_opcode_h.py index 6a04297879f2cb..63bdb70cb75e87 100644 --- a/Tools/scripts/generate_opcode_h.py +++ b/Tools/scripts/generate_opcode_h.py @@ -133,6 +133,7 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna for basic, family in opcode["_specializations"].items(): for specialized in family: deoptcodes[specialized] = basic + deoptcodes["EXTENDED_ARG"] = "EXTENDED_ARG_TRACE" iobj.write("\nconst uint8_t _PyOpcode_Deopt[256] = {\n") for opt, deopt in sorted(deoptcodes.items()): iobj.write(f" [{opt}] = {deopt},\n")