From 6e092c15c454562b296508eb2218d166d9bd3269 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 2 Sep 2022 11:28:21 +0100 Subject: [PATCH 1/9] Decouple Python recursion limit from C stack checking. --- Include/cpython/pystate.h | 9 ++- Include/internal/pycore_ast_state.h | 2 - Include/internal/pycore_ceval.h | 40 ++++++---- Include/internal/pycore_compile.h | 3 - Include/internal/pycore_runtime_init.h | 2 +- Include/internal/pycore_symtable.h | 2 - Lib/test/list_tests.py | 2 +- Lib/test/test_dict.py | 2 +- Lib/test/test_exception_group.py | 2 +- Lib/test/test_isinstance.py | 12 +-- Modules/_testinternalcapi.c | 2 +- Objects/object.c | 4 +- Parser/asdl_c.py | 36 +-------- Python/Python-ast.c | 106 +++++++------------------ Python/ast.c | 46 +++-------- Python/ast_opt.c | 38 ++------- Python/ceval.c | 76 +++++++++--------- Python/errors.c | 6 +- Python/pystate.c | 5 +- Python/symtable.c | 46 ++--------- Python/sysmodule.c | 2 +- 21 files changed, 145 insertions(+), 298 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index cc3c3eae941933..50f4e4abfc3908 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -95,9 +95,12 @@ struct _ts { /* Was this thread state statically allocated? */ int _static; - int recursion_remaining; - int recursion_limit; - int recursion_headroom; /* Allow 50 more calls to handle any errors. */ + int py_recursion_remaining; + int py_recursion_limit; + int py_recursion_headroom; /* Allow 50 more calls to handle any errors. */ + + int c_recursion_remaining; + int c_recursion_headroom; /* Allow 50 more calls to handle any errors. */ /* 'tracing' keeps track of the execution depth when tracing/profiling. This is to prevent the actual trace/profile code from being recorded in diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index f15b4905eed14b..da78bba3b69bdf 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -12,8 +12,6 @@ extern "C" { struct ast_state { int initialized; - int recursion_depth; - int recursion_limit; PyObject *AST_type; PyObject *Add_singleton; PyObject *Add_type; diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 4914948c6ca744..75352306b96cf2 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -114,26 +114,31 @@ extern void _PyEval_DeactivateOpCache(void); /* --- _Py_EnterRecursiveCall() ----------------------------------------- */ -#ifdef USE_STACKCHECK -/* With USE_STACKCHECK macro defined, trigger stack checks in - _Py_CheckRecursiveCall() on every 64th call to _Py_EnterRecursiveCall. */ -static inline int _Py_MakeRecCheck(PyThreadState *tstate) { - return (tstate->recursion_remaining-- <= 0 - || (tstate->recursion_remaining & 63) == 0); -} -#else -static inline int _Py_MakeRecCheck(PyThreadState *tstate) { - return tstate->recursion_remaining-- <= 0; -} -#endif - PyAPI_FUNC(int) _Py_CheckRecursiveCall( PyThreadState *tstate, const char *where); +PyAPI_FUNC(int) _Py_CheckRecursiveCallN(PyThreadState *tstate, int n, + const char *where); + static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate, const char *where) { - return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where)); + int remaining = tstate->c_recursion_remaining; + if (remaining > 0) { + tstate->c_recursion_remaining = remaining - 1; + return 0; + } + return _Py_CheckRecursiveCallN(tstate, 1, where); +} + +static inline int _Py_EnterRecursiveCallN(PyThreadState *tstate, int n, + const char *where) { + int remaining = tstate->c_recursion_remaining; + if (remaining >= n) { + tstate->c_recursion_remaining = remaining - n; + return 0; + } + return _Py_CheckRecursiveCallN(tstate, n, where); } static inline int _Py_EnterRecursiveCall(const char *where) { @@ -142,7 +147,11 @@ static inline int _Py_EnterRecursiveCall(const char *where) { } static inline void _Py_LeaveRecursiveCallTstate(PyThreadState *tstate) { - tstate->recursion_remaining++; + tstate->c_recursion_remaining++; +} + +static inline void _Py_LeaveRecursiveCallN(PyThreadState *tstate, int n) { + tstate->c_recursion_remaining += n; } static inline void _Py_LeaveRecursiveCall(void) { @@ -156,6 +165,7 @@ extern PyObject* _Py_MakeCoro(PyFunctionObject *func); extern int _Py_HandlePending(PyThreadState *tstate); +#define C_RECURSION_LIMT 3000 #ifdef __cplusplus } diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 1a628a08ca4ebf..98eaa2064a3089 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -28,9 +28,6 @@ extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); typedef struct { int optimize; int ff_features; - - int recursion_depth; /* current recursion depth */ - int recursion_limit; /* recursion limit */ } _PyASTOptimizeState; extern int _PyAST_Optimize( diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index c14d25941348bf..f4b4233f63e0a0 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -68,7 +68,7 @@ extern "C" { #define _PyThreadState_INIT \ { \ ._static = 1, \ - .recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \ + .py_recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \ .context_ver = 1, \ } diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 2d64aba22ff905..3d9bf66bfbc91c 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -37,8 +37,6 @@ struct symtable { PyObject *st_private; /* name of current class or NULL */ PyFutureFeatures *st_future; /* module's future features that affect the symbol table */ - int recursion_depth; /* current recursion depth */ - int recursion_limit; /* recursion limit */ }; typedef struct _symtable_entry { diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index fe3ee80b8d461f..98584db17e3912 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -61,7 +61,7 @@ def test_repr(self): def test_repr_deep(self): a = self.type2test([]) - for i in range(sys.getrecursionlimit() + 100): + for i in range(4000): a = self.type2test([a]) self.assertRaises(RecursionError, repr, a) diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 5b8baaf9e6e280..7139a4996c76be 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -596,7 +596,7 @@ def __repr__(self): def test_repr_deep(self): d = {} - for i in range(sys.getrecursionlimit() + 100): + for i in range(4000): d = {1: d} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index aa28e16bedfa68..4e10a0eda4234e 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -405,7 +405,7 @@ def test_basics_split_by_predicate__match(self): class DeepRecursionInSplitAndSubgroup(unittest.TestCase): def make_deep_eg(self): e = TypeError(1) - for i in range(2000): + for i in range(4000): e = ExceptionGroup('eg', [e]) return e diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index a0974640bc1146..4d4e9ee6a6b569 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -8,7 +8,7 @@ from test import support - + class TestIsInstanceExceptions(unittest.TestCase): # Test to make sure that an AttributeError when accessing the instance's # class's bases is masked. This was actually a bug in Python 2.2 and @@ -97,7 +97,7 @@ def getclass(self): class D: pass self.assertRaises(RuntimeError, isinstance, c, D) - + # These tests are similar to above, but tickle certain code paths in # issubclass() instead of isinstance() -- really PyObject_IsSubclass() # vs. PyObject_IsInstance(). @@ -147,7 +147,7 @@ def getbases(self): self.assertRaises(TypeError, issubclass, B, C()) - + # meta classes for creating abstract classes and instances class AbstractClass(object): def __init__(self, bases): @@ -179,7 +179,7 @@ class Super: class Child(Super): pass - + class TestIsInstanceIsSubclass(unittest.TestCase): # Tests to ensure that isinstance and issubclass work on abstract # classes and instances. Before the 2.2 release, TypeErrors were @@ -353,10 +353,10 @@ def blowstack(fxn, arg, compare_to): # Make sure that calling isinstance with a deeply nested tuple for its # argument will raise RecursionError eventually. tuple_arg = (compare_to,) - for cnt in range(sys.getrecursionlimit()+5): + for cnt in range(4000): tuple_arg = (tuple_arg,) fxn(arg, tuple_arg) - + if __name__ == '__main__': unittest.main() diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 02a061b84f85a3..54306aee0baf2a 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -46,7 +46,7 @@ get_recursion_depth(PyObject *self, PyObject *Py_UNUSED(args)) /* subtract one to ignore the frame of the get_recursion_depth() call */ - return PyLong_FromLong(tstate->recursion_limit - tstate->recursion_remaining - 1); + return PyLong_FromLong(tstate->py_recursion_limit - tstate->py_recursion_remaining - 1); } diff --git a/Objects/object.c b/Objects/object.c index 9bbe0eef308fba..2943ef074ac276 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -421,12 +421,12 @@ PyObject_Repr(PyObject *v) /* It is possible for a type to have a tp_repr representation that loops infinitely. */ - if (_Py_EnterRecursiveCallTstate(tstate, + if (_Py_EnterRecursiveCallN(tstate, 2, " while getting the repr of an object")) { return NULL; } res = (*Py_TYPE(v)->tp_repr)(v); - _Py_LeaveRecursiveCallTstate(tstate); + _Py_LeaveRecursiveCallN(tstate, 2); if (res == NULL) { return NULL; diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 13dd44ca0cdc3f..5cf011275f4f79 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1112,8 +1112,6 @@ def visitModule(self, mod): for dfn in mod.dfns: self.visit(dfn) self.file.write(textwrap.dedent(''' - state->recursion_depth = 0; - state->recursion_limit = 0; state->initialized = 1; return 1; } @@ -1261,14 +1259,12 @@ def func_begin(self, name): self.emit('if (!o) {', 1) self.emit("Py_RETURN_NONE;", 2) self.emit("}", 1) - self.emit("if (++state->recursion_depth > state->recursion_limit) {", 1) - self.emit("PyErr_SetString(PyExc_RecursionError,", 2) - self.emit('"maximum recursion depth exceeded during ast construction");', 3) + self.emit('if (_Py_EnterRecursiveCall("during ast construction")) {', 1) self.emit("return 0;", 2) self.emit("}", 1) def func_end(self): - self.emit("state->recursion_depth--;", 1) + self.emit("_Py_LeaveRecursiveCall();", 1) self.emit("return result;", 1) self.emit("failed:", 0) self.emit("Py_XDECREF(value);", 1) @@ -1380,31 +1376,7 @@ class PartingShots(StaticVisitor): return NULL; } - int recursion_limit = Py_GetRecursionLimit(); - int starting_recursion_depth; - /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 3; - PyThreadState *tstate = _PyThreadState_GET(); - if (!tstate) { - return 0; - } - state->recursion_limit = (recursion_limit < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_limit * COMPILER_STACK_FRAME_SCALE : recursion_limit; - int recursion_depth = tstate->recursion_limit - tstate->recursion_remaining; - starting_recursion_depth = (recursion_depth < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_depth * COMPILER_STACK_FRAME_SCALE : recursion_depth; - state->recursion_depth = starting_recursion_depth; - - PyObject *result = ast2obj_mod(state, t); - - /* Check that the recursion depth counting balanced correctly */ - if (result && state->recursion_depth != starting_recursion_depth) { - PyErr_Format(PyExc_SystemError, - "AST constructor recursion depth mismatch (before=%d, after=%d)", - starting_recursion_depth, state->recursion_depth); - return 0; - } - return result; + return ast2obj_mod(state, t); } /* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */ @@ -1470,8 +1442,6 @@ def visit(self, object): def generate_ast_state(module_state, f): f.write('struct ast_state {\n') f.write(' int initialized;\n') - f.write(' int recursion_depth;\n') - f.write(' int recursion_limit;\n') for s in module_state: f.write(' PyObject *' + s + ';\n') f.write('};') diff --git a/Python/Python-ast.c b/Python/Python-ast.c index f485af675ccff7..a7d680a66ad2e7 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -1851,8 +1851,6 @@ init_types(struct ast_state *state) "TypeIgnore(int lineno, string tag)"); if (!state->TypeIgnore_type) return 0; - state->recursion_depth = 0; - state->recursion_limit = 0; state->initialized = 1; return 1; } @@ -3612,9 +3610,7 @@ ast2obj_mod(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (_Py_EnterRecursiveCall("during ast construction")) { return 0; } switch (o->kind) { @@ -3672,7 +3668,7 @@ ast2obj_mod(struct ast_state *state, void* _o) Py_DECREF(value); break; } - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return result; failed: Py_XDECREF(value); @@ -3689,9 +3685,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (_Py_EnterRecursiveCall("during ast construction")) { return 0; } switch (o->kind) { @@ -4237,7 +4231,7 @@ ast2obj_stmt(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return result; failed: Py_XDECREF(value); @@ -4254,9 +4248,7 @@ ast2obj_expr(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (_Py_EnterRecursiveCall("during ast construction")) { return 0; } switch (o->kind) { @@ -4720,7 +4712,7 @@ ast2obj_expr(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return result; failed: Py_XDECREF(value); @@ -4863,9 +4855,7 @@ ast2obj_comprehension(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (_Py_EnterRecursiveCall("during ast construction")) { return 0; } tp = (PyTypeObject *)state->comprehension_type; @@ -4891,7 +4881,7 @@ ast2obj_comprehension(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->is_async, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return result; failed: Py_XDECREF(value); @@ -4908,9 +4898,7 @@ ast2obj_excepthandler(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (_Py_EnterRecursiveCall("during ast construction")) { return 0; } switch (o->kind) { @@ -4956,7 +4944,7 @@ ast2obj_excepthandler(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return result; failed: Py_XDECREF(value); @@ -4973,9 +4961,7 @@ ast2obj_arguments(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (_Py_EnterRecursiveCall("during ast construction")) { return 0; } tp = (PyTypeObject *)state->arguments_type; @@ -5016,7 +5002,7 @@ ast2obj_arguments(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->defaults, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return result; failed: Py_XDECREF(value); @@ -5033,9 +5019,7 @@ ast2obj_arg(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (_Py_EnterRecursiveCall("during ast construction")) { return 0; } tp = (PyTypeObject *)state->arg_type; @@ -5076,7 +5060,7 @@ ast2obj_arg(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return result; failed: Py_XDECREF(value); @@ -5093,9 +5077,7 @@ ast2obj_keyword(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (_Py_EnterRecursiveCall("during ast construction")) { return 0; } tp = (PyTypeObject *)state->keyword_type; @@ -5131,7 +5113,7 @@ ast2obj_keyword(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return result; failed: Py_XDECREF(value); @@ -5148,9 +5130,7 @@ ast2obj_alias(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (_Py_EnterRecursiveCall("during ast construction")) { return 0; } tp = (PyTypeObject *)state->alias_type; @@ -5186,7 +5166,7 @@ ast2obj_alias(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return result; failed: Py_XDECREF(value); @@ -5203,9 +5183,7 @@ ast2obj_withitem(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (_Py_EnterRecursiveCall("during ast construction")) { return 0; } tp = (PyTypeObject *)state->withitem_type; @@ -5221,7 +5199,7 @@ ast2obj_withitem(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->optional_vars, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return result; failed: Py_XDECREF(value); @@ -5238,9 +5216,7 @@ ast2obj_match_case(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (_Py_EnterRecursiveCall("during ast construction")) { return 0; } tp = (PyTypeObject *)state->match_case_type; @@ -5261,7 +5237,7 @@ ast2obj_match_case(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->body, value) == -1) goto failed; Py_DECREF(value); - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return result; failed: Py_XDECREF(value); @@ -5278,9 +5254,7 @@ ast2obj_pattern(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (_Py_EnterRecursiveCall("during ast construction")) { return 0; } switch (o->kind) { @@ -5422,7 +5396,7 @@ ast2obj_pattern(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) goto failed; Py_DECREF(value); - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return result; failed: Py_XDECREF(value); @@ -5439,9 +5413,7 @@ ast2obj_type_ignore(struct ast_state *state, void* _o) if (!o) { Py_RETURN_NONE; } - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during ast construction"); + if (_Py_EnterRecursiveCall("during ast construction")) { return 0; } switch (o->kind) { @@ -5461,7 +5433,7 @@ ast2obj_type_ignore(struct ast_state *state, void* _o) Py_DECREF(value); break; } - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return result; failed: Py_XDECREF(value); @@ -12315,31 +12287,7 @@ PyObject* PyAST_mod2obj(mod_ty t) return NULL; } - int recursion_limit = Py_GetRecursionLimit(); - int starting_recursion_depth; - /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 3; - PyThreadState *tstate = _PyThreadState_GET(); - if (!tstate) { - return 0; - } - state->recursion_limit = (recursion_limit < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_limit * COMPILER_STACK_FRAME_SCALE : recursion_limit; - int recursion_depth = tstate->recursion_limit - tstate->recursion_remaining; - starting_recursion_depth = (recursion_depth < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_depth * COMPILER_STACK_FRAME_SCALE : recursion_depth; - state->recursion_depth = starting_recursion_depth; - - PyObject *result = ast2obj_mod(state, t); - - /* Check that the recursion depth counting balanced correctly */ - if (result && state->recursion_depth != starting_recursion_depth) { - PyErr_Format(PyExc_SystemError, - "AST constructor recursion depth mismatch (before=%d, after=%d)", - starting_recursion_depth, state->recursion_depth); - return 0; - } - return result; + return ast2obj_mod(state, t); } /* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */ diff --git a/Python/ast.c b/Python/ast.c index a0321b58ba8cff..014b36322d8436 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -4,14 +4,14 @@ */ #include "Python.h" #include "pycore_ast.h" // asdl_stmt_seq +#include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_pystate.h" // _PyThreadState_GET() #include #include struct validator { - int recursion_depth; /* current recursion depth */ - int recursion_limit; /* recursion limit */ + int placeholder; }; static int validate_stmts(struct validator *, asdl_stmt_seq *); @@ -161,9 +161,7 @@ validate_constant(struct validator *state, PyObject *value) return 1; if (PyTuple_CheckExact(value) || PyFrozenSet_CheckExact(value)) { - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); + if (_Py_EnterRecursiveCall("during compilation")) { return 0; } @@ -190,7 +188,7 @@ validate_constant(struct validator *state, PyObject *value) } Py_DECREF(it); - --state->recursion_depth; + _Py_LeaveRecursiveCall(); return 1; } @@ -207,9 +205,8 @@ validate_expr(struct validator *state, expr_ty exp, expr_context_ty ctx) { VALIDATE_POSITIONS(exp); int ret = -1; - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); + + if (_Py_EnterRecursiveCall("during compilation")) { return 0; } int check_ctx = 1; @@ -387,7 +384,7 @@ validate_expr(struct validator *state, expr_ty exp, expr_context_ty ctx) PyErr_SetString(PyExc_SystemError, "unexpected expression"); ret = 0; } - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return ret; } @@ -530,9 +527,7 @@ validate_pattern(struct validator *state, pattern_ty p, int star_ok) { VALIDATE_POSITIONS(p); int ret = -1; - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); + if (_Py_EnterRecursiveCall("during compilation")) { return 0; } switch (p->kind) { @@ -668,7 +663,7 @@ validate_pattern(struct validator *state, pattern_ty p, int star_ok) PyErr_SetString(PyExc_SystemError, "unexpected pattern"); ret = 0; } - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return ret; } @@ -701,9 +696,7 @@ validate_stmt(struct validator *state, stmt_ty stmt) VALIDATE_POSITIONS(stmt); int ret = -1; Py_ssize_t i; - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); + if (_Py_EnterRecursiveCall("during compilation")) { return 0; } switch (stmt->kind) { @@ -909,7 +902,7 @@ validate_stmt(struct validator *state, stmt_ty stmt) PyErr_SetString(PyExc_SystemError, "unexpected statement"); ret = 0; } - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return ret; } @@ -975,21 +968,12 @@ _PyAST_Validate(mod_ty mod) int res = -1; struct validator state; PyThreadState *tstate; - int recursion_limit = Py_GetRecursionLimit(); - int starting_recursion_depth; /* Setup recursion depth check counters */ tstate = _PyThreadState_GET(); if (!tstate) { return 0; } - /* Be careful here to prevent overflow. */ - int recursion_depth = tstate->recursion_limit - tstate->recursion_remaining; - starting_recursion_depth = (recursion_depth< INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_depth * COMPILER_STACK_FRAME_SCALE : recursion_depth; - state.recursion_depth = starting_recursion_depth; - state.recursion_limit = (recursion_limit < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_limit * COMPILER_STACK_FRAME_SCALE : recursion_limit; switch (mod->kind) { case Module_kind: @@ -1012,14 +996,6 @@ _PyAST_Validate(mod_ty mod) PyErr_SetString(PyExc_SystemError, "impossible module node"); return 0; } - - /* Check that the recursion depth counting balanced correctly */ - if (res && state.recursion_depth != starting_recursion_depth) { - PyErr_Format(PyExc_SystemError, - "AST validator recursion depth mismatch (before=%d, after=%d)", - starting_recursion_depth, state.recursion_depth); - return 0; - } return res; } diff --git a/Python/ast_opt.c b/Python/ast_opt.c index b1d807bcf10ae1..36117176c8dc14 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -1,6 +1,7 @@ /* AST Optimizer */ #include "Python.h" #include "pycore_ast.h" // _PyAST_GetDocString() +#include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_compile.h" // _PyASTOptimizeState #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_format.h" // F_LJUST @@ -705,9 +706,7 @@ astfold_mod(mod_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) static int astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) { - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); + if (_Py_EnterRecursiveCall("during compilation")) { return 0; } switch (node_->kind) { @@ -808,7 +807,7 @@ astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case Name_kind: if (node_->v.Name.ctx == Load && _PyUnicode_EqualToASCIIString(node_->v.Name.id, "__debug__")) { - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return make_const(node_, PyBool_FromLong(!state->optimize), ctx_); } break; @@ -821,7 +820,7 @@ astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) // No default case, so the compiler will emit a warning if new expression // kinds are added without being handled here } - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return 1; } @@ -868,9 +867,7 @@ astfold_arg(arg_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) static int astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) { - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); + if (_Py_EnterRecursiveCall("during compilation")) { return 0; } switch (node_->kind) { @@ -988,7 +985,7 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) // No default case, so the compiler will emit a warning if new statement // kinds are added without being handled here } - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return 1; } @@ -1020,9 +1017,7 @@ astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) // Currently, this is really only used to form complex/negative numeric // constants in MatchValue and MatchMapping nodes // We still recurse into all subexpressions and subpatterns anyway - if (++state->recursion_depth > state->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); + if (_Py_EnterRecursiveCall("during compilation")) { return 0; } switch (node_->kind) { @@ -1056,7 +1051,7 @@ astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) // No default case, so the compiler will emit a warning if new pattern // kinds are added without being handled here } - state->recursion_depth--; + _Py_LeaveRecursiveCall(); return 1; } @@ -1080,32 +1075,15 @@ int _PyAST_Optimize(mod_ty mod, PyArena *arena, _PyASTOptimizeState *state) { PyThreadState *tstate; - int recursion_limit = Py_GetRecursionLimit(); - int starting_recursion_depth; /* Setup recursion depth check counters */ tstate = _PyThreadState_GET(); if (!tstate) { return 0; } - /* Be careful here to prevent overflow. */ - int recursion_depth = tstate->recursion_limit - tstate->recursion_remaining; - starting_recursion_depth = (recursion_depth < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_depth * COMPILER_STACK_FRAME_SCALE : recursion_depth; - state->recursion_depth = starting_recursion_depth; - state->recursion_limit = (recursion_limit < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_limit * COMPILER_STACK_FRAME_SCALE : recursion_limit; int ret = astfold_mod(mod, arena, state); assert(ret || PyErr_Occurred()); - /* Check that the recursion depth counting balanced correctly */ - if (ret && state->recursion_depth != starting_recursion_depth) { - PyErr_Format(PyExc_SystemError, - "AST optimizer recursion depth mismatch (before=%d, after=%d)", - starting_recursion_depth, state->recursion_depth); - return 0; - } - return ret; } diff --git a/Python/ceval.c b/Python/ceval.c index c61ccd7dfc6f56..efc754100c0c18 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -257,52 +257,43 @@ Py_SetRecursionLimit(int new_limit) PyInterpreterState *interp = _PyInterpreterState_GET(); interp->ceval.recursion_limit = new_limit; for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { - int depth = p->recursion_limit - p->recursion_remaining; - p->recursion_limit = new_limit; - p->recursion_remaining = new_limit - depth; + int depth = p->py_recursion_limit - p->py_recursion_remaining; + p->py_recursion_limit = new_limit; + p->py_recursion_remaining = new_limit - depth; } } /* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall() if the recursion_depth reaches recursion_limit. */ + int -_Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) +_Py_CheckRecursiveCallN(PyThreadState *tstate, int n, const char *where) { - /* Check against global limit first. */ - int depth = tstate->recursion_limit - tstate->recursion_remaining; - if (depth < tstate->interp->ceval.recursion_limit) { - tstate->recursion_limit = tstate->interp->ceval.recursion_limit; - tstate->recursion_remaining = tstate->recursion_limit - depth; - assert(tstate->recursion_remaining > 0); - return 0; - } -#ifdef USE_STACKCHECK - if (PyOS_CheckStack()) { - ++tstate->recursion_remaining; - _PyErr_SetString(tstate, PyExc_MemoryError, "Stack overflow"); - return -1; - } -#endif - if (tstate->recursion_headroom) { - if (tstate->recursion_remaining < -50) { + assert(tstate->c_recursion_remaining < n); + if (tstate->c_recursion_headroom) { + if (tstate->c_recursion_remaining < -50) { /* Overflowing while handling an overflow. Give up. */ Py_FatalError("Cannot recover from stack overflow."); } } else { - if (tstate->recursion_remaining <= 0) { - tstate->recursion_headroom++; + if (tstate->c_recursion_remaining < n) { + tstate->c_recursion_remaining += n; _PyErr_Format(tstate, PyExc_RecursionError, - "maximum recursion depth exceeded%s", + "C stack overflow %s", where); - tstate->recursion_headroom--; - ++tstate->recursion_remaining; + tstate->c_recursion_remaining -= n; return -1; } } return 0; } +int +_Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) +{ + return _Py_CheckRecursiveCallN(tstate, 1, where); +} static const binaryfunc binary_ops[] = { [NB_ADD] = PyNumber_Add, @@ -998,6 +989,9 @@ typedef struct { PyObject* _Py_HOT_FUNCTION _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { + if (_Py_EnterRecursiveCallN(tstate, 3, " calling a Python function")) { + return NULL; + } _Py_EnsureTstateNotNULL(tstate); CALL_STAT_INC(pyeval_calls); @@ -1038,10 +1032,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* support for generator.throw() */ if (throwflag) { - if (_Py_EnterRecursiveCallTstate(tstate, "")) { - tstate->recursion_remaining--; - goto exit_unwind; - } + /* We are raising here anyway, so don't check recursion limit */ + tstate->py_recursion_remaining--; TRACE_FUNCTION_THROW_ENTRY(); DTRACE_FUNCTION_ENTRY(); goto resume_with_error; @@ -1078,11 +1070,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int start_frame: - if (_Py_EnterRecursiveCallTstate(tstate, "")) { - tstate->recursion_remaining--; + + tstate->py_recursion_remaining--; + if (tstate->py_recursion_remaining < 0) { + _PyErr_Format(tstate, PyExc_RecursionError, + "maximum recursion depth exceeded"); goto exit_unwind; } - resume_frame: SET_LOCALS_FROM_FRAME(); @@ -1830,7 +1824,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _PyFrame_SetStackPointer(frame, stack_pointer); TRACE_FUNCTION_EXIT(); DTRACE_FUNCTION_EXIT(); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->py_recursion_remaining++; if (!frame->is_entry) { frame = cframe.current_frame = pop_frame(tstate, frame); _PyFrame_StackPush(frame, retval); @@ -1841,6 +1835,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int tstate->cframe->use_tracing = cframe.use_tracing; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallN(tstate, 3); return retval; } @@ -2046,12 +2041,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _PyFrame_SetStackPointer(frame, stack_pointer); TRACE_FUNCTION_EXIT(); DTRACE_FUNCTION_EXIT(); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->py_recursion_remaining++; /* Restore previous cframe and return. */ tstate->cframe = cframe.previous; tstate->cframe->use_tracing = cframe.use_tracing; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallN(tstate, 3); return retval; } @@ -4915,7 +4911,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallTstate(tstate); + tstate->py_recursion_remaining++; if (!frame->is_entry) { _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); @@ -4933,6 +4929,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int tstate->cframe->use_tracing = cframe.use_tracing; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallN(tstate, 3); return (PyObject *)gen; } @@ -5287,8 +5284,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int exit_unwind: assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->py_recursion_remaining++; if (frame->is_entry) { + _Py_LeaveRecursiveCallN(tstate, 3); /* Restore previous cframe and exit */ tstate->cframe = cframe.previous; tstate->cframe->use_tracing = cframe.use_tracing; @@ -5858,11 +5856,11 @@ _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) // _PyThreadState_PopFrame, since f_code is already cleared at that point: assert((PyObject **)frame + frame->f_code->co_framesize == tstate->datastack_top); - tstate->recursion_remaining--; + tstate->c_recursion_remaining--; assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); assert(frame->owner == FRAME_OWNED_BY_THREAD); _PyFrame_Clear(frame); - tstate->recursion_remaining++; + tstate->c_recursion_remaining++; _PyThreadState_PopFrame(tstate, frame); } diff --git a/Python/errors.c b/Python/errors.c index 2aa748c60c3704..aaa5d010bcb680 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -310,14 +310,14 @@ _PyErr_NormalizeException(PyThreadState *tstate, PyObject **exc, PyObject **val, PyObject **tb) { int recursion_depth = 0; - tstate->recursion_headroom++; + tstate->c_recursion_headroom++; PyObject *type, *value, *initial_tb; restart: type = *exc; if (type == NULL) { /* There was no exception, so nothing to do. */ - tstate->recursion_headroom--; + tstate->c_recursion_headroom--; return; } @@ -369,7 +369,7 @@ _PyErr_NormalizeException(PyThreadState *tstate, PyObject **exc, } *exc = type; *val = value; - tstate->recursion_headroom--; + tstate->c_recursion_headroom--; return; error: diff --git a/Python/pystate.c b/Python/pystate.c index a11f1622ecd4ab..7d4e5f6b4b6049 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -792,8 +792,9 @@ init_threadstate(PyThreadState *tstate, tstate->native_thread_id = PyThread_get_thread_native_id(); #endif - tstate->recursion_limit = interp->ceval.recursion_limit, - tstate->recursion_remaining = interp->ceval.recursion_limit, + tstate->py_recursion_limit = interp->ceval.recursion_limit; + tstate->py_recursion_remaining = interp->ceval.recursion_limit; + tstate->c_recursion_remaining = C_RECURSION_LIMT; tstate->exc_info = &tstate->exc_state; diff --git a/Python/symtable.c b/Python/symtable.c index 0b259b08b61f97..fa76ef67e7e399 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1,5 +1,6 @@ #include "Python.h" #include "pycore_ast.h" // identifier, stmt_ty +#include "pycore_ceval.h" // _Py_LeaveRecursiveCall() #include "pycore_compile.h" // _Py_Mangle(), _PyFuture_FromAST() #include "pycore_parser.h" // _PyParser_ASTFromString() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -277,9 +278,6 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) struct symtable *st = symtable_new(); asdl_stmt_seq *seq; int i; - PyThreadState *tstate; - int recursion_limit = Py_GetRecursionLimit(); - int starting_recursion_depth; if (st == NULL) return NULL; @@ -291,20 +289,6 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) st->st_filename = filename; st->st_future = future; - /* Setup recursion depth check counters */ - tstate = _PyThreadState_GET(); - if (!tstate) { - _PySymtable_Free(st); - return NULL; - } - /* Be careful here to prevent overflow. */ - int recursion_depth = tstate->recursion_limit - tstate->recursion_remaining; - starting_recursion_depth = (recursion_depth < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_depth * COMPILER_STACK_FRAME_SCALE : recursion_depth; - st->recursion_depth = starting_recursion_depth; - st->recursion_limit = (recursion_limit < INT_MAX / COMPILER_STACK_FRAME_SCALE) ? - recursion_limit * COMPILER_STACK_FRAME_SCALE : recursion_limit; - /* Make the initial symbol information gathering pass */ if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) { _PySymtable_Free(st); @@ -340,14 +324,6 @@ _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) _PySymtable_Free(st); return NULL; } - /* Check that the recursion depth counting balanced correctly */ - if (st->recursion_depth != starting_recursion_depth) { - PyErr_Format(PyExc_SystemError, - "symtable analysis recursion depth mismatch (before=%d, after=%d)", - starting_recursion_depth, st->recursion_depth); - _PySymtable_Free(st); - return NULL; - } /* Make the second symbol analysis pass */ if (symtable_analyze(st)) return st; @@ -1125,7 +1101,7 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag, */ #define VISIT_QUIT(ST, X) \ - return --(ST)->recursion_depth,(X) + return _Py_LeaveRecursiveCall(),(X) #define VISIT(ST, TYPE, V) \ if (!symtable_visit_ ## TYPE((ST), (V))) \ @@ -1188,10 +1164,8 @@ symtable_record_directive(struct symtable *st, identifier name, int lineno, static int symtable_visit_stmt(struct symtable *st, stmt_ty s) { - if (++st->recursion_depth > st->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); - VISIT_QUIT(st, 0); + if (_Py_EnterRecursiveCall("during compilation")) { + return 0; } switch (s->kind) { case FunctionDef_kind: @@ -1574,10 +1548,8 @@ symtable_handle_namedexpr(struct symtable *st, expr_ty e) static int symtable_visit_expr(struct symtable *st, expr_ty e) { - if (++st->recursion_depth > st->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); - VISIT_QUIT(st, 0); + if (_Py_EnterRecursiveCall("during compilation")) { + return 0; } switch (e->kind) { case NamedExpr_kind: @@ -1734,10 +1706,8 @@ symtable_visit_expr(struct symtable *st, expr_ty e) static int symtable_visit_pattern(struct symtable *st, pattern_ty p) { - if (++st->recursion_depth > st->recursion_limit) { - PyErr_SetString(PyExc_RecursionError, - "maximum recursion depth exceeded during compilation"); - VISIT_QUIT(st, 0); + if (_Py_EnterRecursiveCall("during compilation")) { + return 0; } switch (p->kind) { case MatchValue_kind: diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 75e64553d88c9f..8a7fabef6cd948 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1217,7 +1217,7 @@ sys_setrecursionlimit_impl(PyObject *module, int new_limit) /* Reject too low new limit if the current recursion depth is higher than the new low-water mark. */ - int depth = tstate->recursion_limit - tstate->recursion_remaining; + int depth = tstate->py_recursion_limit - tstate->py_recursion_remaining; if (depth >= new_limit) { _PyErr_Format(tstate, PyExc_RecursionError, "cannot set the recursion limit to %i at " From 2c8346e9c441db63478f616ed320377ea0b00517 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 1 Sep 2022 14:20:08 +0100 Subject: [PATCH 2/9] Add tests for super deep recursion. --- Lib/test/test_call.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index c00de27b265d27..e1773e71c60cd1 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -813,6 +813,44 @@ def test_multiple_values(self): with self.check_raises_type_error(msg): A().method_two_args("x", "y", x="oops") +@cpython_only +class TestRecursion(unittest.TestCase): + + def test_super_deep(self): + + def recurse(n): + if n: + recurse(n-1) + + def py_recurse(n, m): + if n: + py_recurse(n-1, m) + else: + c_py_recurse(m-1) + + def c_recurse(n): + if n: + _testcapi.pyobject_fastcall(c_recurse, (n-1,)) + + def c_py_recurse(m): + if m: + _testcapi.pyobject_fastcall(py_recurse, (1000, m)) + + depth = sys.getrecursionlimit() + sys.setrecursionlimit(100_000) + try: + recurse(90_000) + with self.assertRaises(RecursionError): + recurse(101_000) + c_recurse(100) + with self.assertRaises(RecursionError): + c_recurse(90_000) + c_py_recurse(90) + with self.assertRaises(RecursionError): + c_py_recurse(100_000) + finally: + sys.setrecursionlimit(depth) + if __name__ == "__main__": unittest.main() From 73016dc71dd5020d0ef34a8ccc40ad972ac0580e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 2 Sep 2022 11:42:19 +0100 Subject: [PATCH 3/9] Add news --- .../2022-09-02-11-42-10.gh-issue-91079.32NfD7.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-09-02-11-42-10.gh-issue-91079.32NfD7.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-02-11-42-10.gh-issue-91079.32NfD7.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-02-11-42-10.gh-issue-91079.32NfD7.rst new file mode 100644 index 00000000000000..4b4b1a205b4c21 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-09-02-11-42-10.gh-issue-91079.32NfD7.rst @@ -0,0 +1,3 @@ +Decouple C stack overflow checking from Python recursion checking. Allows +the recursion limit to be increased safely, and reduces the chance of +segfaults. From 4467ec37196c3891d4104398a9e1162d5d79297c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 2 Sep 2022 12:01:00 +0100 Subject: [PATCH 4/9] Remove useless struct. --- Python/ast.c | 313 +++++++++++++++++++++++++-------------------------- 1 file changed, 151 insertions(+), 162 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index 014b36322d8436..3eae87cea9e22f 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -10,17 +10,14 @@ #include #include -struct validator { - int placeholder; -}; -static int validate_stmts(struct validator *, asdl_stmt_seq *); -static int validate_exprs(struct validator *, asdl_expr_seq *, expr_context_ty, int); -static int validate_patterns(struct validator *, asdl_pattern_seq *, int); +static int validate_stmts(asdl_stmt_seq *); +static int validate_exprs(asdl_expr_seq *, expr_context_ty, int); +static int validate_patterns(asdl_pattern_seq *, int); static int _validate_nonempty_seq(asdl_seq *, const char *, const char *); -static int validate_stmt(struct validator *, stmt_ty); -static int validate_expr(struct validator *, expr_ty, expr_context_ty); -static int validate_pattern(struct validator *, pattern_ty, int); +static int validate_stmt(stmt_ty); +static int validate_expr(expr_ty, expr_context_ty); +static int validate_pattern(pattern_ty, int); #define VALIDATE_POSITIONS(node) \ if (node->lineno > node->end_lineno) { \ @@ -63,7 +60,7 @@ validate_name(PyObject *name) } static int -validate_comprehension(struct validator *state, asdl_comprehension_seq *gens) +validate_comprehension(asdl_comprehension_seq *gens) { Py_ssize_t i; if (!asdl_seq_LEN(gens)) { @@ -72,32 +69,32 @@ validate_comprehension(struct validator *state, asdl_comprehension_seq *gens) } for (i = 0; i < asdl_seq_LEN(gens); i++) { comprehension_ty comp = asdl_seq_GET(gens, i); - if (!validate_expr(state, comp->target, Store) || - !validate_expr(state, comp->iter, Load) || - !validate_exprs(state, comp->ifs, Load, 0)) + if (!validate_expr(comp->target, Store) || + !validate_expr(comp->iter, Load) || + !validate_exprs(comp->ifs, Load, 0)) return 0; } return 1; } static int -validate_keywords(struct validator *state, asdl_keyword_seq *keywords) +validate_keywords(asdl_keyword_seq *keywords) { Py_ssize_t i; for (i = 0; i < asdl_seq_LEN(keywords); i++) - if (!validate_expr(state, (asdl_seq_GET(keywords, i))->value, Load)) + if (!validate_expr((asdl_seq_GET(keywords, i))->value, Load)) return 0; return 1; } static int -validate_args(struct validator *state, asdl_arg_seq *args) +validate_args(asdl_arg_seq *args) { Py_ssize_t i; for (i = 0; i < asdl_seq_LEN(args); i++) { arg_ty arg = asdl_seq_GET(args, i); VALIDATE_POSITIONS(arg); - if (arg->annotation && !validate_expr(state, arg->annotation, Load)) + if (arg->annotation && !validate_expr(arg->annotation, Load)) return 0; } return 1; @@ -119,19 +116,19 @@ expr_context_name(expr_context_ty ctx) } static int -validate_arguments(struct validator *state, arguments_ty args) +validate_arguments(arguments_ty args) { - if (!validate_args(state, args->posonlyargs) || !validate_args(state, args->args)) { + if (!validate_args(args->posonlyargs) || !validate_args(args->args)) { return 0; } if (args->vararg && args->vararg->annotation - && !validate_expr(state, args->vararg->annotation, Load)) { + && !validate_expr(args->vararg->annotation, Load)) { return 0; } - if (!validate_args(state, args->kwonlyargs)) + if (!validate_args(args->kwonlyargs)) return 0; if (args->kwarg && args->kwarg->annotation - && !validate_expr(state, args->kwarg->annotation, Load)) { + && !validate_expr(args->kwarg->annotation, Load)) { return 0; } if (asdl_seq_LEN(args->defaults) > asdl_seq_LEN(args->posonlyargs) + asdl_seq_LEN(args->args)) { @@ -143,11 +140,11 @@ validate_arguments(struct validator *state, arguments_ty args) "kw_defaults on arguments"); return 0; } - return validate_exprs(state, args->defaults, Load, 0) && validate_exprs(state, args->kw_defaults, Load, 1); + return validate_exprs(args->defaults, Load, 0) && validate_exprs(args->kw_defaults, Load, 1); } static int -validate_constant(struct validator *state, PyObject *value) +validate_constant(PyObject *value) { if (value == Py_None || value == Py_Ellipsis) return 1; @@ -179,7 +176,7 @@ validate_constant(struct validator *state, PyObject *value) break; } - if (!validate_constant(state, item)) { + if (!validate_constant(item)) { Py_DECREF(it); Py_DECREF(item); return 0; @@ -201,7 +198,7 @@ validate_constant(struct validator *state, PyObject *value) } static int -validate_expr(struct validator *state, expr_ty exp, expr_context_ty ctx) +validate_expr(expr_ty exp, expr_context_ty ctx) { VALIDATE_POSITIONS(exp); int ret = -1; @@ -258,23 +255,23 @@ validate_expr(struct validator *state, expr_ty exp, expr_context_ty ctx) PyErr_SetString(PyExc_ValueError, "BoolOp with less than 2 values"); return 0; } - ret = validate_exprs(state, exp->v.BoolOp.values, Load, 0); + ret = validate_exprs(exp->v.BoolOp.values, Load, 0); break; case BinOp_kind: - ret = validate_expr(state, exp->v.BinOp.left, Load) && - validate_expr(state, exp->v.BinOp.right, Load); + ret = validate_expr(exp->v.BinOp.left, Load) && + validate_expr(exp->v.BinOp.right, Load); break; case UnaryOp_kind: - ret = validate_expr(state, exp->v.UnaryOp.operand, Load); + ret = validate_expr(exp->v.UnaryOp.operand, Load); break; case Lambda_kind: - ret = validate_arguments(state, exp->v.Lambda.args) && - validate_expr(state, exp->v.Lambda.body, Load); + ret = validate_arguments(exp->v.Lambda.args) && + validate_expr(exp->v.Lambda.body, Load); break; case IfExp_kind: - ret = validate_expr(state, exp->v.IfExp.test, Load) && - validate_expr(state, exp->v.IfExp.body, Load) && - validate_expr(state, exp->v.IfExp.orelse, Load); + ret = validate_expr(exp->v.IfExp.test, Load) && + validate_expr(exp->v.IfExp.body, Load) && + validate_expr(exp->v.IfExp.orelse, Load); break; case Dict_kind: if (asdl_seq_LEN(exp->v.Dict.keys) != asdl_seq_LEN(exp->v.Dict.values)) { @@ -284,34 +281,34 @@ validate_expr(struct validator *state, expr_ty exp, expr_context_ty ctx) } /* null_ok=1 for keys expressions to allow dict unpacking to work in dict literals, i.e. ``{**{a:b}}`` */ - ret = validate_exprs(state, exp->v.Dict.keys, Load, /*null_ok=*/ 1) && - validate_exprs(state, exp->v.Dict.values, Load, /*null_ok=*/ 0); + ret = validate_exprs(exp->v.Dict.keys, Load, /*null_ok=*/ 1) && + validate_exprs(exp->v.Dict.values, Load, /*null_ok=*/ 0); break; case Set_kind: - ret = validate_exprs(state, exp->v.Set.elts, Load, 0); + ret = validate_exprs(exp->v.Set.elts, Load, 0); break; #define COMP(NAME) \ case NAME ## _kind: \ - ret = validate_comprehension(state, exp->v.NAME.generators) && \ - validate_expr(state, exp->v.NAME.elt, Load); \ + ret = validate_comprehension(exp->v.NAME.generators) && \ + validate_expr(exp->v.NAME.elt, Load); \ break; COMP(ListComp) COMP(SetComp) COMP(GeneratorExp) #undef COMP case DictComp_kind: - ret = validate_comprehension(state, exp->v.DictComp.generators) && - validate_expr(state, exp->v.DictComp.key, Load) && - validate_expr(state, exp->v.DictComp.value, Load); + ret = validate_comprehension(exp->v.DictComp.generators) && + validate_expr(exp->v.DictComp.key, Load) && + validate_expr(exp->v.DictComp.value, Load); break; case Yield_kind: - ret = !exp->v.Yield.value || validate_expr(state, exp->v.Yield.value, Load); + ret = !exp->v.Yield.value || validate_expr(exp->v.Yield.value, Load); break; case YieldFrom_kind: - ret = validate_expr(state, exp->v.YieldFrom.value, Load); + ret = validate_expr(exp->v.YieldFrom.value, Load); break; case Await_kind: - ret = validate_expr(state, exp->v.Await.value, Load); + ret = validate_expr(exp->v.Await.value, Load); break; case Compare_kind: if (!asdl_seq_LEN(exp->v.Compare.comparators)) { @@ -324,55 +321,55 @@ validate_expr(struct validator *state, expr_ty exp, expr_context_ty ctx) "of comparators and operands"); return 0; } - ret = validate_exprs(state, exp->v.Compare.comparators, Load, 0) && - validate_expr(state, exp->v.Compare.left, Load); + ret = validate_exprs(exp->v.Compare.comparators, Load, 0) && + validate_expr(exp->v.Compare.left, Load); break; case Call_kind: - ret = validate_expr(state, exp->v.Call.func, Load) && - validate_exprs(state, exp->v.Call.args, Load, 0) && - validate_keywords(state, exp->v.Call.keywords); + ret = validate_expr(exp->v.Call.func, Load) && + validate_exprs(exp->v.Call.args, Load, 0) && + validate_keywords(exp->v.Call.keywords); break; case Constant_kind: - if (!validate_constant(state, exp->v.Constant.value)) { + if (!validate_constant(exp->v.Constant.value)) { return 0; } ret = 1; break; case JoinedStr_kind: - ret = validate_exprs(state, exp->v.JoinedStr.values, Load, 0); + ret = validate_exprs(exp->v.JoinedStr.values, Load, 0); break; case FormattedValue_kind: - if (validate_expr(state, exp->v.FormattedValue.value, Load) == 0) + if (validate_expr(exp->v.FormattedValue.value, Load) == 0) return 0; if (exp->v.FormattedValue.format_spec) { - ret = validate_expr(state, exp->v.FormattedValue.format_spec, Load); + ret = validate_expr(exp->v.FormattedValue.format_spec, Load); break; } ret = 1; break; case Attribute_kind: - ret = validate_expr(state, exp->v.Attribute.value, Load); + ret = validate_expr(exp->v.Attribute.value, Load); break; case Subscript_kind: - ret = validate_expr(state, exp->v.Subscript.slice, Load) && - validate_expr(state, exp->v.Subscript.value, Load); + ret = validate_expr(exp->v.Subscript.slice, Load) && + validate_expr(exp->v.Subscript.value, Load); break; case Starred_kind: - ret = validate_expr(state, exp->v.Starred.value, ctx); + ret = validate_expr(exp->v.Starred.value, ctx); break; case Slice_kind: - ret = (!exp->v.Slice.lower || validate_expr(state, exp->v.Slice.lower, Load)) && - (!exp->v.Slice.upper || validate_expr(state, exp->v.Slice.upper, Load)) && - (!exp->v.Slice.step || validate_expr(state, exp->v.Slice.step, Load)); + ret = (!exp->v.Slice.lower || validate_expr(exp->v.Slice.lower, Load)) && + (!exp->v.Slice.upper || validate_expr(exp->v.Slice.upper, Load)) && + (!exp->v.Slice.step || validate_expr(exp->v.Slice.step, Load)); break; case List_kind: - ret = validate_exprs(state, exp->v.List.elts, ctx, 0); + ret = validate_exprs(exp->v.List.elts, ctx, 0); break; case Tuple_kind: - ret = validate_exprs(state, exp->v.Tuple.elts, ctx, 0); + ret = validate_exprs(exp->v.Tuple.elts, ctx, 0); break; case NamedExpr_kind: - ret = validate_expr(state, exp->v.NamedExpr.value, Load); + ret = validate_expr(exp->v.NamedExpr.value, Load); break; /* This last case doesn't have any checking. */ case Name_kind: @@ -460,9 +457,9 @@ ensure_literal_complex(expr_ty exp) } static int -validate_pattern_match_value(struct validator *state, expr_ty exp) +validate_pattern_match_value(expr_ty exp) { - if (!validate_expr(state, exp, Load)) { + if (!validate_expr(exp, Load)) { return 0; } @@ -472,7 +469,7 @@ validate_pattern_match_value(struct validator *state, expr_ty exp) /* Ellipsis and immutable sequences are not allowed. For True, False and None, MatchSingleton() should be used */ - if (!validate_expr(state, exp, Load)) { + if (!validate_expr(exp, Load)) { return 0; } PyObject *literal = exp->v.Constant.value; @@ -523,7 +520,7 @@ validate_capture(PyObject *name) } static int -validate_pattern(struct validator *state, pattern_ty p, int star_ok) +validate_pattern(pattern_ty p, int star_ok) { VALIDATE_POSITIONS(p); int ret = -1; @@ -532,7 +529,7 @@ validate_pattern(struct validator *state, pattern_ty p, int star_ok) } switch (p->kind) { case MatchValue_kind: - ret = validate_pattern_match_value(state, p->v.MatchValue.value); + ret = validate_pattern_match_value(p->v.MatchValue.value); break; case MatchSingleton_kind: ret = p->v.MatchSingleton.value == Py_None || PyBool_Check(p->v.MatchSingleton.value); @@ -542,7 +539,7 @@ validate_pattern(struct validator *state, pattern_ty p, int star_ok) } break; case MatchSequence_kind: - ret = validate_patterns(state, p->v.MatchSequence.patterns, /*star_ok=*/1); + ret = validate_patterns(p->v.MatchSequence.patterns, /*star_ok=*/1); break; case MatchMapping_kind: if (asdl_seq_LEN(p->v.MatchMapping.keys) != asdl_seq_LEN(p->v.MatchMapping.patterns)) { @@ -570,13 +567,13 @@ validate_pattern(struct validator *state, pattern_ty p, int star_ok) continue; } } - if (!validate_pattern_match_value(state, key)) { + if (!validate_pattern_match_value(key)) { ret = 0; break; } } - ret = validate_patterns(state, p->v.MatchMapping.patterns, /*star_ok=*/0); + ret = validate_patterns(p->v.MatchMapping.patterns, /*star_ok=*/0); break; case MatchClass_kind: if (asdl_seq_LEN(p->v.MatchClass.kwd_attrs) != asdl_seq_LEN(p->v.MatchClass.kwd_patterns)) { @@ -585,7 +582,7 @@ validate_pattern(struct validator *state, pattern_ty p, int star_ok) ret = 0; break; } - if (!validate_expr(state, p->v.MatchClass.cls, Load)) { + if (!validate_expr(p->v.MatchClass.cls, Load)) { ret = 0; break; } @@ -615,12 +612,12 @@ validate_pattern(struct validator *state, pattern_ty p, int star_ok) } } - if (!validate_patterns(state, p->v.MatchClass.patterns, /*star_ok=*/0)) { + if (!validate_patterns(p->v.MatchClass.patterns, /*star_ok=*/0)) { ret = 0; break; } - ret = validate_patterns(state, p->v.MatchClass.kwd_patterns, /*star_ok=*/0); + ret = validate_patterns(p->v.MatchClass.kwd_patterns, /*star_ok=*/0); break; case MatchStar_kind: if (!star_ok) { @@ -644,7 +641,7 @@ validate_pattern(struct validator *state, pattern_ty p, int star_ok) ret = 0; } else { - ret = validate_pattern(state, p->v.MatchAs.pattern, /*star_ok=*/0); + ret = validate_pattern(p->v.MatchAs.pattern, /*star_ok=*/0); } break; case MatchOr_kind: @@ -654,7 +651,7 @@ validate_pattern(struct validator *state, pattern_ty p, int star_ok) ret = 0; break; } - ret = validate_patterns(state, p->v.MatchOr.patterns, /*star_ok=*/0); + ret = validate_patterns(p->v.MatchOr.patterns, /*star_ok=*/0); break; // No default case, so the compiler will emit a warning if new pattern // kinds are added without being handled here @@ -678,20 +675,20 @@ _validate_nonempty_seq(asdl_seq *seq, const char *what, const char *owner) #define validate_nonempty_seq(seq, what, owner) _validate_nonempty_seq((asdl_seq*)seq, what, owner) static int -validate_assignlist(struct validator *state, asdl_expr_seq *targets, expr_context_ty ctx) +validate_assignlist(asdl_expr_seq *targets, expr_context_ty ctx) { return validate_nonempty_seq(targets, "targets", ctx == Del ? "Delete" : "Assign") && - validate_exprs(state, targets, ctx, 0); + validate_exprs(targets, ctx, 0); } static int -validate_body(struct validator *state, asdl_stmt_seq *body, const char *owner) +validate_body(asdl_stmt_seq *body, const char *owner) { - return validate_nonempty_seq(body, "body", owner) && validate_stmts(state, body); + return validate_nonempty_seq(body, "body", owner) && validate_stmts(body); } static int -validate_stmt(struct validator *state, stmt_ty stmt) +validate_stmt(stmt_ty stmt) { VALIDATE_POSITIONS(stmt); int ret = -1; @@ -701,31 +698,31 @@ validate_stmt(struct validator *state, stmt_ty stmt) } switch (stmt->kind) { case FunctionDef_kind: - ret = validate_body(state, stmt->v.FunctionDef.body, "FunctionDef") && - validate_arguments(state, stmt->v.FunctionDef.args) && - validate_exprs(state, stmt->v.FunctionDef.decorator_list, Load, 0) && + ret = validate_body(stmt->v.FunctionDef.body, "FunctionDef") && + validate_arguments(stmt->v.FunctionDef.args) && + validate_exprs(stmt->v.FunctionDef.decorator_list, Load, 0) && (!stmt->v.FunctionDef.returns || - validate_expr(state, stmt->v.FunctionDef.returns, Load)); + validate_expr(stmt->v.FunctionDef.returns, Load)); break; case ClassDef_kind: - ret = validate_body(state, stmt->v.ClassDef.body, "ClassDef") && - validate_exprs(state, stmt->v.ClassDef.bases, Load, 0) && - validate_keywords(state, stmt->v.ClassDef.keywords) && - validate_exprs(state, stmt->v.ClassDef.decorator_list, Load, 0); + ret = validate_body(stmt->v.ClassDef.body, "ClassDef") && + validate_exprs(stmt->v.ClassDef.bases, Load, 0) && + validate_keywords(stmt->v.ClassDef.keywords) && + validate_exprs(stmt->v.ClassDef.decorator_list, Load, 0); break; case Return_kind: - ret = !stmt->v.Return.value || validate_expr(state, stmt->v.Return.value, Load); + ret = !stmt->v.Return.value || validate_expr(stmt->v.Return.value, Load); break; case Delete_kind: - ret = validate_assignlist(state, stmt->v.Delete.targets, Del); + ret = validate_assignlist(stmt->v.Delete.targets, Del); break; case Assign_kind: - ret = validate_assignlist(state, stmt->v.Assign.targets, Store) && - validate_expr(state, stmt->v.Assign.value, Load); + ret = validate_assignlist(stmt->v.Assign.targets, Store) && + validate_expr(stmt->v.Assign.value, Load); break; case AugAssign_kind: - ret = validate_expr(state, stmt->v.AugAssign.target, Store) && - validate_expr(state, stmt->v.AugAssign.value, Load); + ret = validate_expr(stmt->v.AugAssign.target, Store) && + validate_expr(stmt->v.AugAssign.value, Load); break; case AnnAssign_kind: if (stmt->v.AnnAssign.target->kind != Name_kind && @@ -734,65 +731,65 @@ validate_stmt(struct validator *state, stmt_ty stmt) "AnnAssign with simple non-Name target"); return 0; } - ret = validate_expr(state, stmt->v.AnnAssign.target, Store) && + ret = validate_expr(stmt->v.AnnAssign.target, Store) && (!stmt->v.AnnAssign.value || - validate_expr(state, stmt->v.AnnAssign.value, Load)) && - validate_expr(state, stmt->v.AnnAssign.annotation, Load); + validate_expr(stmt->v.AnnAssign.value, Load)) && + validate_expr(stmt->v.AnnAssign.annotation, Load); break; case For_kind: - ret = validate_expr(state, stmt->v.For.target, Store) && - validate_expr(state, stmt->v.For.iter, Load) && - validate_body(state, stmt->v.For.body, "For") && - validate_stmts(state, stmt->v.For.orelse); + ret = validate_expr(stmt->v.For.target, Store) && + validate_expr(stmt->v.For.iter, Load) && + validate_body(stmt->v.For.body, "For") && + validate_stmts(stmt->v.For.orelse); break; case AsyncFor_kind: - ret = validate_expr(state, stmt->v.AsyncFor.target, Store) && - validate_expr(state, stmt->v.AsyncFor.iter, Load) && - validate_body(state, stmt->v.AsyncFor.body, "AsyncFor") && - validate_stmts(state, stmt->v.AsyncFor.orelse); + ret = validate_expr(stmt->v.AsyncFor.target, Store) && + validate_expr(stmt->v.AsyncFor.iter, Load) && + validate_body(stmt->v.AsyncFor.body, "AsyncFor") && + validate_stmts(stmt->v.AsyncFor.orelse); break; case While_kind: - ret = validate_expr(state, stmt->v.While.test, Load) && - validate_body(state, stmt->v.While.body, "While") && - validate_stmts(state, stmt->v.While.orelse); + ret = validate_expr(stmt->v.While.test, Load) && + validate_body(stmt->v.While.body, "While") && + validate_stmts(stmt->v.While.orelse); break; case If_kind: - ret = validate_expr(state, stmt->v.If.test, Load) && - validate_body(state, stmt->v.If.body, "If") && - validate_stmts(state, stmt->v.If.orelse); + ret = validate_expr(stmt->v.If.test, Load) && + validate_body(stmt->v.If.body, "If") && + validate_stmts(stmt->v.If.orelse); break; case With_kind: if (!validate_nonempty_seq(stmt->v.With.items, "items", "With")) return 0; for (i = 0; i < asdl_seq_LEN(stmt->v.With.items); i++) { withitem_ty item = asdl_seq_GET(stmt->v.With.items, i); - if (!validate_expr(state, item->context_expr, Load) || - (item->optional_vars && !validate_expr(state, item->optional_vars, Store))) + if (!validate_expr(item->context_expr, Load) || + (item->optional_vars && !validate_expr(item->optional_vars, Store))) return 0; } - ret = validate_body(state, stmt->v.With.body, "With"); + ret = validate_body(stmt->v.With.body, "With"); break; case AsyncWith_kind: if (!validate_nonempty_seq(stmt->v.AsyncWith.items, "items", "AsyncWith")) return 0; for (i = 0; i < asdl_seq_LEN(stmt->v.AsyncWith.items); i++) { withitem_ty item = asdl_seq_GET(stmt->v.AsyncWith.items, i); - if (!validate_expr(state, item->context_expr, Load) || - (item->optional_vars && !validate_expr(state, item->optional_vars, Store))) + if (!validate_expr(item->context_expr, Load) || + (item->optional_vars && !validate_expr(item->optional_vars, Store))) return 0; } - ret = validate_body(state, stmt->v.AsyncWith.body, "AsyncWith"); + ret = validate_body(stmt->v.AsyncWith.body, "AsyncWith"); break; case Match_kind: - if (!validate_expr(state, stmt->v.Match.subject, Load) + if (!validate_expr(stmt->v.Match.subject, Load) || !validate_nonempty_seq(stmt->v.Match.cases, "cases", "Match")) { return 0; } for (i = 0; i < asdl_seq_LEN(stmt->v.Match.cases); i++) { match_case_ty m = asdl_seq_GET(stmt->v.Match.cases, i); - if (!validate_pattern(state, m->pattern, /*star_ok=*/0) - || (m->guard && !validate_expr(state, m->guard, Load)) - || !validate_body(state, m->body, "match_case")) { + if (!validate_pattern(m->pattern, /*star_ok=*/0) + || (m->guard && !validate_expr(m->guard, Load)) + || !validate_body(m->body, "match_case")) { return 0; } } @@ -800,8 +797,8 @@ validate_stmt(struct validator *state, stmt_ty stmt) break; case Raise_kind: if (stmt->v.Raise.exc) { - ret = validate_expr(state, stmt->v.Raise.exc, Load) && - (!stmt->v.Raise.cause || validate_expr(state, stmt->v.Raise.cause, Load)); + ret = validate_expr(stmt->v.Raise.exc, Load) && + (!stmt->v.Raise.cause || validate_expr(stmt->v.Raise.cause, Load)); break; } if (stmt->v.Raise.cause) { @@ -811,7 +808,7 @@ validate_stmt(struct validator *state, stmt_ty stmt) ret = 1; break; case Try_kind: - if (!validate_body(state, stmt->v.Try.body, "Try")) + if (!validate_body(stmt->v.Try.body, "Try")) return 0; if (!asdl_seq_LEN(stmt->v.Try.handlers) && !asdl_seq_LEN(stmt->v.Try.finalbody)) { @@ -827,17 +824,17 @@ validate_stmt(struct validator *state, stmt_ty stmt) excepthandler_ty handler = asdl_seq_GET(stmt->v.Try.handlers, i); VALIDATE_POSITIONS(handler); if ((handler->v.ExceptHandler.type && - !validate_expr(state, handler->v.ExceptHandler.type, Load)) || - !validate_body(state, handler->v.ExceptHandler.body, "ExceptHandler")) + !validate_expr(handler->v.ExceptHandler.type, Load)) || + !validate_body(handler->v.ExceptHandler.body, "ExceptHandler")) return 0; } ret = (!asdl_seq_LEN(stmt->v.Try.finalbody) || - validate_stmts(state, stmt->v.Try.finalbody)) && + validate_stmts(stmt->v.Try.finalbody)) && (!asdl_seq_LEN(stmt->v.Try.orelse) || - validate_stmts(state, stmt->v.Try.orelse)); + validate_stmts(stmt->v.Try.orelse)); break; case TryStar_kind: - if (!validate_body(state, stmt->v.TryStar.body, "TryStar")) + if (!validate_body(stmt->v.TryStar.body, "TryStar")) return 0; if (!asdl_seq_LEN(stmt->v.TryStar.handlers) && !asdl_seq_LEN(stmt->v.TryStar.finalbody)) { @@ -852,18 +849,18 @@ validate_stmt(struct validator *state, stmt_ty stmt) for (i = 0; i < asdl_seq_LEN(stmt->v.TryStar.handlers); i++) { excepthandler_ty handler = asdl_seq_GET(stmt->v.TryStar.handlers, i); if ((handler->v.ExceptHandler.type && - !validate_expr(state, handler->v.ExceptHandler.type, Load)) || - !validate_body(state, handler->v.ExceptHandler.body, "ExceptHandler")) + !validate_expr(handler->v.ExceptHandler.type, Load)) || + !validate_body(handler->v.ExceptHandler.body, "ExceptHandler")) return 0; } ret = (!asdl_seq_LEN(stmt->v.TryStar.finalbody) || - validate_stmts(state, stmt->v.TryStar.finalbody)) && + validate_stmts(stmt->v.TryStar.finalbody)) && (!asdl_seq_LEN(stmt->v.TryStar.orelse) || - validate_stmts(state, stmt->v.TryStar.orelse)); + validate_stmts(stmt->v.TryStar.orelse)); break; case Assert_kind: - ret = validate_expr(state, stmt->v.Assert.test, Load) && - (!stmt->v.Assert.msg || validate_expr(state, stmt->v.Assert.msg, Load)); + ret = validate_expr(stmt->v.Assert.test, Load) && + (!stmt->v.Assert.msg || validate_expr(stmt->v.Assert.msg, Load)); break; case Import_kind: ret = validate_nonempty_seq(stmt->v.Import.names, "names", "Import"); @@ -882,14 +879,14 @@ validate_stmt(struct validator *state, stmt_ty stmt) ret = validate_nonempty_seq(stmt->v.Nonlocal.names, "names", "Nonlocal"); break; case Expr_kind: - ret = validate_expr(state, stmt->v.Expr.value, Load); + ret = validate_expr(stmt->v.Expr.value, Load); break; case AsyncFunctionDef_kind: - ret = validate_body(state, stmt->v.AsyncFunctionDef.body, "AsyncFunctionDef") && - validate_arguments(state, stmt->v.AsyncFunctionDef.args) && - validate_exprs(state, stmt->v.AsyncFunctionDef.decorator_list, Load, 0) && + ret = validate_body(stmt->v.AsyncFunctionDef.body, "AsyncFunctionDef") && + validate_arguments(stmt->v.AsyncFunctionDef.args) && + validate_exprs(stmt->v.AsyncFunctionDef.decorator_list, Load, 0) && (!stmt->v.AsyncFunctionDef.returns || - validate_expr(state, stmt->v.AsyncFunctionDef.returns, Load)); + validate_expr(stmt->v.AsyncFunctionDef.returns, Load)); break; case Pass_kind: case Break_kind: @@ -907,13 +904,13 @@ validate_stmt(struct validator *state, stmt_ty stmt) } static int -validate_stmts(struct validator *state, asdl_stmt_seq *seq) +validate_stmts(asdl_stmt_seq *seq) { Py_ssize_t i; for (i = 0; i < asdl_seq_LEN(seq); i++) { stmt_ty stmt = asdl_seq_GET(seq, i); if (stmt) { - if (!validate_stmt(state, stmt)) + if (!validate_stmt(stmt)) return 0; } else { @@ -926,13 +923,13 @@ validate_stmts(struct validator *state, asdl_stmt_seq *seq) } static int -validate_exprs(struct validator *state, asdl_expr_seq *exprs, expr_context_ty ctx, int null_ok) +validate_exprs(asdl_expr_seq *exprs, expr_context_ty ctx, int null_ok) { Py_ssize_t i; for (i = 0; i < asdl_seq_LEN(exprs); i++) { expr_ty expr = asdl_seq_GET(exprs, i); if (expr) { - if (!validate_expr(state, expr, ctx)) + if (!validate_expr(expr, ctx)) return 0; } else if (!null_ok) { @@ -946,12 +943,12 @@ validate_exprs(struct validator *state, asdl_expr_seq *exprs, expr_context_ty ct } static int -validate_patterns(struct validator *state, asdl_pattern_seq *patterns, int star_ok) +validate_patterns(asdl_pattern_seq *patterns, int star_ok) { Py_ssize_t i; for (i = 0; i < asdl_seq_LEN(patterns); i++) { pattern_ty pattern = asdl_seq_GET(patterns, i); - if (!validate_pattern(state, pattern, star_ok)) { + if (!validate_pattern(pattern, star_ok)) { return 0; } } @@ -966,28 +963,20 @@ int _PyAST_Validate(mod_ty mod) { int res = -1; - struct validator state; - PyThreadState *tstate; - - /* Setup recursion depth check counters */ - tstate = _PyThreadState_GET(); - if (!tstate) { - return 0; - } switch (mod->kind) { case Module_kind: - res = validate_stmts(&state, mod->v.Module.body); + res = validate_stmts(mod->v.Module.body); break; case Interactive_kind: - res = validate_stmts(&state, mod->v.Interactive.body); + res = validate_stmts(mod->v.Interactive.body); break; case Expression_kind: - res = validate_expr(&state, mod->v.Expression.body, Load); + res = validate_expr(mod->v.Expression.body, Load); break; case FunctionType_kind: - res = validate_exprs(&state, mod->v.FunctionType.argtypes, Load, /*null_ok=*/0) && - validate_expr(&state, mod->v.FunctionType.returns, Load); + res = validate_exprs(mod->v.FunctionType.argtypes, Load, /*null_ok=*/0) && + validate_expr(mod->v.FunctionType.returns, Load); break; // No default case so compiler emits warning for unhandled cases } From a28071f724596712570469f7e5a872887c320866 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 2 Sep 2022 12:38:52 +0100 Subject: [PATCH 5/9] Fix formatting of error message. --- Python/ceval.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ceval.c b/Python/ceval.c index efc754100c0c18..4cdb3084e28d22 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -280,7 +280,7 @@ _Py_CheckRecursiveCallN(PyThreadState *tstate, int n, const char *where) if (tstate->c_recursion_remaining < n) { tstate->c_recursion_remaining += n; _PyErr_Format(tstate, PyExc_RecursionError, - "C stack overflow %s", + "C stack overflow%s", where); tstate->c_recursion_remaining -= n; return -1; From 388d01d02aad430ee3cf09c9432b2cd2fc76594a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 2 Sep 2022 13:05:59 +0100 Subject: [PATCH 6/9] Fix formatting. --- Python/ast.c | 8 ++++---- Python/ast_opt.c | 6 +++--- Python/symtable.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index 3eae87cea9e22f..0016132564bc40 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -158,7 +158,7 @@ validate_constant(PyObject *value) return 1; if (PyTuple_CheckExact(value) || PyFrozenSet_CheckExact(value)) { - if (_Py_EnterRecursiveCall("during compilation")) { + if (_Py_EnterRecursiveCall(" during compilation")) { return 0; } @@ -203,7 +203,7 @@ validate_expr(expr_ty exp, expr_context_ty ctx) VALIDATE_POSITIONS(exp); int ret = -1; - if (_Py_EnterRecursiveCall("during compilation")) { + if (_Py_EnterRecursiveCall(" during compilation")) { return 0; } int check_ctx = 1; @@ -524,7 +524,7 @@ validate_pattern(pattern_ty p, int star_ok) { VALIDATE_POSITIONS(p); int ret = -1; - if (_Py_EnterRecursiveCall("during compilation")) { + if (_Py_EnterRecursiveCall(" during compilation")) { return 0; } switch (p->kind) { @@ -693,7 +693,7 @@ validate_stmt(stmt_ty stmt) VALIDATE_POSITIONS(stmt); int ret = -1; Py_ssize_t i; - if (_Py_EnterRecursiveCall("during compilation")) { + if (_Py_EnterRecursiveCall(" during compilation")) { return 0; } switch (stmt->kind) { diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 36117176c8dc14..3accc04d27f13d 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -706,7 +706,7 @@ astfold_mod(mod_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) static int astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) { - if (_Py_EnterRecursiveCall("during compilation")) { + if (_Py_EnterRecursiveCall(" during compilation")) { return 0; } switch (node_->kind) { @@ -867,7 +867,7 @@ astfold_arg(arg_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) static int astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) { - if (_Py_EnterRecursiveCall("during compilation")) { + if (_Py_EnterRecursiveCall(" during compilation")) { return 0; } switch (node_->kind) { @@ -1017,7 +1017,7 @@ astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) // Currently, this is really only used to form complex/negative numeric // constants in MatchValue and MatchMapping nodes // We still recurse into all subexpressions and subpatterns anyway - if (_Py_EnterRecursiveCall("during compilation")) { + if (_Py_EnterRecursiveCall(" during compilation")) { return 0; } switch (node_->kind) { diff --git a/Python/symtable.c b/Python/symtable.c index fa76ef67e7e399..2ae8f33fdeb040 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1164,7 +1164,7 @@ symtable_record_directive(struct symtable *st, identifier name, int lineno, static int symtable_visit_stmt(struct symtable *st, stmt_ty s) { - if (_Py_EnterRecursiveCall("during compilation")) { + if (_Py_EnterRecursiveCall(" during compilation")) { return 0; } switch (s->kind) { @@ -1548,7 +1548,7 @@ symtable_handle_namedexpr(struct symtable *st, expr_ty e) static int symtable_visit_expr(struct symtable *st, expr_ty e) { - if (_Py_EnterRecursiveCall("during compilation")) { + if (_Py_EnterRecursiveCall(" during compilation")) { return 0; } switch (e->kind) { @@ -1706,7 +1706,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) static int symtable_visit_pattern(struct symtable *st, pattern_ty p) { - if (_Py_EnterRecursiveCall("during compilation")) { + if (_Py_EnterRecursiveCall(" during compilation")) { return 0; } switch (p->kind) { From b4d4989a4678b2325a83e003ec5384fb35d0b7ed Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 2 Sep 2022 13:06:25 +0100 Subject: [PATCH 7/9] Reduce stack consumption of stack check. --- Include/internal/pycore_ceval.h | 13 ++++--------- Python/ceval.c | 7 ++++--- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 75352306b96cf2..709ab0bf2a4e0b 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -123,19 +123,14 @@ PyAPI_FUNC(int) _Py_CheckRecursiveCallN(PyThreadState *tstate, int n, static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate, const char *where) { - int remaining = tstate->c_recursion_remaining; - if (remaining > 0) { - tstate->c_recursion_remaining = remaining - 1; - return 0; - } - return _Py_CheckRecursiveCallN(tstate, 1, where); + return (tstate->c_recursion_remaining-- <= 0) && + _Py_CheckRecursiveCallN(tstate, 1, where); } static inline int _Py_EnterRecursiveCallN(PyThreadState *tstate, int n, const char *where) { - int remaining = tstate->c_recursion_remaining; - if (remaining >= n) { - tstate->c_recursion_remaining = remaining - n; + tstate->c_recursion_remaining -= n; + if (tstate->c_recursion_remaining < 0) { return 0; } return _Py_CheckRecursiveCallN(tstate, n, where); diff --git a/Python/ceval.c b/Python/ceval.c index 4cdb3084e28d22..2a5e918ce708b7 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -277,12 +277,13 @@ _Py_CheckRecursiveCallN(PyThreadState *tstate, int n, const char *where) } } else { - if (tstate->c_recursion_remaining < n) { - tstate->c_recursion_remaining += n; + if (tstate->c_recursion_remaining <= 0) { + tstate->c_recursion_headroom++; _PyErr_Format(tstate, PyExc_RecursionError, "C stack overflow%s", where); - tstate->c_recursion_remaining -= n; + tstate->c_recursion_headroom--; + tstate->c_recursion_remaining += n; return -1; } } From a6ccf957dd8fedabe27d05c295c02fa9d7408d81 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 2 Sep 2022 13:48:46 +0100 Subject: [PATCH 8/9] Reduce amount of C stack available. Should make things more robust. --- Include/internal/pycore_ceval.h | 4 ++-- Lib/test/test_compile.py | 3 +-- Python/ceval.c | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 709ab0bf2a4e0b..4b3ec9ea687b5d 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -130,7 +130,7 @@ static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate, static inline int _Py_EnterRecursiveCallN(PyThreadState *tstate, int n, const char *where) { tstate->c_recursion_remaining -= n; - if (tstate->c_recursion_remaining < 0) { + if (tstate->c_recursion_remaining >= 0) { return 0; } return _Py_CheckRecursiveCallN(tstate, n, where); @@ -160,7 +160,7 @@ extern PyObject* _Py_MakeCoro(PyFunctionObject *func); extern int _Py_HandlePending(PyThreadState *tstate); -#define C_RECURSION_LIMT 3000 +#define C_RECURSION_LIMT 2500 #ifdef __cplusplus } diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index d7b78f686ef88d..2f2aa8a10d2073 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -111,8 +111,7 @@ def __getitem__(self, key): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_extended_arg(self): - # default: 1000 * 2.5 = 2500 repetitions - repeat = int(sys.getrecursionlimit() * 2.5) + repeat = 2000 longexpr = 'x = x or ' + '-x' * repeat g = {} code = ''' diff --git a/Python/ceval.c b/Python/ceval.c index 2a5e918ce708b7..eba484d07640b7 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -269,7 +269,7 @@ Py_SetRecursionLimit(int new_limit) int _Py_CheckRecursiveCallN(PyThreadState *tstate, int n, const char *where) { - assert(tstate->c_recursion_remaining < n); + assert(tstate->c_recursion_remaining < 0); if (tstate->c_recursion_headroom) { if (tstate->c_recursion_remaining < -50) { /* Overflowing while handling an overflow. Give up. */ From b70c3745fb645a9814572738ae98b4f39872804a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 1 Sep 2022 14:46:17 +0100 Subject: [PATCH 9/9] Reduce amount of C stack more to work on Windows x86. --- Include/internal/pycore_ceval.h | 2 +- Lib/test/test_ast.py | 6 +++--- Lib/test/test_compile.py | 15 ++++----------- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 4b3ec9ea687b5d..4dfe79a9c0eb6b 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -160,7 +160,7 @@ extern PyObject* _Py_MakeCoro(PyFunctionObject *func); extern int _Py_HandlePending(PyThreadState *tstate); -#define C_RECURSION_LIMT 2500 +#define C_RECURSION_LIMT 2000 #ifdef __cplusplus } diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 4cfefe4ac3dd1d..44c04b39051e20 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -816,9 +816,9 @@ def next(self): @support.cpython_only def test_ast_recursion_limit(self): - fail_depth = sys.getrecursionlimit() * 3 - crash_depth = sys.getrecursionlimit() * 300 - success_depth = int(fail_depth * 0.75) + fail_depth = 3000 + crash_depth = 100_000 + success_depth = 1500 def check_limit(prefix, repeated): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 2f2aa8a10d2073..511310b8be925f 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -111,7 +111,7 @@ def __getitem__(self, key): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_extended_arg(self): - repeat = 2000 + repeat = 1500 longexpr = 'x = x or ' + '-x' * repeat g = {} code = ''' @@ -545,16 +545,9 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): - # Expected limit is sys.getrecursionlimit() * the scaling factor - # in symtable.c (currently 3) - # We expect to fail *at* that limit, because we use up some of - # the stack depth limit in the test suite code - # So we check the expected limit and 75% of that - # XXX (ncoghlan): duplicating the scaling factor here is a little - # ugly. Perhaps it should be exposed somewhere... - fail_depth = sys.getrecursionlimit() * 3 - crash_depth = sys.getrecursionlimit() * 300 - success_depth = int(fail_depth * 0.75) + fail_depth = 3000 + crash_depth = 100_000 + success_depth = 1500 def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth