From 18f16d3443ee4a74c458a4b8dd533f9865e6b75f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 30 Jul 2023 04:55:32 +0100 Subject: [PATCH 01/10] Increase C stack limit for most functions, except PyEval_EvalDefault() --- Include/cpython/pystate.h | 2 +- Lib/test/list_tests.py | 2 +- Lib/test/mapping_tests.py | 2 +- Lib/test/support/__init__.py | 4 +++- Lib/test/test_compile.py | 21 ++++++++------------- Lib/test/test_dict.py | 2 +- Lib/test/test_dictviews.py | 3 ++- Lib/test/test_exception_group.py | 2 +- Parser/asdl_c.py | 2 +- Python/Python-ast.c | 2 +- Python/ast.c | 2 +- Python/ast_opt.c | 2 +- Python/bytecodes.c | 2 +- Python/ceval.c | 8 +++++++- Python/generated_cases.c.h | 2 +- Python/symtable.c | 11 ++--------- 16 files changed, 33 insertions(+), 36 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 30de4ee4b6c58c..17bb1ff7cee8a3 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -222,7 +222,7 @@ struct _ts { # ifdef __wasi__ # define C_RECURSION_LIMIT 500 # else -# define C_RECURSION_LIMIT 800 +# define C_RECURSION_LIMIT 2500 # endif #endif diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index fe3ee80b8d461f..0d99b6072d5a1c 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -61,7 +61,7 @@ def test_repr(self): def test_repr_deep(self): a = self.type2test([]) - for i in range(sys.getrecursionlimit() + 100): + for i in range(3_000): a = self.type2test([a]) self.assertRaises(RecursionError, repr, a) diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index 613206a0855aea..2d65b91f2f034b 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -624,7 +624,7 @@ def __repr__(self): def test_repr_deep(self): d = self._empty_mapping() - for i in range(sys.getrecursionlimit() + 100): + for i in range(3_000): d0 = d d = self._empty_mapping() d[1] = d0 diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 3b332f49951f0c..39a805baa22a2f 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -64,7 +64,7 @@ "run_with_tz", "PGO", "missing_compiler_executable", "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST", "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT", - "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", + "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", "C_RECURSION_LIMIT", ] @@ -2460,3 +2460,5 @@ def adjust_int_max_str_digits(max_digits): #For recursion tests, easily exceeds default recursion limit EXCEEDS_RECURSION_LIMIT = 5000 + +C_RECURSION_LIMIT = 2500 diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 85ce0a4b39d854..d3094cd89ed961 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -11,10 +11,9 @@ import warnings from test import support from test.support import (script_helper, requires_debug_ranges, - requires_specialization) + requires_specialization, C_RECURSION_LIMIT) from test.support.os_helper import FakePath - class TestSpecifics(unittest.TestCase): def compile_single(self, source): @@ -112,7 +111,7 @@ def __getitem__(self, key): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_extended_arg(self): - repeat = 2000 + repeat = int(C_RECURSION_LIMIT * 0.9) longexpr = 'x = x or ' + '-x' * repeat g = {} code = textwrap.dedent(''' @@ -558,16 +557,12 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): - # Expected limit is sys.getrecursionlimit() * the scaling factor - # in symtable.c (currently 3) - # We expect to fail *at* that limit, because we use up some of - # the stack depth limit in the test suite code - # So we check the expected limit and 75% of that - # XXX (ncoghlan): duplicating the scaling factor here is a little - # ugly. Perhaps it should be exposed somewhere... - fail_depth = sys.getrecursionlimit() * 3 - crash_depth = sys.getrecursionlimit() * 300 - success_depth = int(fail_depth * 0.75) + # Expected limit is C_RECURSION_LIMIT + # Duplicating the limit here is a little ugly. + # Perhaps it should be exposed somewhere... + fail_depth = C_RECURSION_LIMIT + 1 + crash_depth = C_RECURSION_LIMIT * 100 + success_depth = int(C_RECURSION_LIMIT * 0.9) def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 79638340059f65..59ecb38730d3a8 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -596,7 +596,7 @@ def __repr__(self): def test_repr_deep(self): d = {} - for i in range(sys.getrecursionlimit() + 100): + for i in range(3_000): d = {1: d} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index 924f4a6829e19c..2bd9d6eef8cfc6 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -3,6 +3,7 @@ import pickle import sys import unittest +from test.support import C_RECURSION_LIMIT class DictSetTest(unittest.TestCase): @@ -279,7 +280,7 @@ def test_recursive_repr(self): def test_deeply_nested_repr(self): d = {} - for i in range(sys.getrecursionlimit() + 100): + for i in range(C_RECURSION_LIMIT//2 + 100): d = {42: d.values()} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 2658e027ff3e2b..971aff31810444 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -460,7 +460,7 @@ def test_basics_split_by_predicate__match(self): class DeepRecursionInSplitAndSubgroup(unittest.TestCase): def make_deep_eg(self): e = TypeError(1) - for i in range(2000): + for i in range(3_000): e = ExceptionGroup('eg', [e]) return e diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index f159b573ce4727..902d1813a9b02c 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1393,7 +1393,7 @@ class PartingShots(StaticVisitor): int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 3; + int COMPILER_STACK_FRAME_SCALE = 1; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return 0; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 412de79397477b..b425d66ba79b0b 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -13073,7 +13073,7 @@ PyObject* PyAST_mod2obj(mod_ty t) int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 3; + int COMPILER_STACK_FRAME_SCALE = 1; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return 0; diff --git a/Python/ast.c b/Python/ast.c index 68600ce683b974..10a11d234b83d1 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1029,7 +1029,7 @@ validate_type_params(struct validator *state, asdl_type_param_seq *tps) /* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 3 +#define COMPILER_STACK_FRAME_SCALE 1 int _PyAST_Validate(mod_ty mod) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index ad1e312b084b74..fb6b2e0fd45757 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -1112,7 +1112,7 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat #undef CALL_SEQ /* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 3 +#define COMPILER_STACK_FRAME_SCALE 1 int _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0bea7b59597a26..90e26d3c86b380 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -741,7 +741,7 @@ dummy_func( tstate->cframe = cframe.previous; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return retval; } diff --git a/Python/ceval.c b/Python/ceval.c index 17818a0d3c17e6..9c2936f5fae4f6 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -624,6 +624,11 @@ extern const struct _PyCode_DEF(8) _Py_InitCleanup; # pragma warning(disable:4102) #endif + +/* _PyEval_EvalFrameDefault() is a *big* function, + * so consume 3 units of C stack */ +#define PY_EVAL_C_STACK_UNITS 3 + PyObject* _Py_HOT_FUNCTION _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { @@ -676,6 +681,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int frame->previous = &entry_frame; cframe.current_frame = frame; + tstate->c_recursion_remaining -= (PY_EVAL_C_STACK_UNITS - 1); if (_Py_EnterRecursiveCallTstate(tstate, "")) { tstate->c_recursion_remaining--; tstate->py_recursion_remaining--; @@ -907,7 +913,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* Restore previous cframe and exit */ tstate->cframe = cframe.previous; assert(tstate->cframe->current_frame == frame->previous); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return NULL; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e43b3dedf14f5d..9fa549a1203245 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -903,7 +903,7 @@ tstate->cframe = cframe.previous; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return retval; } diff --git a/Python/symtable.c b/Python/symtable.c index 04be3192d6c7c4..eab66671bf9dfc 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -282,17 +282,10 @@ symtable_new(void) return NULL; } -/* When compiling the use of C stack is probably going to be a lot - lighter than when executing Python code but still can overflow - and causing a Python crash if not checked (e.g. eval("()"*300000)). - Using the current recursion limit for the compiler seems too - restrictive (it caused at least one test to fail) so a factor is - used to allow deeper recursion when compiling an expression. - - Using a scaling factor means this should automatically adjust when +/* Using a scaling factor means this should automatically adjust when the recursion limit is adjusted for small or large C stack allocations. */ -#define COMPILER_STACK_FRAME_SCALE 3 +#define COMPILER_STACK_FRAME_SCALE 1 struct symtable * _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) From 8d05620be5252dc84fac1fb675dae7f0501ef1f9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 30 Jul 2023 05:47:58 +0100 Subject: [PATCH 02/10] Add news and what's new item. --- Doc/whatsnew/3.12.rst | 5 +++++ .../2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst | 3 +++ 2 files changed, 8 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 3c39476ced9f48..6553013552d003 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -801,6 +801,11 @@ sys exception instance, rather than to a ``(typ, exc, tb)`` tuple. (Contributed by Irit Katriel in :gh:`103176`.) +* :func:`sys.setrecursionlimit` and :func:`sys.getrecursionlimit`. + The recursion limit now applies only to Python code. Builtin functions do + not use the recursion limit, but are protected by a different mechanism + that prevents recursion from causing a virtual machine crash. + tempfile -------- diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst new file mode 100644 index 00000000000000..a92dd920bc9c91 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst @@ -0,0 +1,3 @@ +Increase C recursion limit for functions other than the main interpreter +from 800 to 2500. This should allow functions like ``list.__repr__`` and +``json.dumps`` to handle all the inputs that they could prior to 3.12 From aa78add6255778390d6a23437fabb123d29f363f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 30 Jul 2023 06:01:33 +0100 Subject: [PATCH 03/10] Reduce C_RECURSION_LIMIT to 2400. 2500 was too high. --- Include/cpython/pystate.h | 2 +- Lib/test/support/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 17bb1ff7cee8a3..11ed070b846718 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -222,7 +222,7 @@ struct _ts { # ifdef __wasi__ # define C_RECURSION_LIMIT 500 # else -# define C_RECURSION_LIMIT 2500 +# define C_RECURSION_LIMIT 2400 # endif #endif diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 39a805baa22a2f..b6bded5f417662 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2461,4 +2461,4 @@ def adjust_int_max_str_digits(max_digits): #For recursion tests, easily exceeds default recursion limit EXCEEDS_RECURSION_LIMIT = 5000 -C_RECURSION_LIMIT = 2500 +C_RECURSION_LIMIT = 2400 From d6793eac7fbd6a06ef7828bf03f26cf488c8177c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 30 Jul 2023 07:16:33 +0100 Subject: [PATCH 04/10] Reduce C recursion limit to 2000. 2400 is too hgh for Windows. --- Include/cpython/pystate.h | 3 ++- Lib/test/support/__init__.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 11ed070b846718..389f317148366e 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -222,7 +222,8 @@ struct _ts { # ifdef __wasi__ # define C_RECURSION_LIMIT 500 # else -# define C_RECURSION_LIMIT 2400 + // This value is duplicated in Lib/test/support/__init__.py +# define C_RECURSION_LIMIT 2000 # endif #endif diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index b6bded5f417662..2def5f6db1f4aa 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2461,4 +2461,5 @@ def adjust_int_max_str_digits(max_digits): #For recursion tests, easily exceeds default recursion limit EXCEEDS_RECURSION_LIMIT = 5000 -C_RECURSION_LIMIT = 2400 +# The default C recursion limit (from Include/cpython/pystate.h). +C_RECURSION_LIMIT = 2000 From c0af892704652637f3fbcf4213d5f2f12a35d638 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 30 Jul 2023 07:23:23 +0100 Subject: [PATCH 05/10] Use symbolic limit rather than hard coded value for recursion tests. --- Lib/test/list_tests.py | 4 ++-- Lib/test/mapping_tests.py | 3 ++- Lib/test/test_dict.py | 4 ++-- Lib/test/test_exception_group.py | 4 ++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index 0d99b6072d5a1c..b1ef332522d2ce 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -6,7 +6,7 @@ from functools import cmp_to_key from test import seq_tests -from test.support import ALWAYS_EQ, NEVER_EQ +from test.support import ALWAYS_EQ, NEVER_EQ, C_RECURSION_LIMIT class CommonTest(seq_tests.CommonTest): @@ -61,7 +61,7 @@ def test_repr(self): def test_repr_deep(self): a = self.type2test([]) - for i in range(3_000): + for i in range(C_RECURSION_LIMIT + 1): a = self.type2test([a]) self.assertRaises(RecursionError, repr, a) diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index 2d65b91f2f034b..5492bbf86d1f87 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -2,6 +2,7 @@ import unittest import collections import sys +from test.support import C_RECURSION_LIMIT class BasicTestMappingProtocol(unittest.TestCase): @@ -624,7 +625,7 @@ def __repr__(self): def test_repr_deep(self): d = self._empty_mapping() - for i in range(3_000): + for i in range(C_RECURSION_LIMIT + 1): d0 = d d = self._empty_mapping() d[1] = d0 diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 59ecb38730d3a8..fbc6ce8282de3c 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -8,7 +8,7 @@ import unittest import weakref from test import support -from test.support import import_helper +from test.support import import_helper, C_RECURSION_LIMIT class DictTest(unittest.TestCase): @@ -596,7 +596,7 @@ def __repr__(self): def test_repr_deep(self): d = {} - for i in range(3_000): + for i in range(C_RECURSION_LIMIT + 1): d = {1: d} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 971aff31810444..a02d54da35e948 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,7 +1,7 @@ import collections.abc import types import unittest - +from test.support import C_RECURSION_LIMIT class TestExceptionGroupTypeHierarchy(unittest.TestCase): def test_exception_group_types(self): @@ -460,7 +460,7 @@ def test_basics_split_by_predicate__match(self): class DeepRecursionInSplitAndSubgroup(unittest.TestCase): def make_deep_eg(self): e = TypeError(1) - for i in range(3_000): + for i in range(C_RECURSION_LIMIT + 1): e = ExceptionGroup('eg', [e]) return e From 534a36cd3bf3c52b96e859475b5e7e3bc5054479 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 30 Jul 2023 14:53:10 +0100 Subject: [PATCH 06/10] Reduce C recursion limit to 1500, set cost of eval loop to 2 frames, and compiler mutliply to 2. --- Include/cpython/pystate.h | 2 +- Lib/test/support/__init__.py | 2 +- Lib/test/test_compile.py | 4 ++-- Parser/asdl_c.py | 2 +- Python/Python-ast.c | 2 +- Python/ast.c | 2 +- Python/ast_opt.c | 2 +- Python/ceval.c | 2 +- Python/symtable.c | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 389f317148366e..56e473cc5e42d5 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -223,7 +223,7 @@ struct _ts { # define C_RECURSION_LIMIT 500 # else // This value is duplicated in Lib/test/support/__init__.py -# define C_RECURSION_LIMIT 2000 +# define C_RECURSION_LIMIT 1500 # endif #endif diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 2def5f6db1f4aa..ba6827c49f5d25 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2462,4 +2462,4 @@ def adjust_int_max_str_digits(max_digits): EXCEEDS_RECURSION_LIMIT = 5000 # The default C recursion limit (from Include/cpython/pystate.h). -C_RECURSION_LIMIT = 2000 +C_RECURSION_LIMIT = 1500 diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index d3094cd89ed961..4815d1f30691cf 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -560,9 +560,9 @@ def test_compiler_recursion_limit(self): # Expected limit is C_RECURSION_LIMIT # Duplicating the limit here is a little ugly. # Perhaps it should be exposed somewhere... - fail_depth = C_RECURSION_LIMIT + 1 + fail_depth = C_RECURSION_LIMIT * 2 + 1 crash_depth = C_RECURSION_LIMIT * 100 - success_depth = int(C_RECURSION_LIMIT * 0.9) + success_depth = int(C_RECURSION_LIMIT * 1.8) def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 902d1813a9b02c..2a36610527f898 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1393,7 +1393,7 @@ class PartingShots(StaticVisitor): int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 1; + int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return 0; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index b425d66ba79b0b..8047b1259c5d86 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -13073,7 +13073,7 @@ PyObject* PyAST_mod2obj(mod_ty t) int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 1; + int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return 0; diff --git a/Python/ast.c b/Python/ast.c index 10a11d234b83d1..74c97f948d15e6 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1029,7 +1029,7 @@ validate_type_params(struct validator *state, asdl_type_param_seq *tps) /* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 1 +#define COMPILER_STACK_FRAME_SCALE 2 int _PyAST_Validate(mod_ty mod) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index fb6b2e0fd45757..82e7559e5b629a 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -1112,7 +1112,7 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat #undef CALL_SEQ /* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 1 +#define COMPILER_STACK_FRAME_SCALE 2 int _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features) diff --git a/Python/ceval.c b/Python/ceval.c index 9c2936f5fae4f6..c780cfebe8f6d2 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -627,7 +627,7 @@ extern const struct _PyCode_DEF(8) _Py_InitCleanup; /* _PyEval_EvalFrameDefault() is a *big* function, * so consume 3 units of C stack */ -#define PY_EVAL_C_STACK_UNITS 3 +#define PY_EVAL_C_STACK_UNITS 2 PyObject* _Py_HOT_FUNCTION _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) diff --git a/Python/symtable.c b/Python/symtable.c index eab66671bf9dfc..e9adbd5d29b1f9 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -285,7 +285,7 @@ symtable_new(void) /* Using a scaling factor means this should automatically adjust when the recursion limit is adjusted for small or large C stack allocations. */ -#define COMPILER_STACK_FRAME_SCALE 1 +#define COMPILER_STACK_FRAME_SCALE 2 struct symtable * _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) From 3bb0cf0b63116de2acf7e1d7025b4991a6272b93 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 30 Jul 2023 14:59:18 +0100 Subject: [PATCH 07/10] Fix comment. --- Lib/test/test_compile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 4815d1f30691cf..770184c5ef9a91 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -557,7 +557,7 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): - # Expected limit is C_RECURSION_LIMIT + # Expected limit is C_RECURSION_LIMIT * 2 # Duplicating the limit here is a little ugly. # Perhaps it should be exposed somewhere... fail_depth = C_RECURSION_LIMIT * 2 + 1 From a8c72d0e3cdfde0ce67e6fe9491f2d5a2010a947 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 30 Jul 2023 15:00:30 +0100 Subject: [PATCH 08/10] Fix news entry --- .../2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst index a92dd920bc9c91..fb0940b456dae5 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst @@ -1,3 +1,3 @@ Increase C recursion limit for functions other than the main interpreter -from 800 to 2500. This should allow functions like ``list.__repr__`` and +from 800 to 1500. This should allow functions like ``list.__repr__`` and ``json.dumps`` to handle all the inputs that they could prior to 3.12 From 7e6a5215e8c46d08703e7a6a971774c789f9a892 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 30 Jul 2023 17:26:03 +0100 Subject: [PATCH 09/10] Skip test for WASI due to limited stack depth --- Lib/test/test_ast.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index a03fa4c7187b05..5346b39043f0f5 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1084,6 +1084,7 @@ def next(self): return self enum._test_simple_enum(_Precedence, ast._Precedence) + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") @support.cpython_only def test_ast_recursion_limit(self): fail_depth = support.EXCEEDS_RECURSION_LIMIT From 0e8c992d80495511e1971a3c5fc63c2943f2fdb2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 31 Jul 2023 09:05:06 +0100 Subject: [PATCH 10/10] Skip test on s390x --- Lib/test/support/__init__.py | 5 +++++ Lib/test/test_call.py | 3 ++- Lib/test/test_zlib.py | 7 ++----- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index ba6827c49f5d25..c3c3cf0a71596c 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -65,6 +65,7 @@ "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST", "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT", "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", "C_RECURSION_LIMIT", + "skip_on_s390x", ] @@ -2463,3 +2464,7 @@ def adjust_int_max_str_digits(max_digits): # The default C recursion limit (from Include/cpython/pystate.h). C_RECURSION_LIMIT = 1500 + +#Windows doesn't have os.uname() but it doesn't support s390x. +skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', + 'skipped on s390x') diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 09a531f8cc627b..c3c3b1853b5736 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,5 +1,5 @@ import unittest -from test.support import cpython_only, requires_limited_api +from test.support import cpython_only, requires_limited_api, skip_on_s390x try: import _testcapi except ImportError: @@ -918,6 +918,7 @@ def test_multiple_values(self): @cpython_only class TestRecursion(unittest.TestCase): + @skip_on_s390x def test_super_deep(self): def recurse(n): diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 3dac70eb12852c..2113757254c0ed 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -7,7 +7,7 @@ import pickle import random import sys -from test.support import bigmemtest, _1G, _4G +from test.support import bigmemtest, _1G, _4G, skip_on_s390x zlib = import_helper.import_module('zlib') @@ -44,10 +44,7 @@ # zlib.decompress(func1(data)) == zlib.decompress(func2(data)) == data # # Make the assumption that s390x always has an accelerator to simplify the skip -# condition. Windows doesn't have os.uname() but it doesn't support s390x. -skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', - 'skipped on s390x') - +# condition. class VersionTestCase(unittest.TestCase):