diff --git a/Lib/test/test_free_threading/test_generators.py b/Lib/test/test_free_threading/test_generators.py new file mode 100644 index 00000000000000..999cb189be15b4 --- /dev/null +++ b/Lib/test/test_free_threading/test_generators.py @@ -0,0 +1,23 @@ +import unittest +import concurrent.futures + +from unittest import TestCase + +from test.support import threading_helper + +@threading_helper.requires_working_threading() +class TestGen(TestCase): + def test_generators_basic(self): + def gen(): + for _ in range(5000): + yield + + + it = gen() + with concurrent.futures.ThreadPoolExecutor() as executor: + for _ in range(5000): + executor.submit(lambda: next(it)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-10-18-40-04.gh-issue-120321.O4UZUe.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-18-40-04.gh-issue-120321.O4UZUe.rst new file mode 100644 index 00000000000000..c2107256f9883c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-18-40-04.gh-issue-120321.O4UZUe.rst @@ -0,0 +1 @@ +Fix generator ``send`` crashing on free-threaded builds. Thanks to ``rostan-t`` for the reproducer. diff --git a/Objects/genobject.c b/Objects/genobject.c index 37b40530589d57..02aa0471a61ac2 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -14,6 +14,7 @@ #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION #include "pystats.h" @@ -158,7 +159,7 @@ gen_dealloc(PyGenObject *gen) } static PySendResult -gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, +gen_send_ex2_unlocked(PyGenObject *gen, PyObject *arg, PyObject **presult, int exc, int closing) { PyThreadState *tstate = _PyThreadState_GET(); @@ -256,6 +257,17 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, return result ? PYGEN_RETURN : PYGEN_ERROR; } +static PySendResult +gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, + int exc, int closing) +{ + PySendResult res; + Py_BEGIN_CRITICAL_SECTION(gen); + res = gen_send_ex2_unlocked(gen, arg, presult, exc, closing); + Py_END_CRITICAL_SECTION(); + return res; +} + static PySendResult PyGen_am_send(PyGenObject *gen, PyObject *arg, PyObject **result) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 84241c64ffae88..a12c897126b470 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1117,7 +1117,9 @@ dummy_func( ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) { PyGenObject *gen = (PyGenObject *)receiver_o; - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + _PyInterpreterFrame *gen_frame; + Py_BEGIN_CRITICAL_SECTION(gen); + gen_frame = &gen->gi_iframe; STACK_SHRINK(1); _PyFrame_StackPush(gen_frame, v); gen->gi_frame_state = FRAME_EXECUTING; @@ -1125,6 +1127,7 @@ dummy_func( tstate->exc_info = &gen->gi_exc_state; assert(next_instr - this_instr + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + Py_END_CRITICAL_SECTION(); DISPATCH_INLINED(gen_frame); } if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 61057221291c0a..f480d334bafd9e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5768,7 +5768,9 @@ ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) { PyGenObject *gen = (PyGenObject *)receiver_o; - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + _PyInterpreterFrame *gen_frame; + Py_BEGIN_CRITICAL_SECTION(gen); + gen_frame = &gen->gi_iframe; STACK_SHRINK(1); _PyFrame_StackPush(gen_frame, v); gen->gi_frame_state = FRAME_EXECUTING; @@ -5776,6 +5778,7 @@ tstate->exc_info = &gen->gi_exc_state; assert(next_instr - this_instr + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); + Py_END_CRITICAL_SECTION(); DISPATCH_INLINED(gen_frame); } if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) {