gh-120321: Lock gen_send

Lib/test/test_free_threading/test_generators.py

@@ -0,0 +1,23 @@
+import unittest
+import concurrent.futures
+
+from unittest import TestCase
+
+from test.support import threading_helper
+
+@threading_helper.requires_working_threading()
+class TestGen(TestCase):
+    def test_generators_basic(self):
+        def gen():
+            for _ in range(5000):
+                yield
+
+
+        it = gen()
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            for _ in range(5000):
+                executor.submit(lambda: next(it))
+
+
+if __name__ == "__main__":
+    unittest.main()

Misc/NEWS.d/next/Core and Builtins/2024-06-10-18-40-04.gh-issue-120321.O4UZUe.rst

@@ -0,0 +1 @@
+Fix generator ``send`` crashing on free-threaded builds. Thanks to ``rostan-t`` for the reproducer.

Objects/genobject.c

@@ -14,6 +14,7 @@
 #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_*
 #include "pycore_pyerrors.h"      // _PyErr_ClearExcState()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
+#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION
 
 #include "pystats.h"
 
@@ -158,7 +159,7 @@ gen_dealloc(PyGenObject *gen)
 }
 
 static PySendResult
-gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult,
+gen_send_ex2_unlocked(PyGenObject *gen, PyObject *arg, PyObject **presult,
              int exc, int closing)
 {
     PyThreadState *tstate = _PyThreadState_GET();
@@ -256,6 +257,17 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult,
     return result ? PYGEN_RETURN : PYGEN_ERROR;
 }
 
+static PySendResult
+gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult,
+             int exc, int closing)
+{
+    PySendResult res;
+    Py_BEGIN_CRITICAL_SECTION(gen);
+    res = gen_send_ex2_unlocked(gen, arg, presult, exc, closing);
+    Py_END_CRITICAL_SECTION();
+    return res;
+}
+
 static PySendResult
 PyGen_am_send(PyGenObject *gen, PyObject *arg, PyObject **result)
 {

Python/bytecodes.c

@@ -1117,14 +1117,17 @@ dummy_func(
                 ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING)
             {
                 PyGenObject *gen = (PyGenObject *)receiver_o;
-                _PyInterpreterFrame *gen_frame = &gen->gi_iframe;
+                _PyInterpreterFrame *gen_frame;
+                Py_BEGIN_CRITICAL_SECTION(gen);
+                gen_frame = &gen->gi_iframe;
                 STACK_SHRINK(1);
                 _PyFrame_StackPush(gen_frame, v);
                 gen->gi_frame_state = FRAME_EXECUTING;
                 gen->gi_exc_state.previous_item = tstate->exc_info;
                 tstate->exc_info = &gen->gi_exc_state;
                 assert(next_instr - this_instr + oparg <= UINT16_MAX);
                 frame->return_offset = (uint16_t)(next_instr - this_instr + oparg);
+                Py_END_CRITICAL_SECTION();
                 DISPATCH_INLINED(gen_frame);
             }
             if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) {