From 8cd30d6ab38dbbf57bf96c290e43afa0bc21b835 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 4 Dec 2017 18:32:57 +0100 Subject: [PATCH 1/4] bpo-20891: Py_Initialize() now creates the GIL The GIL is no longer created "on demand" to fix a race condition when PyGILState_Ensure() is called in a non-Python thread. --- .../2017-12-04-18-34-11.bpo-20891.C2TsfR.rst | 3 +++ Python/pylifecycle.c | 4 ++++ 2 files changed, 7 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-12-04-18-34-11.bpo-20891.C2TsfR.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-12-04-18-34-11.bpo-20891.C2TsfR.rst b/Misc/NEWS.d/next/Core and Builtins/2017-12-04-18-34-11.bpo-20891.C2TsfR.rst new file mode 100644 index 00000000000000..abf9c3c2e390df --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-12-04-18-34-11.bpo-20891.C2TsfR.rst @@ -0,0 +1,3 @@ +Py_Initialize() now creates the GIL. The GIL is no longer created "on demand" +to fix a race condition when PyGILState_Ensure() is called in a non-Python +thread. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index d46784a2f6b709..82ab9154256fda 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -681,9 +681,13 @@ _Py_InitializeCore(const _PyCoreConfig *core_config) Instead we destroy the previously created GIL here, which ensures that we can call Py_Initialize / Py_FinalizeEx multiple times. */ _PyEval_FiniThreads(); + /* Auto-thread-state API */ _PyGILState_Init(interp, tstate); + /* Create the GIL */ + PyEval_InitThreads(); + _Py_ReadyTypes(); if (!_PyFrame_Init()) From ec2686972466eec7fae69b4e441526ec3e2df825 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 4 Dec 2017 19:10:41 +0100 Subject: [PATCH 2/4] Update the documentation Remove also mention of the "thread support", since thread support cannot be disabled anymore. --- Doc/c-api/init.rst | 62 ++++++++++++++++------------------------------ 1 file changed, 22 insertions(+), 40 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 02189a9e35915c..4435b6ba6d9529 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -687,15 +687,14 @@ This is so common that a pair of macros exists to simplify it:: The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the -block. These two macros are still available when Python is compiled without -thread support (they simply have an empty expansion). +block. -When thread support is enabled, the block above expands to the following code:: +The block above expands to the following code:: PyThreadState *_save; _save = PyEval_SaveThread(); - ...Do some blocking I/O operation... + ... Do some blocking I/O operation ... PyEval_RestoreThread(_save); .. index:: @@ -818,36 +817,23 @@ code, or when embedding the Python interpreter: This is a no-op when called for a second time. + .. versionchanged:: 3.7 + The :term:`GIL` is now initialized by :c:func:`Py_Initialize()`. + .. versionchanged:: 3.2 This function cannot be called before :c:func:`Py_Initialize()` anymore. .. index:: module: _thread - .. note:: - - When only the main thread exists, no GIL operations are needed. This is a - common situation (most Python programs do not use threads), and the lock - operations slow the interpreter down a bit. Therefore, the lock is not - created initially. This situation is equivalent to having acquired the lock: - when there is only a single thread, all object accesses are safe. Therefore, - when this function initializes the global interpreter lock, it also acquires - it. Before the Python :mod:`_thread` module creates a new thread, knowing - that either it has the lock or the lock hasn't been created yet, it calls - :c:func:`PyEval_InitThreads`. When this call returns, it is guaranteed that - the lock has been created and that the calling thread has acquired it. - - It is **not** safe to call this function when it is unknown which thread (if - any) currently has the global interpreter lock. - - This function is not available when thread support is disabled at compile time. - .. c:function:: int PyEval_ThreadsInitialized() Returns a non-zero value if :c:func:`PyEval_InitThreads` has been called. This function can be called without holding the GIL, and therefore can be used to - avoid calls to the locking API when running single-threaded. This function is - not available when thread support is disabled at compile time. + avoid calls to the locking API when running single-threaded. + + .. versionchanged:: 3.7 + The :term:`GIL` is now initialized by :c:func:`Py_Initialize()`. .. c:function:: PyThreadState* PyEval_SaveThread() @@ -855,8 +841,7 @@ code, or when embedding the Python interpreter: Release the global interpreter lock (if it has been created and thread support is enabled) and reset the thread state to *NULL*, returning the previous thread state (which is not *NULL*). If the lock has been created, - the current thread must have acquired it. (This function is available even - when thread support is disabled at compile time.) + the current thread must have acquired it. .. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) @@ -864,8 +849,7 @@ code, or when embedding the Python interpreter: Acquire the global interpreter lock (if it has been created and thread support is enabled) and set the thread state to *tstate*, which must not be *NULL*. If the lock has been created, the current thread must not have - acquired it, otherwise deadlock ensues. (This function is available even - when thread support is disabled at compile time.) + acquired it, otherwise deadlock ensues. .. c:function:: PyThreadState* PyThreadState_Get() @@ -957,7 +941,7 @@ example usage in the Python source distribution. This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. Note that it contains an opening brace; it must be matched with a following :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this - macro. It is a no-op when thread support is disabled at compile time. + macro. .. c:macro:: Py_END_ALLOW_THREADS @@ -965,29 +949,29 @@ example usage in the Python source distribution. This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains a closing brace; it must be matched with an earlier :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of - this macro. It is a no-op when thread support is disabled at compile time. + this macro. .. c:macro:: Py_BLOCK_THREADS This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to - :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. It is a no-op when - thread support is disabled at compile time. + :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. .. c:macro:: Py_UNBLOCK_THREADS This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable - declaration. It is a no-op when thread support is disabled at compile time. + declaration. Low-level API ------------- -All of the following functions are only available when thread support is enabled -at compile time, and must be called only when the global interpreter lock has -been created. +All of the following functions must be called after :c:func:`Py_Initialize`. + +.. versionchanged:: 3.7 + :c:func:`Py_Initialize()` now initializes the :term:`GIL`. .. c:function:: PyInterpreterState* PyInterpreterState_New() @@ -1068,8 +1052,7 @@ been created. If this thread already has the lock, deadlock ensues. :c:func:`PyEval_RestoreThread` is a higher-level function which is always - available (even when thread support isn't enabled or when threads have - not been initialized). + available (even when threads have not been initialized). .. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) @@ -1081,8 +1064,7 @@ been created. reported. :c:func:`PyEval_SaveThread` is a higher-level function which is always - available (even when thread support isn't enabled or when threads have - not been initialized). + available (even when threads have not been initialized). .. c:function:: void PyEval_AcquireLock() From 0caeef0b50506bb5ef27f2234d097c9127c09570 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 4 Dec 2017 19:12:28 +0100 Subject: [PATCH 3/4] Simplify PyEval_SaveThread/PyEval_RestoreThread Make the assumption that the GIL is always created (by Py_Initialize()). --- Python/ceval.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 128ec2c0045486..52a42b00724e6f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -254,8 +254,8 @@ PyEval_SaveThread(void) PyThreadState *tstate = PyThreadState_Swap(NULL); if (tstate == NULL) Py_FatalError("PyEval_SaveThread: NULL tstate"); - if (gil_created()) - drop_gil(tstate); + assert(gil_created()); + drop_gil(tstate); return tstate; } @@ -264,17 +264,18 @@ PyEval_RestoreThread(PyThreadState *tstate) { if (tstate == NULL) Py_FatalError("PyEval_RestoreThread: NULL tstate"); - if (gil_created()) { - int err = errno; - take_gil(tstate); - /* _Py_Finalizing is protected by the GIL */ - if (_Py_IsFinalizing() && !_Py_CURRENTLY_FINALIZING(tstate)) { - drop_gil(tstate); - PyThread_exit_thread(); - Py_UNREACHABLE(); - } - errno = err; + assert(gil_created()); + + int err = errno; + take_gil(tstate); + /* _Py_Finalizing is protected by the GIL */ + if (_Py_IsFinalizing() && !_Py_CURRENTLY_FINALIZING(tstate)) { + drop_gil(tstate); + PyThread_exit_thread(); + Py_UNREACHABLE(); } + errno = err; + PyThreadState_Swap(tstate); } From b8b3a1d73ed3d793cd04b66a0a1c9fe2755e70f5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 29 Jan 2018 11:40:04 +0100 Subject: [PATCH 4/4] doc: rephrase versionchanged --- Doc/c-api/init.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 4435b6ba6d9529..bae49d5ba8125e 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -818,7 +818,8 @@ code, or when embedding the Python interpreter: This is a no-op when called for a second time. .. versionchanged:: 3.7 - The :term:`GIL` is now initialized by :c:func:`Py_Initialize()`. + This function is now called by :c:func:`Py_Initialize()`, so you don't + have to call it yourself anymore. .. versionchanged:: 3.2 This function cannot be called before :c:func:`Py_Initialize()` anymore.