From 06cb695541ad3be2b58d575dcdbae62795947094 Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Thu, 3 Jan 2019 23:51:08 +0100 Subject: [PATCH 1/2] Stackless issue #197: stackless call "asyncio._CTask" Enable stackless calls of coroutines wrapped in "asyncio._CTask", if soft-switching is enabled. Needs test cases. Depends on pull request #198. --- Modules/_asynciomodule.c | 143 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 136 insertions(+), 7 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index a5cf687d8893ca..8654e68d8161d9 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -1,5 +1,7 @@ +//#define STACKLESS_OFF #include "Python.h" #include "structmember.h" +#include "stackless_api.h" /*[clinic input] @@ -1685,6 +1687,8 @@ static PyObject * TaskStepMethWrapper_call(TaskStepMethWrapper *o, PyObject *args, PyObject *kwds) { + STACKLESS_GETARG(); + if (kwds != NULL && PyDict_GET_SIZE(kwds) != 0) { PyErr_SetString(PyExc_TypeError, "function takes no keyword arguments"); return NULL; @@ -1693,7 +1697,10 @@ TaskStepMethWrapper_call(TaskStepMethWrapper *o, PyErr_SetString(PyExc_TypeError, "function takes no positional arguments"); return NULL; } - return task_step(o->sw_task, o->sw_arg); + STACKLESS_PROMOTE_ALL(); + PyObject * result = task_step(o->sw_task, o->sw_arg); + STACKLESS_ASSERT(); + return result; } static int @@ -1720,6 +1727,12 @@ static PyGetSetDef TaskStepMethWrapper_getsetlist[] = { {NULL} /* Sentinel */ }; +#ifdef STACKLESS +static PyMappingMethods TaskStepMethWrapper_as_mapping = { + .slpflags.tp_call = -1, +}; +#endif + PyTypeObject TaskStepMethWrapper_Type = { PyVarObject_HEAD_INIT(NULL, 0) "TaskStepMethWrapper", @@ -1729,9 +1742,12 @@ PyTypeObject TaskStepMethWrapper_Type = { .tp_dealloc = (destructor)TaskStepMethWrapper_dealloc, .tp_call = (ternaryfunc)TaskStepMethWrapper_call, .tp_getattro = PyObject_GenericGetAttr, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_STACKLESS_EXTENSION, .tp_traverse = (traverseproc)TaskStepMethWrapper_traverse, .tp_clear = (inquiry)TaskStepMethWrapper_clear, +#ifdef STACKLESS + .tp_as_mapping = &TaskStepMethWrapper_as_mapping, +#endif }; static PyObject * @@ -1759,6 +1775,7 @@ static PyObject * TaskWakeupMethWrapper_call(TaskWakeupMethWrapper *o, PyObject *args, PyObject *kwds) { + STACKLESS_GETARG(); PyObject *fut; if (kwds != NULL && PyDict_GET_SIZE(kwds) != 0) { @@ -1769,7 +1786,10 @@ TaskWakeupMethWrapper_call(TaskWakeupMethWrapper *o, return NULL; } - return task_wakeup(o->ww_task, fut); + STACKLESS_PROMOTE_ALL(); + PyObject * result = task_wakeup(o->ww_task, fut); + STACKLESS_ASSERT(); + return result; } static int @@ -1795,6 +1815,12 @@ TaskWakeupMethWrapper_dealloc(TaskWakeupMethWrapper *o) Py_TYPE(o)->tp_free(o); } +#ifdef STACKLESS +static PyMappingMethods TaskWakeupMethWrapper_as_mapping = { + .slpflags.tp_call = -1, +}; +#endif + PyTypeObject TaskWakeupMethWrapper_Type = { PyVarObject_HEAD_INIT(NULL, 0) "TaskWakeupMethWrapper", @@ -1803,9 +1829,12 @@ PyTypeObject TaskWakeupMethWrapper_Type = { .tp_dealloc = (destructor)TaskWakeupMethWrapper_dealloc, .tp_call = (ternaryfunc)TaskWakeupMethWrapper_call, .tp_getattro = PyObject_GenericGetAttr, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_STACKLESS_EXTENSION, .tp_traverse = (traverseproc)TaskWakeupMethWrapper_traverse, .tp_clear = (inquiry)TaskWakeupMethWrapper_clear, +#ifdef STACKLESS + .tp_as_mapping = &TaskWakeupMethWrapper_as_mapping, +#endif }; static PyObject * @@ -2468,14 +2497,70 @@ task_set_error_soon(TaskObj *task, PyObject *et, const char *format, ...) Py_RETURN_NONE; } +#ifdef STACKLESS +static PyObject * +task_step_impl_part1(TaskObj *task, PyObject *exc, int *failed); +static PyObject * +task_step_impl_part2(TaskObj *task, PyObject *result); +static PyObject * +task_step_tail(TaskObj *task, PyObject *res); + +static PyObject * +task_step_impl_stackless(PyObject *retval, long *step, PyObject **ob1, PyObject **ob2, PyObject **ob3, long *n, void **any) +{ + Py_XINCREF(retval); + assert(*ob1); + assert(PyObject_TypeCheck(*ob1, &TaskType)); + TaskObj *task = (TaskObj *)(*ob1); + int failed = 0; + + switch (*step) { + case 0: + (*step)++; + Py_XSETREF(retval, task_step_impl_part1(task, *ob2, &failed)); + if (STACKLESS_UNWINDING(retval)) + return retval; + /* no break */ + case 1: + break; + default: + PyErr_SetString(PyExc_SystemError, "invalid state"); + Py_XDECREF(retval); + return NULL; + } + + STACKLESS_RETRACT(); + if (!failed) + retval = task_step_impl_part2(task, retval); + if (n) + retval = task_step_tail(task, retval); + return retval; +} + +static PyStacklessFunctionDeclarationObject task_step_impl_declaration = { + PyObject_HEAD_INIT(NULL) + task_step_impl_stackless, + "task_step_impl_stackless" +}; +#endif + static PyObject * task_step_impl(TaskObj *task, PyObject *exc) { +#ifdef STACKLESS + return PyStackless_CallFunction(&task_step_impl_declaration, Py_None, + (PyObject *)task, exc, NULL, 0, NULL); +} + +static PyObject * +task_step_impl_part1(TaskObj *task, PyObject *exc, int *failed) +{ +#endif + STACKLESS_GETARG(); int res; int clear_exc = 0; PyObject *result = NULL; PyObject *coro; - PyObject *o; if (task->task_state != STATE_PENDING) { PyErr_Format(asyncio_InvalidStateError, @@ -2523,14 +2608,17 @@ task_step_impl(TaskObj *task, PyObject *exc) if (exc == NULL) { if (PyGen_CheckExact(coro) || PyCoro_CheckExact(coro)) { + STACKLESS_PROMOTE_ALL(); result = _PyGen_Send((PyGenObject*)coro, Py_None); } else { + STACKLESS_PROMOTE_ALL(); result = _PyObject_CallMethodIdObjArgs(coro, &PyId_send, Py_None, NULL); } } else { + STACKLESS_PROMOTE_ALL(); result = _PyObject_CallMethodIdObjArgs(coro, &PyId_throw, exc, NULL); if (clear_exc) { @@ -2538,6 +2626,24 @@ task_step_impl(TaskObj *task, PyObject *exc) Py_DECREF(exc); } } + STACKLESS_ASSERT(); + +#ifdef STACKLESS + return result; + +fail: + *failed = 1; + Py_XDECREF(result); + return NULL; +} + + +static PyObject * +task_step_impl_part2(TaskObj *task, PyObject *result) +{ + int res; +#endif + PyObject *o; if (result == NULL) { PyObject *et, *ev, *tb; @@ -2830,8 +2936,18 @@ task_step(TaskObj *task, PyObject *exc) return NULL; } +#ifndef STACKLESS res = task_step_impl(task, exc); +#else + res = PyStackless_CallFunction(&task_step_impl_declaration, Py_None, + (PyObject *)task, exc, NULL, 1, NULL); + return res; +} +static PyObject * +task_step_tail(TaskObj *task, PyObject *res) +{ +#endif if (res == NULL) { PyObject *et, *ev, *tb; PyErr_Fetch(&et, &ev, &tb); @@ -2853,6 +2969,7 @@ task_step(TaskObj *task, PyObject *exc) static PyObject * task_wakeup(TaskObj *task, PyObject *o) { + STACKLESS_GETARG(); PyObject *et, *ev, *tb; PyObject *result; assert(o); @@ -2867,7 +2984,10 @@ task_wakeup(TaskObj *task, PyObject *o) break; /* exception raised */ case 0: Py_DECREF(fut_result); - return task_step(task, NULL); + STACKLESS_PROMOTE_ALL(); + result = task_step(task, NULL); + STACKLESS_ASSERT(); + return result; default: assert(res == 1); result = task_step(task, fut_result); @@ -2879,7 +2999,10 @@ task_wakeup(TaskObj *task, PyObject *o) PyObject *fut_result = PyObject_CallMethod(o, "result", NULL); if (fut_result != NULL) { Py_DECREF(fut_result); - return task_step(task, NULL); + STACKLESS_PROMOTE_ALL(); + result = task_step(task, NULL); + STACKLESS_ASSERT(); + return result; } /* exception raised */ } @@ -3343,5 +3466,11 @@ PyInit__asyncio(void) return NULL; } +#ifdef STACKLESS + if (PyStackless_InitFunctionDeclaration(&task_step_impl_declaration, m, &_asynciomodule) < 0) { + return NULL; + } +#endif + return m; } From d6330bce08a56e41f095bef75dab7487cf0badab Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Sun, 6 Jan 2019 13:30:33 +0100 Subject: [PATCH 2/2] Stackless issue #199: stackless call "asyncio._CTask" Remove an unused static function, add test cases and change-log. --- Modules/_asynciomodule.c | 16 ++++------ Stackless/changelog.txt | 7 +++++ Stackless/unittests/test_generator.py | 44 +++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 10 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 8654e68d8161d9..43df7a3ec0b276 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2540,23 +2540,19 @@ task_step_impl_stackless(PyObject *retval, long *step, PyObject **ob1, PyObject static PyStacklessFunctionDeclarationObject task_step_impl_declaration = { PyObject_HEAD_INIT(NULL) task_step_impl_stackless, - "task_step_impl_stackless" + "_task_step_impl_stackless" }; -#endif static PyObject * -task_step_impl(TaskObj *task, PyObject *exc) +task_step_impl_part1(TaskObj *task, PyObject *exc, int *failed) { -#ifdef STACKLESS - return PyStackless_CallFunction(&task_step_impl_declaration, Py_None, - (PyObject *)task, exc, NULL, 0, NULL); -} + STACKLESS_GETARG(); +#else /* #ifdef STACKLESS */ static PyObject * -task_step_impl_part1(TaskObj *task, PyObject *exc, int *failed) +task_step_impl(TaskObj *task, PyObject *exc) { -#endif - STACKLESS_GETARG(); +#endif /* #ifdef STACKLESS */ int res; int clear_exc = 0; PyObject *result = NULL; diff --git a/Stackless/changelog.txt b/Stackless/changelog.txt index 11b5eeb47452eb..19465cb4695ef2 100644 --- a/Stackless/changelog.txt +++ b/Stackless/changelog.txt @@ -9,6 +9,13 @@ What's New in Stackless 3.X.X? *Release date: 20XX-XX-XX* +- https://github.com/stackless-dev/stackless/issues/199 + Enable stackless calls of coroutines wrapped in "asyncio._CTask", if + soft-switching is enabled. Now asyncio runs coroutines stackless. + +- https://github.com/stackless-dev/stackless/issues/196 + Stackless can now pickle coroutine_wrapper objects. + - https://github.com/stackless-dev/stackless/issues/190 Silently ignore attempts to close a running generator, coroutine or asynchronous generator. This avoids spurious error messages, if such an diff --git a/Stackless/unittests/test_generator.py b/Stackless/unittests/test_generator.py index 79d5cb40ccbfac..7e64376998a093 100644 --- a/Stackless/unittests/test_generator.py +++ b/Stackless/unittests/test_generator.py @@ -6,6 +6,7 @@ import pickle import contextlib import sys +import asyncio from support import test_main # @UnusedImport from support import StacklessTestCase, captured_stderr @@ -242,5 +243,48 @@ def test_finalizer(self): self._test_finalizer(pf_dump, pf_load) +class TestStacklessOperations(StacklessTestCase): + def assertLevel(self, expected=0): + self.assertTrue(stackless.current.alive) + if stackless.enable_softswitch(None): + self.assertEqual(stackless.current.nesting_level, expected) + else: + self.assertGreater(stackless.current.nesting_level, expected) + + @types.coroutine + def coro1yield(self): + yield + + async def coro(self): + self.assertLevel() + await self.coro1yield() + self.assertLevel() + await self.coro1yield() + self.assertLevel() + + def _test_asyncio(self, task_class): + async def test(): + try: + await self.coro() + finally: + loop.stop() + + asyncio.set_event_loop(asyncio.new_event_loop()) + self.addCleanup(asyncio.set_event_loop, None) + loop = asyncio.get_event_loop() + task = task_class(test()) + asyncio.ensure_future(task) + try: + loop.run_forever() + finally: + loop.close() + + def test_asyncio_PyTask(self): + self._test_asyncio(asyncio.tasks._PyTask) + + def test_asyncio_CTask(self): + self._test_asyncio(asyncio.tasks._CTask) + + if __name__ == '__main__': unittest.main()