diff --git a/Doc/c-api/stackless.rst b/Doc/c-api/stackless.rst index a48b461b6fe65f..313c1ce16fafe8 100644 --- a/Doc/c-api/stackless.rst +++ b/Doc/c-api/stackless.rst @@ -3,6 +3,37 @@ |SLP| C-API =========== +.. note:: + + Some switching functions have a variant with the + same name, but ending on "_nr". These are non-recursive + versions with the same functionality, but they might + avoid a hard stack switch. + Their return value is ternary, and they require the + caller to return to its frame, properly. + All three different cases must be treated. + + Ternary return from an integer function: + + ===== ============= =============================== + value meaning action + ===== ============= =============================== + -1 failure return NULL + 1 soft switched return :c:data:`Py_UnwindToken` + 0 hard switched return :c:data:`Py_None` + ===== ============= =============================== + + Ternary return from a PyObject * function: + + ============== ============= =============================== + value meaning action + ============== ============= =============================== + NULL failure return NULL + Py_UnwindToken soft switched return :c:data:`Py_UnwindToken` + other hard switched return value + ============== ============= =============================== + + |SLP| provides the following C functions. Tasklets @@ -262,8 +293,8 @@ Channels Gets the balance for *self*. See :attr:`channel.balance`. -stackless module ----------------- +Module :py:mod:`stackless` +-------------------------- .. c:function:: PyObject *PyStackless_Schedule(PyObject *retval, int remove) @@ -354,7 +385,8 @@ Soft-switchable extension functions A soft switchable extension function or method is a function or method defined by an extension module written in C. In contrast to an normal C-function you -can soft-switch tasklets while this function executes. At the C-language level +can soft-switch tasklets while this function executes. Soft-switchable functions +obey the stackless-protocol. At the C-language level such a function or method is made from 3 C-definitions: 1. A declaration object of type :c:type:`PyStacklessFunctionDeclaration_Type`. @@ -378,10 +410,10 @@ Typedef ``slp_softswitchablefunc``:: .. c:type:: PyStacklessFunctionDeclarationObject - This subtype of :c:type:`PyObject` represents a Stackless soft switchable - extension function declaration object. + This subtype of :c:type:`PyObject` represents a Stackless soft switchable + extension function declaration object. - Here is the structure definition:: + Here is the structure definition:: typedef struct { PyObject_HEAD @@ -390,42 +422,145 @@ Typedef ``slp_softswitchablefunc``:: const char * module_name; } PyStacklessFunctionDeclarationObject; - .. c:member:: slp_softswitchablefunc PyStacklessFunctionDeclarationObject.sfunc + .. c:member:: slp_softswitchablefunc PyStacklessFunctionDeclarationObject.sfunc - Pointer to implementation function. + Pointer to implementation function. - .. c:member:: const char * PyStacklessFunctionDeclarationObject.name + .. c:member:: const char * PyStacklessFunctionDeclarationObject.name - Name of the function. + Name of the function. - .. c:member:: const char * PyStacklessFunctionDeclarationObject.module_name + .. c:member:: const char * PyStacklessFunctionDeclarationObject.module_name - Name of the containing module. + Name of the containing module. .. c:var:: PyTypeObject PyStacklessFunctionDeclaration_Type - This instance of :c:type:`PyTypeObject` represents the Stackless - soft switchable extension function declaration type. + This instance of :c:type:`PyTypeObject` represents the Stackless + soft switchable extension function declaration type. .. c:function:: int PyStacklessFunctionDeclarationType_CheckExact(PyObject *p) - Return true if *p* is a PyStacklessFunctionDeclarationObject object, but - not an instance of a subtype of this type. + Return true if *p* is a PyStacklessFunctionDeclarationObject object, but + not an instance of a subtype of this type. .. c:function:: PyObject* PyStackless_CallFunction(PyStacklessFunctionDeclarationObject *sfd, PyObject *arg, PyObject *ob1, PyObject *ob2, PyObject *ob3, long n, void *any) - Invoke the soft switchable extension, which is represented by *sfd*. - Pass *arg* as initial value for argument *retval* and *ob1*, *ob2*, *ob3*, - *n* and *any* as general purpose in-out-arguments. + Invoke the soft switchable extension, which is represented by *sfd*. + Pass *arg* as initial value for argument *retval* and *ob1*, *ob2*, *ob3*, + *n* and *any* as general purpose in-out-arguments. - Return the result of the function call or :c:data:`Py_UnwindToken`. + Return the result of the function call or :c:data:`Py_UnwindToken`. .. c:function:: int PyStackless_InitFunctionDeclaration(PyStacklessFunctionDeclarationObject *sfd, PyObject *module, PyModuleDef *module_def) - Initialize the fields :c:member:`PyStacklessFunctionDeclarationObject.name` and - :c:member:`PyStacklessFunctionDeclarationObject.module_name` of *sfd*. + Initialize the fields :c:member:`PyStacklessFunctionDeclarationObject.name` and + :c:member:`PyStacklessFunctionDeclarationObject.module_name` of *sfd*. + +Within the body of a soft switchable extension function (or any other C-function, that obyes the stackless-protocol) +you need the following macros. + +Macros for the "stackless-protocol" +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +How to does Stackless Python decide, if a function may return an unwind-token? +There is one global variable "_PyStackless_TRY_STACKLESS"[#]_ which is used +like an implicit parameter. Since we don't have a real parameter, +the flag is copied into the local variable "stackless" and cleared. +This is done by the STACKLESS_GETARG() macro, which should be added to +the top of the function's declarations. + +The idea is to keep the chances to introduce error to the minimum. +A function can safely do some tests and return before calling +anything, since the flag is in a local variable. +Depending on context, this flag is propagated to other called +functions. They *must* obey the protocol. To make this sure, +the STACKLESS_ASSERT() macro has to be called after every such call. + +Many internal functions have been patched to support this protocol. +Their first action is a direct or indirect call of the macro +:c:func:`STACKLESS_GETARG`. + +.. c:function:: STACKLESS_GETARG() + + Define the local variable ``int stackless`` and move the global + "_PyStackless_TRY_STACKLESS" flag into the local variable "stackless". + After a call to :c:func:`STACKLESS_GETARG` the value of + "_PyStackless_TRY_STACKLESS" is 0. + +.. c:function:: STACKLESS_PROMOTE_ALL() + + All STACKLESS_PROMOTE_xxx macros are used to propagate the stackless-flag + from the local variable "stackless" to the global variable + "_PyStackless_TRY_STACKLESS". The macro :c:func:`STACKLESS_PROMOTE_ALL` does + this unconditionally. It is used for cases where we know that the called + function will take care of our object, and we need no test. For example, + :c:func:`PyObject_Call` and all other Py{Object,Function,CFunction}_*Call* + functions use STACKLESS_PROMOTE_xxx itself, so we don't need to check further. + +.. c:function:: STACKLESS_PROMOTE_FLAG(flag) + + This macro is the most general conditional variant. + If the local variable "stackless" was set, it sets the global + variable "_PyStackless_TRY_STACKLESS" to *flag* and returns *flag*. + Otherwise the macro returns 0. It is used for special cases, + like PyCFunction objects. PyCFunction_Type + says that it supports a stackless call, but the final action depends + on the METH_STACKLESS flag in the object to be called. Therefore, + PyCFunction_Call uses ``STACKLESS_PROMOTE_FLAG(flags & METH_STACKLESS)`` to + take care of PyCFunctions which don't care about it. -debugging and monitoring functions + Another example is the "next" method of iterators. To support this, + the wrapperobject's type has the Py_TPFLAGS_HAVE_STACKLESS_CALL + flag set, but wrapper_call then examines the wrapper descriptors + flags if PyWrapperFlag_STACKLESS is set. "next" has it set. + It also checks whether Py_TPFLAGS_HAVE_STACKLESS_CALL is set + for the iterator's type. + +.. c:function:: STACKLESS_PROMOTE_METHOD(obj, slot_name) + + If the local variable "stackless" was set and if the type method for the + slot *slot_name* of the type of object *obj* obeys the stackless-protocol, + then _PyStackless_TRY_STACKLESS is set to 1, and we + expect that the function handles it correctly. + +.. c:function:: STACKLESS_PROMOTE(obj) + + A special optimized variant of ``STACKLESS_PROMOTE_METHOD(`` *obj* ``, tp_call)``. + +.. c:function:: STACKLESS_ASSERT() + + In debug builds this macro asserts that _PyStackless_TRY_STACKLESS was cleared. + This debug feature tries to ensure that no unexpected nonrecursive call can happen. + In release builds this macro does nothing. + +.. c:function:: STACKLESS_RETRACT() + + Set the global variable "_PyStackless_TRY_STACKLESS" unconditionally to 0. + Rarely used. + +Examples +~~~~~~~~ + +The Stackless test-module :py:mod:`_teststackless` contains the following +example for a soft switchable function. +To call it use +``PyStackless_CallFunction(&demo_soft_switchable_declaration, result, NULL, NULL, NULL, action, NULL)``. + +.. include:: ../../Stackless/module/_teststackless.c + :code: c + :encoding: utf-8 + :start-after: /*DO-NOT-REMOVE-OR-MODIFY-THIS-MARKER:ssf-example-start*/ + :end-before: /*DO-NOT-REMOVE-OR-MODIFY-THIS-MARKER:ssf-example-end*/ + +Another, more realistic example is :py:const:`_asyncio._task_step_impl_stackless`, defined in +"Modules/_asynciomodules.c". + + +.. [#] Actually "_PyStackless_TRY_STACKLESS" is a macro that expands to a C L-value. As long as + |CPY| uses the GIL, this L-value is a global variable. + +Debugging and monitoring Functions ---------------------------------- .. c:function:: int PyStackless_SetChannelCallback(PyObject *callable) @@ -460,6 +595,12 @@ Stack unwinding A singleton that indicates C-stack unwinding +.. note:: + + :c:data:`Py_UnwindToken` is *never* inc/decref'ed. Use the + macro :c:func:`STACKLESS_UNWINDING` to test for + Py_UnwindToken. + .. c:function:: int STACKLESS_UNWINDING(obj) Return 1, if *obj* is :c:data:`Py_UnwindToken` and 0 otherwise. diff --git a/Stackless/changelog.txt b/Stackless/changelog.txt index 11b5eeb47452eb..0da97db8b83f9f 100644 --- a/Stackless/changelog.txt +++ b/Stackless/changelog.txt @@ -19,7 +19,10 @@ What's New in Stackless 3.X.X? Previously they contained a reference to a non existing variable. - https://github.com/stackless-dev/stackless/issues/178 +- https://github.com/stackless-dev/stackless/issues/198 There is a new, provisional API to write soft switchable functions in C. + You can also use this API to convert existing extension functions to + the stackless-protocol. - https://github.com/stackless-dev/stackless/issues/149 The Stackless version is now "3.7". diff --git a/Stackless/core/cframeobject.c b/Stackless/core/cframeobject.c index ef42ffd64354e7..2af02d3da13857 100644 --- a/Stackless/core/cframeobject.c +++ b/Stackless/core/cframeobject.c @@ -436,7 +436,8 @@ execute_soft_switchable_func(PyFrameObject *f, int exc, PyObject *retval) PyObject *ob1, *ob2, *ob3; PyStacklessFunctionDeclarationObject *ssfd = (PyStacklessFunctionDeclarationObject*)cf->any2; - if (retval == NULL) { + if (retval == NULL && !PyErr_Occurred()) { + PyErr_SetString(PyExc_SystemError, "retval is NULL, but no error occurred"); SLP_STORE_NEXT_FRAME(ts, cf->f_back); return NULL; } @@ -452,7 +453,7 @@ execute_soft_switchable_func(PyFrameObject *f, int exc, PyObject *retval) Py_XINCREF(ob3); STACKLESS_PROPOSE_ALL(ts); /* use Py_SETREF, because we own a ref to retval. */ - Py_SETREF(retval, ssfd->sfunc(retval, &cf->i, &cf->ob1, &cf->ob2, &cf->ob3, &cf->n, &cf->any1)); + Py_XSETREF(retval, ssfd->sfunc(retval, &cf->i, &cf->ob1, &cf->ob2, &cf->ob3, &cf->n, &cf->any1)); STACKLESS_RETRACT(); STACKLESS_ASSERT(); Py_XDECREF(ob1); @@ -484,18 +485,29 @@ PyStackless_CallFunction(PyStacklessFunctionDeclarationObject *ssfd, PyObject *a { STACKLESS_GETARG(); PyThreadState *ts = PyThreadState_GET(); + PyObject *et=NULL, *ev=NULL, *tb=NULL; assert(ssfd); assert(ts->st.main != NULL); + if (arg == NULL) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_SystemError, "No error occurred, but arg is NULL"); + return NULL; + } + PyErr_Fetch(&et, &ev, &tb); + } + if (stackless) { /* Only soft-switch, if the caller can handle Py_UnwindToken. * If called from Python-code, this is always the case, * but if you call this method from a C-function, you don't know. */ PyCFrameObject *cf; cf = slp_cframe_new(execute_soft_switchable_func, 1); - if (cf == NULL) + if (cf == NULL) { + _PyErr_ChainExceptions(et, ev, tb); return NULL; + } cf->any2 = ssfd; Py_XINCREF(ob1); @@ -510,9 +522,9 @@ PyStackless_CallFunction(PyStacklessFunctionDeclarationObject *ssfd, PyObject *a cf->any1 = any; SLP_STORE_NEXT_FRAME(ts, (PyFrameObject *) cf); Py_DECREF(cf); + Py_XINCREF(arg); if (arg == NULL) - arg = Py_None; - Py_INCREF(arg); + PyErr_Restore(et, ev, tb); return STACKLESS_PACK(ts, arg); } else { /* @@ -521,9 +533,7 @@ PyStackless_CallFunction(PyStacklessFunctionDeclarationObject *ssfd, PyObject *a PyObject *retval, *saved_ob1, *saved_ob2, *saved_ob3; long step = 0; - if (arg == NULL) - arg = Py_None; - Py_INCREF(arg); + Py_XINCREF(arg); saved_ob1 = ob1; saved_ob2 = ob2; @@ -531,13 +541,15 @@ PyStackless_CallFunction(PyStacklessFunctionDeclarationObject *ssfd, PyObject *a Py_XINCREF(saved_ob1); Py_XINCREF(saved_ob2); Py_XINCREF(saved_ob3); + if (arg == NULL) + PyErr_Restore(et, ev, tb); retval = ssfd->sfunc(arg, &step, &ob1, &ob2, &ob3, &n, &any); STACKLESS_ASSERT(); assert(!STACKLESS_UNWINDING(retval)); Py_XDECREF(saved_ob1); Py_XDECREF(saved_ob2); Py_XDECREF(saved_ob3); - Py_DECREF(arg); + Py_XDECREF(arg); return retval; } } diff --git a/Stackless/core/stackless_impl.h b/Stackless/core/stackless_impl.h index 69656f10c07a90..132f52c640aff4 100644 --- a/Stackless/core/stackless_impl.h +++ b/Stackless/core/stackless_impl.h @@ -423,32 +423,25 @@ PyTaskletObject * slp_get_watchdog(PyThreadState *ts, int interrupt); /* macros for setting/resetting the stackless flag */ +#ifdef _PyStackless_TRY_STACKLESS +#undef _PyStackless_TRY_STACKLESS +#endif +#define _PyStackless_TRY_STACKLESS (_PyRuntime.st.try_stackless) + #define STACKLESS_POSSIBLE(tstate) \ ((tstate)->interp->st.enable_softswitch && (tstate)->st.unwinding_retval == NULL) -#define STACKLESS_GETARG() \ - int stackless = (assert(SLP_CURRENT_FRAME_IS_VALID(PyThreadState_GET())), \ - stackless = (_PyRuntime.st.try_stackless), \ - (_PyRuntime.st.try_stackless) = 0, \ - stackless) - -#define STACKLESS_PROMOTE(func) \ - (stackless ? (_PyRuntime.st.try_stackless) = \ - Py_TYPE(func)->tp_flags & Py_TPFLAGS_HAVE_STACKLESS_CALL : 0) - -#define STACKLESS_PROMOTE_FLAG(flag) \ - (stackless ? (_PyRuntime.st.try_stackless) = (flag) : 0) - -#define STACKLESS_PROMOTE_METHOD(obj, meth) do { \ - if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_HAVE_STACKLESS_EXTENSION) && \ - Py_TYPE(obj)->tp_as_mapping) \ - (_PyRuntime.st.try_stackless) = stackless && Py_TYPE(obj)->tp_as_mapping->slpflags.meth; \ -} while (0) +#ifdef STACKLESS__GETARG_ASSERT +#undef STACKLESS__GETARG_ASSERT +#endif +#define STACKLESS__GETARG_ASSERT \ + assert(SLP_CURRENT_FRAME_IS_VALID(PyThreadState_GET())) +/* descr must be of type PyWrapperDescrObject, but this type is undocumented. + * Therefore this macro is in stackless_impl.h and not in stackless_api.h + */ #define STACKLESS_PROMOTE_WRAPPER(descr) \ - ((_PyRuntime.st.try_stackless) = stackless && (descr)->d_slpmask) - -#define STACKLESS_PROMOTE_ALL() ((void)((_PyRuntime.st.try_stackless) = stackless, NULL)) + STACKLESS_PROMOTE_FLAG((descr)->d_slpmask) #define STACKLESS_PROPOSE(tstate, func) {int stackless = STACKLESS_POSSIBLE(tstate); \ STACKLESS_PROMOTE(func);} @@ -459,11 +452,7 @@ PyTaskletObject * slp_get_watchdog(PyThreadState *ts, int interrupt); #define STACKLESS_PROPOSE_METHOD(tstate, obj, meth) {int stackless = STACKLESS_POSSIBLE(tstate); \ STACKLESS_PROMOTE_METHOD(obj, meth);} -#define STACKLESS_PROPOSE_ALL(tstate) (_PyRuntime.st.try_stackless) = STACKLESS_POSSIBLE(tstate) - -#define STACKLESS_RETRACT() (_PyRuntime.st.try_stackless) = 0; - -#define STACKLESS_ASSERT() assert(!(_PyRuntime.st.try_stackless)) +#define STACKLESS_PROPOSE_ALL(tstate) _PyStackless_TRY_STACKLESS = STACKLESS_POSSIBLE(tstate) /* this is just a tag to denote which methods are stackless */ @@ -477,7 +466,7 @@ PyTaskletObject * slp_get_watchdog(PyThreadState *ts, int interrupt); /* How this works: - There is one global variable (_PyRuntime.st.try_stackless) which is used + There is one global variable _PyStackless_TRY_STACKLESS which is used like an implicit parameter. Since we don't have a real parameter, the flag is copied into the local variable "stackless" and cleared. This is done by the STACKLESS_GETARG() macro, which should be added to @@ -493,7 +482,7 @@ PyTaskletObject * slp_get_watchdog(PyThreadState *ts, int interrupt); STACKLESS_GETARG() - move the (_PyRuntime.st.try_stackless) flag into the local variable "stackless". + move the _PyStackless_TRY_STACKLESS flag into the local variable "stackless". STACKLESS_PROMOTE_ALL() @@ -506,7 +495,7 @@ PyTaskletObject * slp_get_watchdog(PyThreadState *ts, int interrupt); if stackless was set and the function's type has set Py_TPFLAGS_HAVE_STACKLESS_CALL, then this flag will be - put back into (_PyRuntime.st.try_stackless), and we expect that the + put back into _PyStackless_TRY_STACKLESS, and we expect that the function handles it correctly. STACKLESS_PROMOTE_FLAG(flag) @@ -526,7 +515,7 @@ PyTaskletObject * slp_get_watchdog(PyThreadState *ts, int interrupt); STACKLESS_ASSERT() - make sure that (_PyRuntime.st.try_stackless) was cleared. This debug feature + make sure that _PyStackless_TRY_STACKLESS was cleared. This debug feature tries to ensure that no unexpected nonrecursive call can happen. Some functions which are known to be stackless by nature @@ -787,18 +776,10 @@ long slp_parse_thread_id(PyObject *thread_id, unsigned long *id); #define SLP_PEEK_NEXT_FRAME(tstate) \ ((tstate)->frame) -#define STACKLESS_GETARG() int stackless = 0 -#define STACKLESS_PROMOTE(func) stackless = 0 -#define STACKLESS_PROMOTE_FLAG(flag) stackless = 0 -#define STACKLESS_PROMOTE_METHOD(obj, meth) stackless = 0 -#define STACKLESS_PROMOTE_WRAPPER(descr) stackless = 0 -#define STACKLESS_PROMOTE_ALL() stackless = 0 #define STACKLESS_PROPOSE(tstate, func) assert(1) #define STACKLESS_PROPOSE_FLAG(tstate, flag) assert(1) #define STACKLESS_PROPOSE_ALL(tstate) assert(1) #define STACKLESS_PROPOSE_METHOD(tstate, obj, meth) assert(1) -#define STACKLESS_RETRACT() assert(1) -#define STACKLESS_ASSERT() assert(1) #define STACKLESS_RETVAL(tstate, obj) (obj) #define STACKLESS_ASSERT_UNWINDING_VALUE_IS_NOT(tstate, obj, val) assert(1) diff --git a/Stackless/core/stackless_structs.h b/Stackless/core/stackless_structs.h index 531308aee40884..8a2ef2c58b89c1 100644 --- a/Stackless/core/stackless_structs.h +++ b/Stackless/core/stackless_structs.h @@ -289,6 +289,18 @@ PyAPI_DATA(PyTypeObject) PyChannel_Type; #define PyChannel_Check(op) PyObject_TypeCheck(op, &PyChannel_Type) #define PyChannel_CheckExact(op) (Py_TYPE(op) == &PyChannel_Type) +/****************************************************** + Macros for the stackless protocol + ******************************************************/ + +#ifndef _PyStackless_TRY_STACKLESS +PyAPI_DATA(int * const) _PyStackless__TryStacklessPtr; +#define _PyStackless_TRY_STACKLESS (*_PyStackless__TryStacklessPtr) +#endif +#ifndef STACKLESS__GETARG_ASSERT +#define STACKLESS__GETARG_ASSERT ((void)0) +#endif + #endif /* #ifdef STACKLESS */ #ifdef __cplusplus diff --git a/Stackless/core/stackless_util.c b/Stackless/core/stackless_util.c index 79135644254d64..048c0b1fb0dbda 100644 --- a/Stackless/core/stackless_util.c +++ b/Stackless/core/stackless_util.c @@ -3,6 +3,8 @@ #ifdef STACKLESS #include "stackless_impl.h" +int * const _PyStackless__TryStacklessPtr = &_PyStackless_TRY_STACKLESS; + /* Initialize the Stackless runtime state */ void slp_initialize(struct _stackless_runtime_state * state) { diff --git a/Stackless/module/_teststackless.c b/Stackless/module/_teststackless.c index 43240397619954..9a6753dd7a4a47 100644 --- a/Stackless/module/_teststackless.c +++ b/Stackless/module/_teststackless.c @@ -34,29 +34,69 @@ static PyTypeObject SoftSwitchableDemo_Type; #define SoftSwitchableDemoObject_Check(v) (Py_TYPE(v) == &SoftSwitchableDemo_Type) -/* SoftSwitchableDemo methods */ +/* The documentation includes the following code as an example and needs + * the next comment as marker. + */ +/*DO-NOT-REMOVE-OR-MODIFY-THIS-MARKER:ssf-example-start*/ +/* + * SoftSwitchableDemo methods + * + * The purpose of this demo is to test soft switchable extension methods and + * give an example of their usage. Otherwise the demo code is very simplistic. + */ static PyObject * -demo_soft_switchable(PyObject *retval, long *step, PyObject **ob1, PyObject **ob2, PyObject **ob3, long *n, void **any) +demo_soft_switchable(PyObject *retval, long *step, PyObject **ob1, + PyObject **ob2, PyObject **ob3, long *n, void **any) { - int do_schedule = *n; + /* + * Boiler plate code for soft switchable functions + */ + + /* Every function, that supports the stackless-protocol starts with this + * macro. It defines the local variable "int stackless" and moves the value + * of the global variable "_PyStackless_TRY_STACKLESS" to "stackless". + */ + STACKLESS_GETARG(); + + /* This function returns a new reference. + * If retval is NULL, then PyErr_Occurred() is true. + */ + Py_XINCREF(retval); + + /* + * Optional: define a struct for additional state + */ struct { /* * If *ob1, *ob2, *ob3, *n and *any are insufficient for the state of this method, * you can define state variables here and store this structure in *any. */ - int var1; + int var1; // Minimal example } *state = *any; - Py_INCREF(retval); /* we need our own reference */ + /* + * Specific vars for this example. + */ + int do_schedule = *n; + if (*step == 0 && *n >= 100) + *step = *n; // n==100: demo for calling back to Python + /* + * Always present: the switch, that is used to jump to the next step. + * + * Of course you are not limited to a linear sequence of steps, but + * this is simplest use case. If you think of a state machine, the variable + * (*step) is the number of the next state to enter. + * The initial value of (*step) is 0. + */ switch(*step) { case 0: - (*step)++; + (*step)++; // set to the next step /* - * Initialize state + * Optional: initialize the state structure. */ *any = state = PyMem_Calloc(1, sizeof(*state)); if (state == NULL) { @@ -65,7 +105,7 @@ demo_soft_switchable(PyObject *retval, long *step, PyObject **ob1, PyObject **ob } /* - * Add your business logic here + * Code specific for this example */ state->var1++; /* This example is a bit simplistic */ @@ -73,7 +113,21 @@ demo_soft_switchable(PyObject *retval, long *step, PyObject **ob1, PyObject **ob * Now eventually schedule. */ if (do_schedule) { + /* + * The function PyStackless_Schedule() supports the stackless-protocol. + * Therefore we may call TACKLESS_PROMOTE_ALL(). This macro copies the + * local variable "stackless" into "_PyStackless_TRY_STACKLESS". + */ + STACKLESS_PROMOTE_ALL(); // enable a stackless call Py_SETREF(retval, PyStackless_Schedule(retval, do_schedule > 1)); + /* + * In debug builds STACKLESS_ASSERT asserts, that the previously called + * function did reset "_PyStackless_TRY_STACKLESS". If you call + * STACKLESS_PROMOTE_ALL() (or one of its variants) prior to calling + * a function that does not support the stackless protocol, the + * assertion fails. + */ + STACKLESS_ASSERT(); // sanity check in debug builds if (STACKLESS_UNWINDING(retval)) return retval; else if (retval == NULL) @@ -96,7 +150,9 @@ demo_soft_switchable(PyObject *retval, long *step, PyObject **ob1, PyObject **ob * Now eventually schedule. */ if (do_schedule) { + STACKLESS_PROMOTE_ALL(); // enable a stackless call Py_SETREF(retval, PyStackless_Schedule(retval, do_schedule > 1)); + STACKLESS_ASSERT(); // sanity check in debug builds if (STACKLESS_UNWINDING(retval)) return retval; else if (retval == NULL) @@ -108,18 +164,68 @@ demo_soft_switchable(PyObject *retval, long *step, PyObject **ob1, PyObject **ob */ /* no break */ case 2: + /* + * Prepare the result + */ + Py_SETREF(retval, PyLong_FromLong(*n)); + break; + + /* + * Demo code for calling back to Python. + */ + case 100: + (*step)++; // set to the next step + /* + * Here we demonstrate a stackless callback into Python code. + * This test assumes, that result is a callable object. + * + * The API function PyObject_Call supports the stackless protocoll. + * Therefore we may call STACKLESS_PROMOTE_ALL(); + */ + PyObject * args = PyTuple_New(0); + STACKLESS_PROMOTE_ALL(); + Py_SETREF(retval, PyObject_Call(retval, args, NULL)); + STACKLESS_ASSERT(); + if (STACKLESS_UNWINDING(retval)) + return retval; + /* no break */ + + case 101: + (*step)++; // set to the next step + + /* + * Error handling for the callback. + * + * This needs a step of its own, if the callback supports the + * stackless-protocol and fails. + */ + if (retval == NULL) { + assert(PyErr_Occurred()); + + if (! PyErr_ExceptionMatches(PyExc_Exception)) + // it is a BaseException, don't handle it + return NULL; + + /* + * A very simple error handling: fetch the exception set it as + * context of a new RuntimeError. + */ + PyObject *et, *ev, *tb; + PyErr_Fetch(&et, &ev, &tb); + PyErr_SetString(PyExc_RuntimeError, "demo_soft_switchable callback failed"); + _PyErr_ChainExceptions(et, ev, tb); + } + break; + default: + /* + * Boiler plate code: error handling + */ PyErr_SetString(PyExc_SystemError, "invalid state"); Py_CLEAR(retval); - goto exit_func; } - /* - * Prepare the result - */ - Py_SETREF(retval, PyLong_FromLong(*n)); - exit_func: PyMem_Free(*any); *any = NULL; @@ -133,6 +239,7 @@ static PyStacklessFunctionDeclarationObject demo_soft_switchable_declaration = { "demo_soft_switchable" }; +/*DO-NOT-REMOVE-OR-MODIFY-THIS-MARKER:ssf-example-end*/ static PyObject * SoftSwitchableDemo_demo(SoftSwitchableDemoObject *self, PyObject *args) diff --git a/Stackless/stackless_api.h b/Stackless/stackless_api.h index 28376815d386e4..937d87cfac10b5 100644 --- a/Stackless/stackless_api.h +++ b/Stackless/stackless_api.h @@ -19,18 +19,20 @@ extern "C" { All three different cases must be treated. Ternary return from an integer function: - value meaning action - -1 failure return NULL - 1 soft switched return Py_UnwindToken - 0 hard switched return Py_None + value meaning action + -1 failure return NULL + 1 soft switched return Py_UnwindToken + 0 hard switched return Py_None Ternary return from a PyObject * function: - value meaning action - NULL failure return NULL + value meaning action + NULL failure return NULL Py_UnwindToken soft switched return Py_UnwindToken - other hard switched return value + other hard switched return value - Note: Py_UnwindToken is *never* inc/decref'ed. + Note: Py_UnwindToken is *never* inc/decref'ed. Use the + macro STACKLESS_UNWINDING(retval) to test for + Py_UnwindToken ******************************************************/ @@ -44,7 +46,7 @@ extern "C" { * func must (yet) be a callable object (normal usecase) */ PyAPI_FUNC(PyTaskletObject *) PyTasklet_New(PyTypeObject *type, PyObject *func); -/* 0 = success -1 = failure */ +/* 0 = success -1 = failure */ /* * bind a tasklet function to parameters, making it ready to run, @@ -66,7 +68,7 @@ PyAPI_FUNC(int) PyTasklet_BindThread(PyTaskletObject *task, unsigned long thread * forces the tasklet to run immediately. */ PyAPI_FUNC(int) PyTasklet_Run(PyTaskletObject *task); -/* 0 = success -1 = failure */ +/* 0 = success -1 = failure */ PyAPI_FUNC(int) PyTasklet_Run_nr(PyTaskletObject *task); /* 1 = soft switched 0 = hard switched -1 = failure */ @@ -74,7 +76,7 @@ PyAPI_FUNC(int) PyTasklet_Run_nr(PyTaskletObject *task); * raw switching. The previous tasklet is paused. */ PyAPI_FUNC(int) PyTasklet_Switch(PyTaskletObject *task); -/* 0 = success -1 = failure */ +/* 0 = success -1 = failure */ PyAPI_FUNC(int) PyTasklet_Switch_nr(PyTaskletObject *task); /* 1 = soft switched 0 = hard switched -1 = failure */ @@ -85,7 +87,7 @@ PyAPI_FUNC(int) PyTasklet_Switch_nr(PyTaskletObject *task); * might give an inconsistent system state. */ PyAPI_FUNC(int) PyTasklet_Remove(PyTaskletObject *task); -/* 0 = success -1 = failure */ +/* 0 = success -1 = failure */ /* * insert a tasklet into the runnables queue, if it isn't @@ -93,7 +95,7 @@ PyAPI_FUNC(int) PyTasklet_Remove(PyTaskletObject *task); * blocked or dead. */ PyAPI_FUNC(int) PyTasklet_Insert(PyTaskletObject *task); -/* 0 = success -1 = failure */ +/* 0 = success -1 = failure */ /* * raising an exception for a tasklet. @@ -105,8 +107,8 @@ PyAPI_FUNC(int) PyTasklet_Insert(PyTaskletObject *task); */ PyAPI_FUNC(int) PyTasklet_RaiseException(PyTaskletObject *self, - PyObject *klass, PyObject *args); -/* 0 = success -1 = failure. + PyObject *klass, PyObject *args); +/* 0 = success -1 = failure. * Note that this call always ends in some exception, so the * caller always should return NULL. */ @@ -125,7 +127,7 @@ PyAPI_FUNC(int) PyTasklet_Throw(PyTaskletObject *self, PyAPI_FUNC(int) PyTasklet_Kill(PyTaskletObject *self); PyAPI_FUNC(int) PyTasklet_KillEx(PyTaskletObject *self, int pending); -/* 0 = success -1 = failure. +/* 0 = success -1 = failure. * Note that this call always ends in some exception, so the * caller always should return NULL. */ @@ -210,7 +212,7 @@ PyAPI_FUNC(PyChannelObject *) PyChannel_New(PyTypeObject *type); * if nobody is listening, you will get blocked and scheduled. */ PyAPI_FUNC(int) PyChannel_Send(PyChannelObject *self, PyObject *arg); -/* 0 = success -1 = failure */ +/* 0 = success -1 = failure */ PyAPI_FUNC(int) PyChannel_Send_nr(PyChannelObject *self, PyObject *arg); /* 1 = soft switched 0 = hard switched -1 = failure */ @@ -381,7 +383,7 @@ typedef struct { PyAPI_DATA(PyTypeObject) PyStacklessFunctionDeclaration_Type; #define PyStacklessFunctionDeclarationType_CheckExact(op) \ - (Py_TYPE(op) == &PyStacklessFunctionDeclaration_Type) + (Py_TYPE(op) == &PyStacklessFunctionDeclaration_Type) PyAPI_FUNC(PyObject *) PyStackless_CallFunction( PyStacklessFunctionDeclarationObject *sfd, PyObject *arg, @@ -390,6 +392,105 @@ PyAPI_FUNC(PyObject *) PyStackless_CallFunction( PyAPI_FUNC(int) PyStackless_InitFunctionDeclaration( PyStacklessFunctionDeclarationObject *sfd, PyObject *module, PyModuleDef *module_def); +/* + +Macros for the "stackless protocol" +=================================== + +How to does Stackless Python decide, if a function may return an unwind-token. + +There is one global variable _PyStackless_TRY_STACKLESS which is used +like an implicit parameter. Since we don't have a real parameter, +the flag is copied into the local variable "stackless" and cleared. +This is done by the STACKLESS_GETARG() macro, which should be added to +the top of the function's declarations. +The idea is to keep the chances to introduce error to the minimum. +A function can safely do some tests and return before calling +anything, since the flag is in a local variable. +Depending on context, this flag is propagated to other called +functions. They *must* obey the protocol. To make this sure, +the STACKLESS_ASSERT() macro has to be called after every such call. + +Many internal functions have been patched to support this protocol. + +STACKLESS_GETARG() + + Move the _PyStackless_TRY_STACKLESS flag into the local variable "stackless". + +STACKLESS_PROMOTE_ALL() + + is used for cases where we know that the called function will take + care of our object, and we need no test. For example, PyObject_Call + and all other Py{Object,Function,CFunction}_*Call* functions use + STACKLESS_PROMOTE_xxx, itself, so we don't need to check further. + +STACKLESS_PROMOTE(func) + + If stackless was set and the function's type has set + Py_TPFLAGS_HAVE_STACKLESS_CALL, then this flag will be + put back into _PyStackless_TRY_STACKLESS, and we expect that the + function handles it correctly. + +STACKLESS_PROMOTE_FLAG(flag) + + is used for special cases, like PyCFunction objects. PyCFunction_Type + says that it supports a stackless call, but the final action depends + on the METH_STACKLESS flag in the object to be called. Therefore, + PyCFunction_Call uses PROMOTE_FLAG(flags & METH_STACKLESS) to + take care of PyCFunctions which don't care about it. + + Another example is the "next" method of iterators. To support this, + the wrapperobject's type has the Py_TPFLAGS_HAVE_STACKLESS_CALL + flag set, but wrapper_call then examines the wrapper descriptors + flags if PyWrapperFlag_STACKLESS is set. "next" has it set. + It also checks whether Py_TPFLAGS_HAVE_STACKLESS_CALL is set + for the iterator's type. + +STACKLESS_ASSERT() + + Make sure that _PyStackless_TRY_STACKLESS was cleared. This debug feature + tries to ensure that no unexpected nonrecursive call can happen. + +STACKLESS_RETRACT() + + Reset _PyStackless_TRY_STACKLESS. Rarely needed. + +*/ + +#define STACKLESS_GETARG() \ + int stackless = (STACKLESS__GETARG_ASSERT, \ + stackless = _PyStackless_TRY_STACKLESS, \ + _PyStackless_TRY_STACKLESS = 0, \ + stackless) + +#define STACKLESS_PROMOTE_ALL() ((void)(_PyStackless_TRY_STACKLESS = stackless, NULL)) + +#define STACKLESS_PROMOTE_FLAG(flag) \ + (stackless ? (_PyStackless_TRY_STACKLESS = (flag)) : 0) + +#define STACKLESS_PROMOTE_METHOD(obj, slot_name) \ + STACKLESS_PROMOTE_FLAG( \ + (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_HAVE_STACKLESS_EXTENSION) && \ + Py_TYPE(obj)->tp_as_mapping && \ + Py_TYPE(obj)->tp_as_mapping->slpflags.slot_name) + +#define STACKLESS_PROMOTE(obj) \ + STACKLESS_PROMOTE_FLAG( \ + Py_TYPE(obj)->tp_flags & Py_TPFLAGS_HAVE_STACKLESS_CALL) + +#define STACKLESS_RETRACT() (_PyStackless_TRY_STACKLESS = 0) + +#define STACKLESS_ASSERT() assert(!_PyStackless_TRY_STACKLESS) + +#else /* STACKLESS */ +/* turn the stackless flag macros into dummies */ +#define STACKLESS_GETARG() int stackless = 0 +#define STACKLESS_PROMOTE_ALL() (stackless = 0) +#define STACKLESS_PROMOTE_FLAG(flag) (stackless = 0) +#define STACKLESS_RETRACT() assert(1) +#define STACKLESS_ASSERT() assert(1) +#endif +#ifdef STACKLESS /****************************************************** @@ -462,13 +563,13 @@ PyAPI_DATA(PyUnwindObject *) Py_UnwindToken; * Run any callable as the "main" Python(r) function. */ PyAPI_FUNC(PyObject *) PyStackless_Call_Main(PyObject *func, - PyObject *args, PyObject *kwds); + PyObject *args, PyObject *kwds); /* * Convenience: Run any method as the "main" Python(r) function. */ PyAPI_FUNC(PyObject *) PyStackless_CallMethod_Main(PyObject *o, char *name, - char *format, ...); + char *format, ...); /* *convenience: Run any cmethod as the "main" Python(r) function. @@ -476,6 +577,7 @@ PyAPI_FUNC(PyObject *) PyStackless_CallMethod_Main(PyObject *o, char *name, PyAPI_FUNC(PyObject *) PyStackless_CallCMethod_Main( PyMethodDef *meth, PyObject *self, char *format, ...); + #endif /* STACKLESS */ #ifdef __cplusplus diff --git a/Stackless/unittests/test_capi.py b/Stackless/unittests/test_capi.py index 8d2bfdc99db48b..9200a1bae8e16d 100644 --- a/Stackless/unittests/test_capi.py +++ b/Stackless/unittests/test_capi.py @@ -290,6 +290,38 @@ def test_softswitchablefunc(self): f = _teststackless.softswitchablefunc self._test_func(f) + def callback_ok(self): + if stackless.enable_softswitch(None): + self.assertEqual(stackless.current.nesting_level, 0) + else: + self.assertGreaterEqual(stackless.current.nesting_level, 1) + return self + + def test_softswitchablefunc_callback_ok(self): + def task(): + self.assertIs(self, _teststackless.softswitchablefunc(100, self.callback_ok)) + + t = stackless.tasklet(task)() + stackless.run() + self.assertFalse(t.alive) + + def callback_fail(self): + if stackless.enable_softswitch(None): + self.assertEqual(stackless.current.nesting_level, 0) + else: + self.assertGreaterEqual(stackless.current.nesting_level, 1) + 1 / 0 + + def test_softswitchablefunc_callback_fail(self): + def task(): + with self.assertRaisesRegex(RuntimeError, "demo_soft_switchable callback failed") as cm: + _teststackless.softswitchablefunc(100, self.callback_fail) + self.assertIsInstance(cm.exception.__context__, ZeroDivisionError) + + t = stackless.tasklet(task)() + stackless.run() + self.assertFalse(t.alive) + class TestDemoExtensions(unittest.TestCase): def setUp(self): diff --git a/Tools/c-globals/ignored-globals.txt b/Tools/c-globals/ignored-globals.txt index 072b4e080daf71..3ce795ff2007e8 100644 --- a/Tools/c-globals/ignored-globals.txt +++ b/Tools/c-globals/ignored-globals.txt @@ -482,6 +482,10 @@ whatstrings ####################################### # Stackless +# Stackless/core/stackless_util.c +# justification: the Stackless API needs it +_PyStackless__TryStacklessPtr + # Objects/exceptions.c # justification: other exceptions are globals too _PyExc_TaskletExit