Initial (incomplete) attempt at supporting the Py3.7 exception state stack.

Cython/Compiler/Nodes.py

@@ -4256,7 +4256,10 @@ class GeneratorBodyDefNode(DefNode):
             code.put_xgiveref(Naming.retval_cname)
         else:
             code.put_xdecref_clear(Naming.retval_cname, py_object_type)
+        # For Py3.7, clearing is already done below.
+        code.putln("#if !CYTHON_USE_EXC_INFO_STACK")
         code.putln("__Pyx_Coroutine_ResetAndClearException(%s);" % Naming.generator_cname)
+        code.putln("#endif")
         code.putln('%s->resume_label = -1;' % Naming.generator_cname)
         # clean up as early as possible to help breaking any reference cycles
         code.putln('__Pyx_Coroutine_clear((PyObject*)%s);' % Naming.generator_cname)

Cython/Utility/Coroutine.c

@@ -364,13 +364,23 @@ static void __Pyx_Generator_Replace_StopIteration(CYTHON_UNUSED int in_async_gen
 
 typedef PyObject *(*__pyx_coroutine_body_t)(PyObject *, PyThreadState *, PyObject *);
 
+#if CYTHON_USE_EXC_INFO_STACK
+// See  https://bugs.python.org/issue25612
+#define __Pyx_ExcInfoStruct  _PyErr_StackItem
+#else
+// Minimal replacement struct for Py<3.7, without the Py3.7 exception state stack.
 typedef struct {
-    PyObject_HEAD
-    __pyx_coroutine_body_t body;
-    PyObject *closure;
     PyObject *exc_type;
     PyObject *exc_value;
     PyObject *exc_traceback;
+} __Pyx_ExcInfoStruct;
+#endif
+
+typedef struct {
+    PyObject_HEAD
+    __pyx_coroutine_body_t body;
+    PyObject *closure;
+    __Pyx_ExcInfoStruct gi_exc_state;
     PyObject *gi_weakreflist;
     PyObject *classobj;
     PyObject *yieldfrom;
@@ -391,20 +401,26 @@ static __pyx_CoroutineObject *__Pyx__Coroutine_NewInit(
             __pyx_CoroutineObject *gen, __pyx_coroutine_body_t body, PyObject *code, PyObject *closure,
             PyObject *name, PyObject *qualname, PyObject *module_name); /*proto*/
 
+static CYTHON_INLINE void __Pyx_Coroutine_ExceptionClear(__Pyx_ExcInfoStruct *self);
 static int __Pyx_Coroutine_clear(PyObject *self); /*proto*/
 static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value); /*proto*/
 static PyObject *__Pyx_Coroutine_Close(PyObject *self); /*proto*/
 static PyObject *__Pyx_Coroutine_Throw(PyObject *gen, PyObject *args); /*proto*/
 
 // macros for exception state swapping instead of inline functions to make use of the local thread state context
+#if CYTHON_USE_EXC_INFO_STACK
+#define __Pyx_Coroutine_SwapException(self)  __Pyx_Coroutine_ResetFrameBackpointer(self)
+#define __Pyx_Coroutine_ResetAndClearException(self)  __Pyx_Coroutine_ExceptionClear(&(self)->gi_exc_state)
+#else
 #define __Pyx_Coroutine_SwapException(self) { \
-    __Pyx_ExceptionSwap(&(self)->exc_type, &(self)->exc_value, &(self)->exc_traceback); \
+    __Pyx_ExceptionSwap(&(self)->gi_exc_state.exc_type, &(self)->gi_exc_state.exc_value, &(self)->gi_exc_state.exc_traceback); \
     __Pyx_Coroutine_ResetFrameBackpointer(self); \
     }
 #define __Pyx_Coroutine_ResetAndClearException(self) { \
-    __Pyx_ExceptionReset((self)->exc_type, (self)->exc_value, (self)->exc_traceback); \
-    (self)->exc_type = (self)->exc_value = (self)->exc_traceback = NULL; \
+    __Pyx_ExceptionReset((self)->gi_exc_state.exc_type, (self)->gi_exc_state.exc_value, (self)->gi_exc_state.exc_traceback); \
+    (self)->gi_exc_state.exc_type = (self)->gi_exc_state.exc_value = (self)->gi_exc_state.exc_traceback = NULL; \
     }
+#endif
 
 #if CYTHON_FAST_THREAD_STATE
 #define __Pyx_PyGen_FetchStopIterationValue(pvalue) \
@@ -573,18 +589,19 @@ static int __Pyx_PyGen__FetchStopIterationValue(CYTHON_UNUSED PyThreadState *$lo
 }
 
 static CYTHON_INLINE
-void __Pyx_Coroutine_ExceptionClear(__pyx_CoroutineObject *self) {
-    PyObject *exc_type = self->exc_type;
-    PyObject *exc_value = self->exc_value;
-    PyObject *exc_traceback = self->exc_traceback;
-
-    self->exc_type = NULL;
-    self->exc_value = NULL;
-    self->exc_traceback = NULL;
-
-    Py_XDECREF(exc_type);
-    Py_XDECREF(exc_value);
-    Py_XDECREF(exc_traceback);
+void __Pyx_Coroutine_ExceptionClear(__Pyx_ExcInfoStruct *exc_state) {
+    PyObject *t, *v, *tb;
+    t = exc_state->exc_type;
+    v = exc_state->exc_value;
+    tb = exc_state->exc_traceback;
+
+    exc_state->exc_type = NULL;
+    exc_state->exc_value = NULL;
+    exc_state->exc_traceback = NULL;
+
+    Py_XDECREF(t);
+    Py_XDECREF(v);
+    Py_XDECREF(tb);
 }
 
 #define __Pyx_Coroutine_AlreadyRunningError(gen)  (__Pyx__Coroutine_AlreadyRunningError(gen), (PyObject*)NULL)
@@ -649,6 +666,7 @@ static
 PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, int closing) {
     __Pyx_PyThreadState_declare
     PyThreadState *tstate;
+    __Pyx_ExcInfoStruct *exc_state;
     PyObject *retval;
 
     assert(!self->is_running);
@@ -671,42 +689,58 @@ PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, i
 #endif
 
     // Traceback/Frame rules:
-    // - on entry, save external exception state in self->exc_*, restore it on exit
-    // - on exit, keep internally generated exceptions in self->exc_*, clear everything else
+    // - on entry, save external exception state in self->gi_exc_state, restore it on exit
+    // - on exit, keep internally generated exceptions in self->gi_exc_state, clear everything else
     // - on entry, set "f_back" pointer of internal exception traceback to (current) outer call frame
     // - on exit, clear "f_back" of internal exception traceback
     // - do not touch external frames and tracebacks
 
-    if (self->exc_type) {
-#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON
+    exc_state = &self->gi_exc_state;
+    if (exc_state->exc_type) {
+        #if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON
         // FIXME: what to do in PyPy?
-#else
+        #else
         // Generators always return to their most recent caller, not
         // necessarily their creator.
-        if (self->exc_traceback) {
-            PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback;
+        if (exc_state->exc_traceback) {
+            PyTracebackObject *tb = (PyTracebackObject *) exc_state->exc_traceback;
             PyFrameObject *f = tb->tb_frame;
 
             Py_XINCREF(tstate->frame);
             assert(f->f_back == NULL);
             f->f_back = tstate->frame;
         }
-#endif
+        #endif
+    }
+
+#if CYTHON_USE_EXC_INFO_STACK
+    // See  https://bugs.python.org/issue25612
+    exc_state->previous_item = tstate->exc_info;
+    tstate->exc_info = exc_state;
+#else
+    if (exc_state->exc_type) {
         // We were in an except handler when we left,
         // restore the exception state which was put aside.
-        __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value,
-                            &self->exc_traceback);
+        __Pyx_ExceptionSwap(&exc_state->exc_type, &exc_state->exc_value, &exc_state->exc_traceback);
         // self->exc_* now holds the exception state of the caller
     } else {
         // save away the exception state of the caller
-        __Pyx_Coroutine_ExceptionClear(self);
-        __Pyx_ExceptionSave(&self->exc_type, &self->exc_value, &self->exc_traceback);
+        __Pyx_Coroutine_ExceptionClear(exc_state);
+        __Pyx_ExceptionSave(&exc_state->exc_type, &exc_state->exc_value, &exc_state->exc_traceback);
     }
+#endif
 
     self->is_running = 1;
     retval = self->body((PyObject *) self, tstate, value);
     self->is_running = 0;
 
+#if CYTHON_USE_EXC_INFO_STACK
+    // See  https://bugs.python.org/issue25612
+    exc_state = &self->gi_exc_state;
+    tstate->exc_info = exc_state->previous_item;
+    exc_state->previous_item = NULL;
+#endif
+
     return retval;
 }
 
@@ -714,11 +748,13 @@ static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__pyx_CoroutineO
     // Don't keep the reference to f_back any longer than necessary.  It
     // may keep a chain of frames alive or it could create a reference
     // cycle.
-    if (likely(self->exc_traceback)) {
+    PyObject *tb = self->gi_exc_state.exc_traceback;
+
+    if (likely(tb)) {
 #if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON
     // FIXME: what to do in PyPy?
 #else
-        PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback;
+        PyTracebackObject *tb = (PyTracebackObject *) tb;
         PyFrameObject *f = tb->tb_frame;
         Py_CLEAR(f->f_back);
 #endif
@@ -1039,14 +1075,18 @@ static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) {
     return __Pyx__Coroutine_Throw(self, typ, val, tb, args, 1);
 }
 
+static CYTHON_INLINE int __Pyx_Coroutine_traverse_excstate(__Pyx_ExcInfoStruct *exc_state, visitproc visit, void *arg) {
+    Py_VISIT(exc_state->exc_type);
+    Py_VISIT(exc_state->exc_value);
+    Py_VISIT(exc_state->exc_traceback);
+    return 0;
+}
+
 static int __Pyx_Coroutine_traverse(__pyx_CoroutineObject *gen, visitproc visit, void *arg) {
     Py_VISIT(gen->closure);
     Py_VISIT(gen->classobj);
     Py_VISIT(gen->yieldfrom);
-    Py_VISIT(gen->exc_type);
-    Py_VISIT(gen->exc_value);
-    Py_VISIT(gen->exc_traceback);
-    return 0;
+    return __Pyx_Coroutine_traverse_excstate(&gen->gi_exc_state, visit, arg);
 }
 
 static int __Pyx_Coroutine_clear(PyObject *self) {
@@ -1055,9 +1095,7 @@ static int __Pyx_Coroutine_clear(PyObject *self) {
     Py_CLEAR(gen->closure);
     Py_CLEAR(gen->classobj);
     Py_CLEAR(gen->yieldfrom);
-    Py_CLEAR(gen->exc_type);
-    Py_CLEAR(gen->exc_value);
-    Py_CLEAR(gen->exc_traceback);
+    __Pyx_Coroutine_ExceptionClear(&gen->gi_exc_state);
 #ifdef __Pyx_AsyncGen_USED
     if (__Pyx_AsyncGen_CheckExact(self)) {
         Py_CLEAR(((__pyx_PyAsyncGenObject*)gen)->ag_finalizer);
@@ -1315,9 +1353,12 @@ static __pyx_CoroutineObject *__Pyx__Coroutine_NewInit(
     gen->resume_label = 0;
     gen->classobj = NULL;
     gen->yieldfrom = NULL;
-    gen->exc_type = NULL;
-    gen->exc_value = NULL;
-    gen->exc_traceback = NULL;
+    gen->gi_exc_state.exc_type = NULL;
+    gen->gi_exc_state.exc_value = NULL;
+    gen->gi_exc_state.exc_traceback = NULL;
+#if CYTHON_USE_EXC_INFO_STACK
+    gen->gi_exc_state.previous_item = NULL;
+#endif
     gen->gi_weakreflist = NULL;
     Py_XINCREF(qualname);
     gen->gi_qualname = qualname;
@@ -1862,8 +1903,8 @@ static void __Pyx__ReturnWithStopIteration(PyObject* value) {
     }
     #if CYTHON_FAST_THREAD_STATE
     __Pyx_PyThreadState_assign
-    #if PY_VERSION_HEX >= 0x030700A3
-    if (!$local_tstate_cname->exc_state.exc_type)
+    #if CYTHON_USE_EXC_INFO_STACK
+    if (!$local_tstate_cname->exc_info->exc_type)
     #else
     if (!$local_tstate_cname->exc_type)
     #endif

Cython/Utility/Exceptions.c

@@ -307,6 +307,31 @@ bad:
 }
 #endif
 
+
+/////////////// GetTopmostException.proto ///////////////
+
+#if CYTHON_USE_EXC_INFO_STACK
+static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate);
+#endif
+
+/////////////// GetTopmostException ///////////////
+
+#if CYTHON_USE_EXC_INFO_STACK
+// Copied from errors.c in CPython.
+static _PyErr_StackItem *
+__Pyx_PyErr_GetTopmostException(PyThreadState *tstate)
+{
+    _PyErr_StackItem *exc_info = tstate->exc_info;
+    while ((exc_info->exc_type == NULL || exc_info->exc_type == Py_None) &&
+           exc_info->previous_item != NULL)
+    {
+        exc_info = exc_info->previous_item;
+    }
+    return exc_info;
+}
+#endif
+
+
 /////////////// GetException.proto ///////////////
 //@substitute: naming
 //@requires: PyThreadStateGet
@@ -360,13 +385,16 @@ static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb)
     *value = local_value;
     *tb = local_tb;
 #if CYTHON_FAST_THREAD_STATE
-    #if PY_VERSION_HEX >= 0x030700A3
-    tmp_type = tstate->exc_state.exc_type;
-    tmp_value = tstate->exc_state.exc_value;
-    tmp_tb = tstate->exc_state.exc_traceback;
-    tstate->exc_state.exc_type = local_type;
-    tstate->exc_state.exc_value = local_value;
-    tstate->exc_state.exc_traceback = local_tb;
+    #if CYTHON_USE_EXC_INFO_STACK
+    {
+        _PyErr_StackItem *exc_info = tstate->exc_info;
+        tmp_type = exc_info->exc_type;
+        tmp_value = exc_info->exc_value;
+        tmp_tb = exc_info->exc_traceback;
+        exc_info->exc_type = local_type;
+        exc_info->exc_value = local_value;
+        exc_info->exc_traceback = local_tb;
+    }
     #else
     tmp_type = tstate->exc_type;
     tmp_value = tstate->exc_value;
@@ -398,16 +426,18 @@ bad:
 
 static CYTHON_INLINE void __Pyx_ReraiseException(void); /*proto*/
 
-/////////////// ReRaiseException.proto ///////////////
+/////////////// ReRaiseException ///////////////
+//@requires: GetTopmostException
 
 static CYTHON_INLINE void __Pyx_ReraiseException(void) {
     PyObject *type = NULL, *value = NULL, *tb = NULL;
 #if CYTHON_FAST_THREAD_STATE
     PyThreadState *tstate = PyThreadState_GET();
-    #if PY_VERSION_HEX >= 0x030700A3
-    type = tstate->exc_state.exc_type;
-    value = tstate->exc_state.exc_value;
-    tb = tstate->exc_state.exc_traceback;
+    #if CYTHON_USE_EXC_INFO_STACK
+    _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate);
+    type = exc_info->exc_type;
+    value = exc_info->exc_value;
+    tb = exc_info->exc_traceback;
     #else
     type = tstate->exc_type;
     value = tstate->exc_value;
@@ -456,10 +486,11 @@ static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject
 
 #if CYTHON_FAST_THREAD_STATE
 static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) {
-    #if PY_VERSION_HEX >= 0x030700A3
-    *type = tstate->exc_state.exc_type;
-    *value = tstate->exc_state.exc_value;
-    *tb = tstate->exc_state.exc_traceback;
+    #if CYTHON_USE_EXC_INFO_STACK
+    _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate);
+    *type = exc_info->exc_type;
+    *value = exc_info->exc_value;
+    *tb = exc_info->exc_traceback;
     #else
     *type = tstate->exc_type;
     *value = tstate->exc_value;
@@ -473,13 +504,14 @@ static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject *
 static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) {
     PyObject *tmp_type, *tmp_value, *tmp_tb;
 
-    #if PY_VERSION_HEX >= 0x030700A3
-    tmp_type = tstate->exc_state.exc_type;
-    tmp_value = tstate->exc_state.exc_value;
-    tmp_tb = tstate->exc_state.exc_traceback;
-    tstate->exc_state.exc_type = type;
-    tstate->exc_state.exc_value = value;
-    tstate->exc_state.exc_traceback = tb;
+    #if CYTHON_USE_EXC_INFO_STACK
+    _PyErr_StackItem *exc_info = tstate->exc_info;
+    tmp_type = exc_info->exc_type;
+    tmp_value = exc_info->exc_value;
+    tmp_tb = exc_info->exc_traceback;
+    exc_info->exc_type = type;
+    exc_info->exc_value = value;
+    exc_info->exc_traceback = tb;
     #else
     tmp_type = tstate->exc_type;
     tmp_value = tstate->exc_value;
@@ -511,14 +543,15 @@ static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value,
 static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) {
     PyObject *tmp_type, *tmp_value, *tmp_tb;
 
-    #if PY_VERSION_HEX >= 0x030700A3
-    tmp_type = tstate->exc_state.exc_type;
-    tmp_value = tstate->exc_state.exc_value;
-    tmp_tb = tstate->exc_state.exc_traceback;
+    #if CYTHON_USE_EXC_INFO_STACK
+    _PyErr_StackItem *exc_info = tstate->exc_info;
+    tmp_type = exc_info->exc_type;
+    tmp_value = exc_info->exc_value;
+    tmp_tb = exc_info->exc_traceback;
 
-    tstate->exc_state.exc_type = *type;
-    tstate->exc_state.exc_value = *value;
-    tstate->exc_state.exc_traceback = *tb;
+    exc_info->exc_type = *type;
+    exc_info->exc_value = *value;
+    exc_info->exc_traceback = *tb;
     #else
     tmp_type = tstate->exc_type;
     tmp_value = tstate->exc_value;

Cython/Utility/ModuleSetupCode.c

@@ -81,6 +81,8 @@
   #define CYTHON_USE_TP_FINALIZE 0
   #undef CYTHON_USE_DICT_VERSIONS
   #define CYTHON_USE_DICT_VERSIONS 0
+  #undef CYTHON_USE_EXC_INFO_STACK
+  #define CYTHON_USE_EXC_INFO_STACK 0
 
 #elif defined(PYSTON_VERSION)
   #define CYTHON_COMPILING_IN_PYPY 0
@@ -122,6 +124,8 @@
   #define CYTHON_USE_TP_FINALIZE 0
   #undef CYTHON_USE_DICT_VERSIONS
   #define CYTHON_USE_DICT_VERSIONS 0
+  #undef CYTHON_USE_EXC_INFO_STACK
+  #define CYTHON_USE_EXC_INFO_STACK 0
 
 #else
   #define CYTHON_COMPILING_IN_PYPY 0
@@ -186,6 +190,9 @@
   #ifndef CYTHON_USE_DICT_VERSIONS
     #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX >= 0x030600B1)
   #endif
+  #ifndef CYTHON_USE_EXC_INFO_STACK
+    #define CYTHON_USE_EXC_INFO_STACK (PY_VERSION_HEX >= 0x030700A3)
+  #endif
 #endif
 
 #if !defined(CYTHON_FAST_PYCCALL)

tests/run/test_exceptions.pyx

 # Python test set -- part 5, built-in exceptions
+# Copied from CPython 3.7.
+
+# cython: language_level=3
+# mode: run
+# tag: generator, exception, tryfinally, tryexcept
 
 import copy
 import os
@@ -12,6 +17,12 @@ from test.support import (TESTFN, captured_stderr, check_impl_detail,
                           check_warnings, cpython_only, gc_collect, run_unittest,
                           no_tracing, unlink, import_module, script_helper,
                           SuppressCrashReport)
+
+no_tracing = unittest.skip("For nested functions, Cython generates a C call without recursion checks.")
+
+cpython_only = unittest.skip("Tests for _testcapi make no sense here.")
+
+
 class NaiveException(Exception):
     def __init__(self, x):
         self.x = x
@@ -80,8 +91,8 @@ class ExceptionTests(unittest.TestCase):
         self.raise_catch(MemoryError, "MemoryError")
 
         self.raise_catch(NameError, "NameError")
-        try: x = undefined_variable
-        except NameError: pass
+        #try: x = undefined_variable
+        #except NameError: pass
 
         self.raise_catch(OverflowError, "OverflowError")
         x = 1
@@ -649,16 +660,19 @@ class ExceptionTests(unittest.TestCase):
         obj = wr()
         self.assertIsNone(obj)
 
+    '''
+    # This is currently a compile error in Cython-3, although it works in Cython-2 => could also work in Cy-3.
     def test_exception_target_in_nested_scope(self):
         # issue 4617: This used to raise a SyntaxError
         # "can not delete variable 'e' referenced in nested scope"
         def print_error():
             e
         try:
-            something
+            1/0
         except Exception as e:
             print_error()
             # implicit "del e" here
+    '''
 
     def test_generator_leaking(self):
         # Test that generator exception state doesn't leak into the calling