work around crash in CPython's yield-from implementation for generators that...

work around crash in CPython's yield-from implementation for generators that "return" values using a StopIteration exception valu

CHANGES.rst

@@ -44,6 +44,9 @@ Features added
 Bugs fixed
 ----------
 
+* Calling "yield from" from Python on a Cython generator that returned a value
+  triggered a crash in CPython (issue 23996).  This is now being worked around.
+
 * Language level 3 did not enable true division (a.k.a. float division) for
   integer operands.
 

Cython/Compiler/Nodes.py

@@ -5503,7 +5503,10 @@ class ReturnStatNode(StatNode):
                 # return value == raise StopIteration(value), but uncatchable
                 code.putln("%s = NULL;" % Naming.retval_cname)
                 if not self.value.is_none:
-                    code.putln("PyErr_SetObject(PyExc_StopIteration, %s);" % (
+                    # CPython 3.3+ crashes in yield-from when the StopIteration is not instantiated
+                    code.globalstate.use_utility_code(
+                        UtilityCode.load_cached("ReturnWithStopIteration", "Generator.c"))
+                    code.putln("__Pyx_ReturnWithStopIteration(%s);" % (
                         self.value.py_result()))
                 self.value.generate_disposal_code(code)
             else:

Cython/Utility/Generator.c

@@ -751,6 +751,34 @@ static int __pyx_Generator_init(void) {
 }
 
 
+/////////////// ReturnWithStopIteration.proto ///////////////
+
+#if CYTHON_COMPILING_IN_CPYTHON
+// CPython 3.3+ crashes in yield-from when the StopIteration is not instantiated
+#define __Pyx_ReturnWithStopIteration(value)  if (value == Py_None); else __Pyx__ReturnWithStopIteration(value)
+static void __Pyx__ReturnWithStopIteration(PyObject* value); /*proto*/
+#else
+#define __Pyx_ReturnWithStopIteration(value)  PyErr_SetObject(PyExc_StopIteration, value)
+#endif
+
+/////////////// ReturnWithStopIteration ///////////////
+
+#if CYTHON_COMPILING_IN_CPYTHON
+static void __Pyx__ReturnWithStopIteration(PyObject* value) {
+    PyObject *exc, *args;
+    args = PyTuple_New(1);
+    if (!args) return;
+    Py_INCREF(value);
+    PyTuple_SET_ITEM(args, 0, value);
+    exc = PyObject_Call(PyExc_StopIteration, args, NULL);
+    Py_DECREF(args);
+    if (!exc) return;
+    Py_INCREF(PyExc_StopIteration);
+    PyErr_Restore(PyExc_StopIteration, exc, NULL);
+}
+#endif
+
+
 //////////////////// PatchModuleWithGenerator.proto ////////////////////
 
 #ifdef __Pyx_Generator_USED

runtests.py

@@ -329,6 +329,7 @@ VER_DEP_MODULES = {
     (3,1): (_is_py3_before_32, lambda x: x in ['run.pyclass_special_methods',
                                                ]),
     (3,3) : (operator.lt, lambda x: x in ['build.package_compilation',
+                                          'yield_from_py33',
                                           ]),
     (3,4,0,'beta',3) : (operator.le, lambda x: x in ['run.py34_signature',
                                           ]),